1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/indir_table.h" 42 #include "esw/acl/ofld.h" 43 #include "esw/indir_table.h" 44 #include "rdma.h" 45 #include "en.h" 46 #include "fs_core.h" 47 #include "lib/devcom.h" 48 #include "lib/eq.h" 49 #include "lib/fs_chains.h" 50 #include "en_tc.h" 51 52 /* There are two match-all miss flows, one for unicast dst mac and 53 * one for multicast. 54 */ 55 #define MLX5_ESW_MISS_FLOWS (2) 56 #define UPLINK_REP_INDEX 0 57 58 /* Per vport tables */ 59 60 #define MLX5_ESW_VPORT_TABLE_SIZE 128 61 62 /* This struct is used as a key to the hash table and we need it to be packed 63 * so hash result is consistent 64 */ 65 struct mlx5_vport_key { 66 u32 chain; 67 u16 prio; 68 u16 vport; 69 u16 vhca_id; 70 } __packed; 71 72 struct mlx5_vport_tbl_attr { 73 u16 chain; 74 u16 prio; 75 u16 vport; 76 }; 77 78 struct mlx5_vport_table { 79 struct hlist_node hlist; 80 struct mlx5_flow_table *fdb; 81 u32 num_rules; 82 struct mlx5_vport_key key; 83 }; 84 85 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 86 87 static struct mlx5_flow_table * 88 esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns) 89 { 90 struct mlx5_flow_table_attr ft_attr = {}; 91 struct mlx5_flow_table *fdb; 92 93 ft_attr.autogroup.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS; 94 ft_attr.max_fte = MLX5_ESW_VPORT_TABLE_SIZE; 95 ft_attr.prio = FDB_PER_VPORT; 96 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 97 if (IS_ERR(fdb)) { 98 esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n", 99 PTR_ERR(fdb)); 100 } 101 102 return fdb; 103 } 104 105 static u32 flow_attr_to_vport_key(struct mlx5_eswitch *esw, 106 struct mlx5_vport_tbl_attr *attr, 107 struct mlx5_vport_key *key) 108 { 109 key->vport = attr->vport; 110 key->chain = attr->chain; 111 key->prio = attr->prio; 112 key->vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 113 return jhash(key, sizeof(*key), 0); 114 } 115 116 /* caller must hold vports.lock */ 117 static struct mlx5_vport_table * 118 esw_vport_tbl_lookup(struct mlx5_eswitch *esw, struct mlx5_vport_key *skey, u32 key) 119 { 120 struct mlx5_vport_table *e; 121 122 hash_for_each_possible(esw->fdb_table.offloads.vports.table, e, hlist, key) 123 if (!memcmp(&e->key, skey, sizeof(*skey))) 124 return e; 125 126 return NULL; 127 } 128 129 static void 130 esw_vport_tbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr) 131 { 132 struct mlx5_vport_table *e; 133 struct mlx5_vport_key key; 134 u32 hkey; 135 136 mutex_lock(&esw->fdb_table.offloads.vports.lock); 137 hkey = flow_attr_to_vport_key(esw, attr, &key); 138 e = esw_vport_tbl_lookup(esw, &key, hkey); 139 if (!e || --e->num_rules) 140 goto out; 141 142 hash_del(&e->hlist); 143 mlx5_destroy_flow_table(e->fdb); 144 kfree(e); 145 out: 146 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 147 } 148 149 static struct mlx5_flow_table * 150 esw_vport_tbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr) 151 { 152 struct mlx5_core_dev *dev = esw->dev; 153 struct mlx5_flow_namespace *ns; 154 struct mlx5_flow_table *fdb; 155 struct mlx5_vport_table *e; 156 struct mlx5_vport_key skey; 157 u32 hkey; 158 159 mutex_lock(&esw->fdb_table.offloads.vports.lock); 160 hkey = flow_attr_to_vport_key(esw, attr, &skey); 161 e = esw_vport_tbl_lookup(esw, &skey, hkey); 162 if (e) { 163 e->num_rules++; 164 goto out; 165 } 166 167 e = kzalloc(sizeof(*e), GFP_KERNEL); 168 if (!e) { 169 fdb = ERR_PTR(-ENOMEM); 170 goto err_alloc; 171 } 172 173 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 174 if (!ns) { 175 esw_warn(dev, "Failed to get FDB namespace\n"); 176 fdb = ERR_PTR(-ENOENT); 177 goto err_ns; 178 } 179 180 fdb = esw_vport_tbl_create(esw, ns); 181 if (IS_ERR(fdb)) 182 goto err_ns; 183 184 e->fdb = fdb; 185 e->num_rules = 1; 186 e->key = skey; 187 hash_add(esw->fdb_table.offloads.vports.table, &e->hlist, hkey); 188 out: 189 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 190 return e->fdb; 191 192 err_ns: 193 kfree(e); 194 err_alloc: 195 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 196 return fdb; 197 } 198 199 int mlx5_esw_vport_tbl_get(struct mlx5_eswitch *esw) 200 { 201 struct mlx5_vport_tbl_attr attr; 202 struct mlx5_flow_table *fdb; 203 struct mlx5_vport *vport; 204 int i; 205 206 attr.chain = 0; 207 attr.prio = 1; 208 mlx5_esw_for_all_vports(esw, i, vport) { 209 attr.vport = vport->vport; 210 fdb = esw_vport_tbl_get(esw, &attr); 211 if (IS_ERR(fdb)) 212 goto out; 213 } 214 return 0; 215 216 out: 217 mlx5_esw_vport_tbl_put(esw); 218 return PTR_ERR(fdb); 219 } 220 221 void mlx5_esw_vport_tbl_put(struct mlx5_eswitch *esw) 222 { 223 struct mlx5_vport_tbl_attr attr; 224 struct mlx5_vport *vport; 225 int i; 226 227 attr.chain = 0; 228 attr.prio = 1; 229 mlx5_esw_for_all_vports(esw, i, vport) { 230 attr.vport = vport->vport; 231 esw_vport_tbl_put(esw, &attr); 232 } 233 } 234 235 /* End: Per vport tables */ 236 237 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 238 u16 vport_num) 239 { 240 int idx = mlx5_eswitch_vport_num_to_index(esw, vport_num); 241 242 WARN_ON(idx > esw->total_vports - 1); 243 return &esw->offloads.vport_reps[idx]; 244 } 245 246 static void 247 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 248 struct mlx5_flow_spec *spec, 249 struct mlx5_esw_flow_attr *attr) 250 { 251 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) && 252 attr && attr->in_rep) 253 spec->flow_context.flow_source = 254 attr->in_rep->vport == MLX5_VPORT_UPLINK ? 255 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 256 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 257 } 258 259 static void 260 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 261 struct mlx5_flow_spec *spec, 262 struct mlx5_flow_attr *attr, 263 struct mlx5_eswitch *src_esw, 264 u16 vport) 265 { 266 void *misc2; 267 void *misc; 268 269 /* Use metadata matching because vport is not represented by single 270 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 271 */ 272 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 273 if (mlx5_esw_indir_table_decap_vport(attr)) 274 vport = mlx5_esw_indir_table_decap_vport(attr); 275 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 276 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 277 mlx5_eswitch_get_vport_metadata_for_match(src_esw, 278 vport)); 279 280 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 281 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 282 mlx5_eswitch_get_vport_metadata_mask()); 283 284 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 285 } else { 286 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 287 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 288 289 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 290 MLX5_SET(fte_match_set_misc, misc, 291 source_eswitch_owner_vhca_id, 292 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 293 294 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 295 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 296 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 297 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 298 source_eswitch_owner_vhca_id); 299 300 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 301 } 302 } 303 304 static int 305 esw_setup_decap_indir(struct mlx5_eswitch *esw, 306 struct mlx5_flow_attr *attr, 307 struct mlx5_flow_spec *spec) 308 { 309 struct mlx5_flow_table *ft; 310 311 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE)) 312 return -EOPNOTSUPP; 313 314 ft = mlx5_esw_indir_table_get(esw, attr, spec, 315 mlx5_esw_indir_table_decap_vport(attr), true); 316 return PTR_ERR_OR_ZERO(ft); 317 } 318 319 static void 320 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 321 struct mlx5_flow_attr *attr) 322 { 323 if (mlx5_esw_indir_table_decap_vport(attr)) 324 mlx5_esw_indir_table_put(esw, attr, 325 mlx5_esw_indir_table_decap_vport(attr), 326 true); 327 } 328 329 static int 330 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 331 struct mlx5_flow_act *flow_act, 332 struct mlx5_eswitch *esw, 333 struct mlx5_flow_attr *attr, 334 struct mlx5_flow_spec *spec, 335 int i) 336 { 337 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 338 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 339 dest[i].ft = attr->dest_ft; 340 341 if (mlx5_esw_indir_table_decap_vport(attr)) 342 return esw_setup_decap_indir(esw, attr, spec); 343 return 0; 344 } 345 346 static void 347 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, 348 struct mlx5_flow_act *flow_act, 349 struct mlx5_fs_chains *chains, 350 int i) 351 { 352 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 353 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 354 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 355 } 356 357 static int 358 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 359 struct mlx5_flow_act *flow_act, 360 struct mlx5_fs_chains *chains, 361 u32 chain, u32 prio, u32 level, 362 int i) 363 { 364 struct mlx5_flow_table *ft; 365 366 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 367 ft = mlx5_chains_get_table(chains, chain, prio, level); 368 if (IS_ERR(ft)) 369 return PTR_ERR(ft); 370 371 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 372 dest[i].ft = ft; 373 return 0; 374 } 375 376 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 377 int from, int to) 378 { 379 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 380 struct mlx5_fs_chains *chains = esw_chains(esw); 381 int i; 382 383 for (i = from; i < to; i++) 384 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 385 mlx5_chains_put_table(chains, 0, 1, 0); 386 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 387 esw_attr->dests[i].mdev)) 388 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport, 389 false); 390 } 391 392 static bool 393 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 394 { 395 int i; 396 397 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 398 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 399 return true; 400 return false; 401 } 402 403 static int 404 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 405 struct mlx5_flow_act *flow_act, 406 struct mlx5_eswitch *esw, 407 struct mlx5_fs_chains *chains, 408 struct mlx5_flow_attr *attr, 409 int *i) 410 { 411 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 412 int j, err; 413 414 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE)) 415 return -EOPNOTSUPP; 416 417 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 418 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 419 if (err) 420 goto err_setup_chain; 421 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 422 flow_act->pkt_reformat = esw_attr->dests[j].pkt_reformat; 423 } 424 return 0; 425 426 err_setup_chain: 427 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 428 return err; 429 } 430 431 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 432 struct mlx5_flow_attr *attr) 433 { 434 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 435 436 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 437 } 438 439 static bool 440 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 441 { 442 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 443 int i; 444 445 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 446 if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 447 esw_attr->dests[i].mdev)) 448 return true; 449 return false; 450 } 451 452 static int 453 esw_setup_indir_table(struct mlx5_flow_destination *dest, 454 struct mlx5_flow_act *flow_act, 455 struct mlx5_eswitch *esw, 456 struct mlx5_flow_attr *attr, 457 struct mlx5_flow_spec *spec, 458 bool ignore_flow_lvl, 459 int *i) 460 { 461 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 462 int j, err; 463 464 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE)) 465 return -EOPNOTSUPP; 466 467 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 468 if (ignore_flow_lvl) 469 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 470 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 471 472 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec, 473 esw_attr->dests[j].rep->vport, false); 474 if (IS_ERR(dest[*i].ft)) { 475 err = PTR_ERR(dest[*i].ft); 476 goto err_indir_tbl_get; 477 } 478 } 479 480 if (mlx5_esw_indir_table_decap_vport(attr)) { 481 err = esw_setup_decap_indir(esw, attr, spec); 482 if (err) 483 goto err_indir_tbl_get; 484 } 485 486 return 0; 487 488 err_indir_tbl_get: 489 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 490 return err; 491 } 492 493 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 494 { 495 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 496 497 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 498 esw_cleanup_decap_indir(esw, attr); 499 } 500 501 static void 502 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 503 { 504 mlx5_chains_put_table(chains, chain, prio, level); 505 } 506 507 static void 508 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 509 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 510 int attr_idx, int dest_idx, bool pkt_reformat) 511 { 512 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 513 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport; 514 dest[dest_idx].vport.vhca_id = 515 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 516 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 517 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 518 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP) { 519 if (pkt_reformat) { 520 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 521 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 522 } 523 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 524 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 525 } 526 } 527 528 static int 529 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 530 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 531 int i) 532 { 533 int j; 534 535 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 536 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 537 return i; 538 } 539 540 static int 541 esw_setup_dests(struct mlx5_flow_destination *dest, 542 struct mlx5_flow_act *flow_act, 543 struct mlx5_eswitch *esw, 544 struct mlx5_flow_attr *attr, 545 struct mlx5_flow_spec *spec, 546 int *i) 547 { 548 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 549 struct mlx5_fs_chains *chains = esw_chains(esw); 550 int err = 0; 551 552 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 553 MLX5_CAP_GEN(esw_attr->in_mdev, reg_c_preserve) && 554 mlx5_eswitch_vport_match_metadata_enabled(esw)) 555 attr->flags |= MLX5_ESW_ATTR_FLAG_SRC_REWRITE; 556 557 if (attr->dest_ft) { 558 esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i); 559 (*i)++; 560 } else if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) { 561 esw_setup_slow_path_dest(dest, flow_act, chains, *i); 562 (*i)++; 563 } else if (attr->dest_chain) { 564 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 565 1, 0, *i); 566 (*i)++; 567 } else if (esw_is_indir_table(esw, attr)) { 568 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i); 569 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 570 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 571 } else { 572 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 573 } 574 575 return err; 576 } 577 578 static void 579 esw_cleanup_dests(struct mlx5_eswitch *esw, 580 struct mlx5_flow_attr *attr) 581 { 582 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 583 struct mlx5_fs_chains *chains = esw_chains(esw); 584 585 if (attr->dest_ft) { 586 esw_cleanup_decap_indir(esw, attr); 587 } else if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) { 588 if (attr->dest_chain) 589 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 590 else if (esw_is_indir_table(esw, attr)) 591 esw_cleanup_indir_table(esw, attr); 592 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 593 esw_cleanup_chain_src_port_rewrite(esw, attr); 594 } 595 } 596 597 struct mlx5_flow_handle * 598 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 599 struct mlx5_flow_spec *spec, 600 struct mlx5_flow_attr *attr) 601 { 602 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {}; 603 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 604 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 605 struct mlx5_fs_chains *chains = esw_chains(esw); 606 bool split = !!(esw_attr->split_count); 607 struct mlx5_vport_tbl_attr fwd_attr; 608 struct mlx5_flow_handle *rule; 609 struct mlx5_flow_table *fdb; 610 int i = 0; 611 612 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 613 return ERR_PTR(-EOPNOTSUPP); 614 615 flow_act.action = attr->action; 616 /* if per flow vlan pop/push is emulated, don't set that into the firmware */ 617 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 618 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | 619 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 620 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 621 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 622 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 623 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 624 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 625 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 626 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 627 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 628 } 629 } 630 631 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 632 633 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 634 int err; 635 636 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 637 if (err) { 638 rule = ERR_PTR(err); 639 goto err_create_goto_table; 640 } 641 } 642 643 if (esw_attr->decap_pkt_reformat) 644 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 645 646 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 647 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 648 dest[i].counter_id = mlx5_fc_id(attr->counter); 649 i++; 650 } 651 652 if (attr->outer_match_level != MLX5_MATCH_NONE) 653 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 654 if (attr->inner_match_level != MLX5_MATCH_NONE) 655 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 656 657 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 658 flow_act.modify_hdr = attr->modify_hdr; 659 660 if (split) { 661 fwd_attr.chain = attr->chain; 662 fwd_attr.prio = attr->prio; 663 fwd_attr.vport = esw_attr->in_rep->vport; 664 665 fdb = esw_vport_tbl_get(esw, &fwd_attr); 666 } else { 667 if (attr->chain || attr->prio) 668 fdb = mlx5_chains_get_table(chains, attr->chain, 669 attr->prio, 0); 670 else 671 fdb = attr->ft; 672 673 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_NO_IN_PORT)) 674 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 675 esw_attr->in_mdev->priv.eswitch, 676 esw_attr->in_rep->vport); 677 } 678 if (IS_ERR(fdb)) { 679 rule = ERR_CAST(fdb); 680 goto err_esw_get; 681 } 682 683 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 684 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 685 &flow_act, dest, i); 686 else 687 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 688 if (IS_ERR(rule)) 689 goto err_add_rule; 690 else 691 atomic64_inc(&esw->offloads.num_flows); 692 693 return rule; 694 695 err_add_rule: 696 if (split) 697 esw_vport_tbl_put(esw, &fwd_attr); 698 else if (attr->chain || attr->prio) 699 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 700 err_esw_get: 701 esw_cleanup_dests(esw, attr); 702 err_create_goto_table: 703 return rule; 704 } 705 706 struct mlx5_flow_handle * 707 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 708 struct mlx5_flow_spec *spec, 709 struct mlx5_flow_attr *attr) 710 { 711 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {}; 712 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 713 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 714 struct mlx5_fs_chains *chains = esw_chains(esw); 715 struct mlx5_vport_tbl_attr fwd_attr; 716 struct mlx5_flow_table *fast_fdb; 717 struct mlx5_flow_table *fwd_fdb; 718 struct mlx5_flow_handle *rule; 719 int i, err = 0; 720 721 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 722 if (IS_ERR(fast_fdb)) { 723 rule = ERR_CAST(fast_fdb); 724 goto err_get_fast; 725 } 726 727 fwd_attr.chain = attr->chain; 728 fwd_attr.prio = attr->prio; 729 fwd_attr.vport = esw_attr->in_rep->vport; 730 fwd_fdb = esw_vport_tbl_get(esw, &fwd_attr); 731 if (IS_ERR(fwd_fdb)) { 732 rule = ERR_CAST(fwd_fdb); 733 goto err_get_fwd; 734 } 735 736 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 737 for (i = 0; i < esw_attr->split_count; i++) { 738 if (esw_is_indir_table(esw, attr)) 739 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i); 740 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 741 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr, 742 &i); 743 else 744 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 745 746 if (err) { 747 rule = ERR_PTR(err); 748 goto err_chain_src_rewrite; 749 } 750 } 751 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 752 dest[i].ft = fwd_fdb; 753 i++; 754 755 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 756 esw_attr->in_mdev->priv.eswitch, 757 esw_attr->in_rep->vport); 758 759 if (attr->outer_match_level != MLX5_MATCH_NONE) 760 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 761 762 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 763 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 764 765 if (IS_ERR(rule)) { 766 i = esw_attr->split_count; 767 goto err_chain_src_rewrite; 768 } 769 770 atomic64_inc(&esw->offloads.num_flows); 771 772 return rule; 773 err_chain_src_rewrite: 774 esw_put_dest_tables_loop(esw, attr, 0, i); 775 esw_vport_tbl_put(esw, &fwd_attr); 776 err_get_fwd: 777 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 778 err_get_fast: 779 return rule; 780 } 781 782 static void 783 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 784 struct mlx5_flow_handle *rule, 785 struct mlx5_flow_attr *attr, 786 bool fwd_rule) 787 { 788 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 789 struct mlx5_fs_chains *chains = esw_chains(esw); 790 bool split = (esw_attr->split_count > 0); 791 struct mlx5_vport_tbl_attr fwd_attr; 792 int i; 793 794 mlx5_del_flow_rules(rule); 795 796 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) { 797 /* unref the term table */ 798 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 799 if (esw_attr->dests[i].termtbl) 800 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 801 } 802 } 803 804 atomic64_dec(&esw->offloads.num_flows); 805 806 if (fwd_rule || split) { 807 fwd_attr.chain = attr->chain; 808 fwd_attr.prio = attr->prio; 809 fwd_attr.vport = esw_attr->in_rep->vport; 810 } 811 812 if (fwd_rule) { 813 esw_vport_tbl_put(esw, &fwd_attr); 814 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 815 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count); 816 } else { 817 if (split) 818 esw_vport_tbl_put(esw, &fwd_attr); 819 else if (attr->chain || attr->prio) 820 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 821 esw_cleanup_dests(esw, attr); 822 } 823 } 824 825 void 826 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 827 struct mlx5_flow_handle *rule, 828 struct mlx5_flow_attr *attr) 829 { 830 __mlx5_eswitch_del_rule(esw, rule, attr, false); 831 } 832 833 void 834 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 835 struct mlx5_flow_handle *rule, 836 struct mlx5_flow_attr *attr) 837 { 838 __mlx5_eswitch_del_rule(esw, rule, attr, true); 839 } 840 841 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) 842 { 843 struct mlx5_eswitch_rep *rep; 844 int i, err = 0; 845 846 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); 847 mlx5_esw_for_each_host_func_rep(esw, i, rep, esw->esw_funcs.num_vfs) { 848 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 849 continue; 850 851 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); 852 if (err) 853 goto out; 854 } 855 856 out: 857 return err; 858 } 859 860 static struct mlx5_eswitch_rep * 861 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) 862 { 863 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; 864 865 in_rep = attr->in_rep; 866 out_rep = attr->dests[0].rep; 867 868 if (push) 869 vport = in_rep; 870 else if (pop) 871 vport = out_rep; 872 else 873 vport = in_rep; 874 875 return vport; 876 } 877 878 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, 879 bool push, bool pop, bool fwd) 880 { 881 struct mlx5_eswitch_rep *in_rep, *out_rep; 882 883 if ((push || pop) && !fwd) 884 goto out_notsupp; 885 886 in_rep = attr->in_rep; 887 out_rep = attr->dests[0].rep; 888 889 if (push && in_rep->vport == MLX5_VPORT_UPLINK) 890 goto out_notsupp; 891 892 if (pop && out_rep->vport == MLX5_VPORT_UPLINK) 893 goto out_notsupp; 894 895 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ 896 if (!push && !pop && fwd) 897 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK) 898 goto out_notsupp; 899 900 /* protects against (1) setting rules with different vlans to push and 901 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) 902 */ 903 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0])) 904 goto out_notsupp; 905 906 return 0; 907 908 out_notsupp: 909 return -EOPNOTSUPP; 910 } 911 912 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, 913 struct mlx5_flow_attr *attr) 914 { 915 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 916 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 917 struct mlx5_eswitch_rep *vport = NULL; 918 bool push, pop, fwd; 919 int err = 0; 920 921 /* nop if we're on the vlan push/pop non emulation mode */ 922 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 923 return 0; 924 925 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 926 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 927 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && 928 !attr->dest_chain); 929 930 mutex_lock(&esw->state_lock); 931 932 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd); 933 if (err) 934 goto unlock; 935 936 attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 937 938 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 939 940 if (!push && !pop && fwd) { 941 /* tracks VF --> wire rules without vlan push action */ 942 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) { 943 vport->vlan_refcount++; 944 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 945 } 946 947 goto unlock; 948 } 949 950 if (!push && !pop) 951 goto unlock; 952 953 if (!(offloads->vlan_push_pop_refcount)) { 954 /* it's the 1st vlan rule, apply global vlan pop policy */ 955 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); 956 if (err) 957 goto out; 958 } 959 offloads->vlan_push_pop_refcount++; 960 961 if (push) { 962 if (vport->vlan_refcount) 963 goto skip_set_push; 964 965 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0], 966 0, SET_VLAN_INSERT | SET_VLAN_STRIP); 967 if (err) 968 goto out; 969 vport->vlan = esw_attr->vlan_vid[0]; 970 skip_set_push: 971 vport->vlan_refcount++; 972 } 973 out: 974 if (!err) 975 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 976 unlock: 977 mutex_unlock(&esw->state_lock); 978 return err; 979 } 980 981 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, 982 struct mlx5_flow_attr *attr) 983 { 984 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 985 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 986 struct mlx5_eswitch_rep *vport = NULL; 987 bool push, pop, fwd; 988 int err = 0; 989 990 /* nop if we're on the vlan push/pop non emulation mode */ 991 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 992 return 0; 993 994 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED)) 995 return 0; 996 997 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 998 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 999 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 1000 1001 mutex_lock(&esw->state_lock); 1002 1003 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 1004 1005 if (!push && !pop && fwd) { 1006 /* tracks VF --> wire rules without vlan push action */ 1007 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) 1008 vport->vlan_refcount--; 1009 1010 goto out; 1011 } 1012 1013 if (push) { 1014 vport->vlan_refcount--; 1015 if (vport->vlan_refcount) 1016 goto skip_unset_push; 1017 1018 vport->vlan = 0; 1019 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, 1020 0, 0, SET_VLAN_STRIP); 1021 if (err) 1022 goto out; 1023 } 1024 1025 skip_unset_push: 1026 offloads->vlan_push_pop_refcount--; 1027 if (offloads->vlan_push_pop_refcount) 1028 goto out; 1029 1030 /* no more vlan rules, stop global vlan pop policy */ 1031 err = esw_set_global_vlan_pop(esw, 0); 1032 1033 out: 1034 mutex_unlock(&esw->state_lock); 1035 return err; 1036 } 1037 1038 struct mlx5_flow_handle * 1039 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, u16 vport, 1040 u32 sqn) 1041 { 1042 struct mlx5_flow_act flow_act = {0}; 1043 struct mlx5_flow_destination dest = {}; 1044 struct mlx5_flow_handle *flow_rule; 1045 struct mlx5_flow_spec *spec; 1046 void *misc; 1047 1048 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1049 if (!spec) { 1050 flow_rule = ERR_PTR(-ENOMEM); 1051 goto out; 1052 } 1053 1054 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1055 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 1056 /* source vport is the esw manager */ 1057 MLX5_SET(fte_match_set_misc, misc, source_port, esw->manager_vport); 1058 1059 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1060 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 1061 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1062 1063 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1064 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1065 dest.vport.num = vport; 1066 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1067 1068 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1069 spec, &flow_act, &dest, 1); 1070 if (IS_ERR(flow_rule)) 1071 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule)); 1072 out: 1073 kvfree(spec); 1074 return flow_rule; 1075 } 1076 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1077 1078 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1079 { 1080 mlx5_del_flow_rules(rule); 1081 } 1082 1083 static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) 1084 { 1085 struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules; 1086 int i = 0, num_vfs = esw->esw_funcs.num_vfs, vport_num; 1087 1088 if (!num_vfs || !flows) 1089 return; 1090 1091 mlx5_esw_for_each_vf_vport_num(esw, vport_num, num_vfs) 1092 mlx5_del_flow_rules(flows[i++]); 1093 1094 kvfree(flows); 1095 } 1096 1097 static int 1098 mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw) 1099 { 1100 int num_vfs, vport_num, rule_idx = 0, err = 0; 1101 struct mlx5_flow_destination dest = {}; 1102 struct mlx5_flow_act flow_act = {0}; 1103 struct mlx5_flow_handle *flow_rule; 1104 struct mlx5_flow_handle **flows; 1105 struct mlx5_flow_spec *spec; 1106 1107 num_vfs = esw->esw_funcs.num_vfs; 1108 flows = kvzalloc(num_vfs * sizeof(*flows), GFP_KERNEL); 1109 if (!flows) 1110 return -ENOMEM; 1111 1112 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1113 if (!spec) { 1114 err = -ENOMEM; 1115 goto alloc_err; 1116 } 1117 1118 MLX5_SET(fte_match_param, spec->match_criteria, 1119 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1120 MLX5_SET(fte_match_param, spec->match_criteria, 1121 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1122 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1123 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1124 1125 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1126 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1127 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1128 1129 mlx5_esw_for_each_vf_vport_num(esw, vport_num, num_vfs) { 1130 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1131 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1132 dest.vport.num = vport_num; 1133 1134 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1135 spec, &flow_act, &dest, 1); 1136 if (IS_ERR(flow_rule)) { 1137 err = PTR_ERR(flow_rule); 1138 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n", 1139 rule_idx, PTR_ERR(flow_rule)); 1140 goto rule_err; 1141 } 1142 flows[rule_idx++] = flow_rule; 1143 } 1144 1145 esw->fdb_table.offloads.send_to_vport_meta_rules = flows; 1146 kvfree(spec); 1147 return 0; 1148 1149 rule_err: 1150 while (--rule_idx >= 0) 1151 mlx5_del_flow_rules(flows[rule_idx]); 1152 kvfree(spec); 1153 alloc_err: 1154 kvfree(flows); 1155 return err; 1156 } 1157 1158 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1159 { 1160 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1161 MLX5_FDB_TO_VPORT_REG_C_1; 1162 } 1163 1164 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1165 { 1166 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1167 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1168 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1169 u8 curr, wanted; 1170 int err; 1171 1172 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1173 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1174 return 0; 1175 1176 MLX5_SET(query_esw_vport_context_in, in, opcode, 1177 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1178 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1179 if (err) 1180 return err; 1181 1182 curr = MLX5_GET(query_esw_vport_context_out, out, 1183 esw_vport_context.fdb_to_vport_reg_c_id); 1184 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1185 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1186 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1187 1188 if (enable) 1189 curr |= wanted; 1190 else 1191 curr &= ~wanted; 1192 1193 MLX5_SET(modify_esw_vport_context_in, min, 1194 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1195 MLX5_SET(modify_esw_vport_context_in, min, 1196 field_select.fdb_to_vport_reg_c_id, 1); 1197 1198 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1199 if (!err) { 1200 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1201 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1202 else 1203 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1204 } 1205 1206 return err; 1207 } 1208 1209 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1210 struct mlx5_core_dev *peer_dev, 1211 struct mlx5_flow_spec *spec, 1212 struct mlx5_flow_destination *dest) 1213 { 1214 void *misc; 1215 1216 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1217 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1218 misc_parameters_2); 1219 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1220 mlx5_eswitch_get_vport_metadata_mask()); 1221 1222 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1223 } else { 1224 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1225 misc_parameters); 1226 1227 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1228 MLX5_CAP_GEN(peer_dev, vhca_id)); 1229 1230 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1231 1232 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1233 misc_parameters); 1234 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1235 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1236 source_eswitch_owner_vhca_id); 1237 } 1238 1239 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1240 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1241 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1242 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1243 } 1244 1245 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1246 struct mlx5_eswitch *peer_esw, 1247 struct mlx5_flow_spec *spec, 1248 u16 vport) 1249 { 1250 void *misc; 1251 1252 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1253 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1254 misc_parameters_2); 1255 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1256 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1257 vport)); 1258 } else { 1259 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1260 misc_parameters); 1261 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1262 } 1263 } 1264 1265 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1266 struct mlx5_core_dev *peer_dev) 1267 { 1268 struct mlx5_flow_destination dest = {}; 1269 struct mlx5_flow_act flow_act = {0}; 1270 struct mlx5_flow_handle **flows; 1271 struct mlx5_flow_handle *flow; 1272 struct mlx5_flow_spec *spec; 1273 /* total vports is the same for both e-switches */ 1274 int nvports = esw->total_vports; 1275 void *misc; 1276 int err, i; 1277 1278 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1279 if (!spec) 1280 return -ENOMEM; 1281 1282 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1283 1284 flows = kvzalloc(nvports * sizeof(*flows), GFP_KERNEL); 1285 if (!flows) { 1286 err = -ENOMEM; 1287 goto alloc_flows_err; 1288 } 1289 1290 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1291 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1292 misc_parameters); 1293 1294 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1295 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1296 spec, MLX5_VPORT_PF); 1297 1298 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1299 spec, &flow_act, &dest, 1); 1300 if (IS_ERR(flow)) { 1301 err = PTR_ERR(flow); 1302 goto add_pf_flow_err; 1303 } 1304 flows[MLX5_VPORT_PF] = flow; 1305 } 1306 1307 if (mlx5_ecpf_vport_exists(esw->dev)) { 1308 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1309 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1310 spec, &flow_act, &dest, 1); 1311 if (IS_ERR(flow)) { 1312 err = PTR_ERR(flow); 1313 goto add_ecpf_flow_err; 1314 } 1315 flows[mlx5_eswitch_ecpf_idx(esw)] = flow; 1316 } 1317 1318 mlx5_esw_for_each_vf_vport_num(esw, i, mlx5_core_max_vfs(esw->dev)) { 1319 esw_set_peer_miss_rule_source_port(esw, 1320 peer_dev->priv.eswitch, 1321 spec, i); 1322 1323 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1324 spec, &flow_act, &dest, 1); 1325 if (IS_ERR(flow)) { 1326 err = PTR_ERR(flow); 1327 goto add_vf_flow_err; 1328 } 1329 flows[i] = flow; 1330 } 1331 1332 esw->fdb_table.offloads.peer_miss_rules = flows; 1333 1334 kvfree(spec); 1335 return 0; 1336 1337 add_vf_flow_err: 1338 nvports = --i; 1339 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, nvports) 1340 mlx5_del_flow_rules(flows[i]); 1341 1342 if (mlx5_ecpf_vport_exists(esw->dev)) 1343 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]); 1344 add_ecpf_flow_err: 1345 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 1346 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]); 1347 add_pf_flow_err: 1348 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1349 kvfree(flows); 1350 alloc_flows_err: 1351 kvfree(spec); 1352 return err; 1353 } 1354 1355 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) 1356 { 1357 struct mlx5_flow_handle **flows; 1358 int i; 1359 1360 flows = esw->fdb_table.offloads.peer_miss_rules; 1361 1362 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, 1363 mlx5_core_max_vfs(esw->dev)) 1364 mlx5_del_flow_rules(flows[i]); 1365 1366 if (mlx5_ecpf_vport_exists(esw->dev)) 1367 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]); 1368 1369 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 1370 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]); 1371 1372 kvfree(flows); 1373 } 1374 1375 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1376 { 1377 struct mlx5_flow_act flow_act = {0}; 1378 struct mlx5_flow_destination dest = {}; 1379 struct mlx5_flow_handle *flow_rule = NULL; 1380 struct mlx5_flow_spec *spec; 1381 void *headers_c; 1382 void *headers_v; 1383 int err = 0; 1384 u8 *dmac_c; 1385 u8 *dmac_v; 1386 1387 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1388 if (!spec) { 1389 err = -ENOMEM; 1390 goto out; 1391 } 1392 1393 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1394 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1395 outer_headers); 1396 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1397 outer_headers.dmac_47_16); 1398 dmac_c[0] = 0x01; 1399 1400 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1401 dest.vport.num = esw->manager_vport; 1402 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1403 1404 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1405 spec, &flow_act, &dest, 1); 1406 if (IS_ERR(flow_rule)) { 1407 err = PTR_ERR(flow_rule); 1408 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1409 goto out; 1410 } 1411 1412 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1413 1414 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1415 outer_headers); 1416 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1417 outer_headers.dmac_47_16); 1418 dmac_v[0] = 0x01; 1419 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1420 spec, &flow_act, &dest, 1); 1421 if (IS_ERR(flow_rule)) { 1422 err = PTR_ERR(flow_rule); 1423 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1424 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1425 goto out; 1426 } 1427 1428 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1429 1430 out: 1431 kvfree(spec); 1432 return err; 1433 } 1434 1435 struct mlx5_flow_handle * 1436 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1437 { 1438 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1439 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1440 struct mlx5_flow_context *flow_context; 1441 struct mlx5_flow_handle *flow_rule; 1442 struct mlx5_flow_destination dest; 1443 struct mlx5_flow_spec *spec; 1444 void *misc; 1445 1446 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1447 return ERR_PTR(-EOPNOTSUPP); 1448 1449 spec = kzalloc(sizeof(*spec), GFP_KERNEL); 1450 if (!spec) 1451 return ERR_PTR(-ENOMEM); 1452 1453 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1454 misc_parameters_2); 1455 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1456 ESW_CHAIN_TAG_METADATA_MASK); 1457 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1458 misc_parameters_2); 1459 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1460 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1461 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1462 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1463 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1464 1465 flow_context = &spec->flow_context; 1466 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1467 flow_context->flow_tag = tag; 1468 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1469 dest.ft = esw->offloads.ft_offloads; 1470 1471 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1472 kfree(spec); 1473 1474 if (IS_ERR(flow_rule)) 1475 esw_warn(esw->dev, 1476 "Failed to create restore rule for tag: %d, err(%d)\n", 1477 tag, (int)PTR_ERR(flow_rule)); 1478 1479 return flow_rule; 1480 } 1481 1482 u32 1483 esw_get_max_restore_tag(struct mlx5_eswitch *esw) 1484 { 1485 return ESW_CHAIN_TAG_METADATA_MASK; 1486 } 1487 1488 #define MAX_PF_SQ 256 1489 #define MAX_SQ_NVPORTS 32 1490 1491 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1492 u32 *flow_group_in) 1493 { 1494 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1495 flow_group_in, 1496 match_criteria); 1497 1498 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1499 MLX5_SET(create_flow_group_in, flow_group_in, 1500 match_criteria_enable, 1501 MLX5_MATCH_MISC_PARAMETERS_2); 1502 1503 MLX5_SET(fte_match_param, match_criteria, 1504 misc_parameters_2.metadata_reg_c_0, 1505 mlx5_eswitch_get_vport_metadata_mask()); 1506 } else { 1507 MLX5_SET(create_flow_group_in, flow_group_in, 1508 match_criteria_enable, 1509 MLX5_MATCH_MISC_PARAMETERS); 1510 1511 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1512 misc_parameters.source_port); 1513 } 1514 } 1515 1516 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1517 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1518 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1519 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1520 { 1521 struct mlx5_core_dev *dev = esw->dev; 1522 1523 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1524 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1525 1526 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1527 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1528 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1529 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1530 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1531 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1532 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1533 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1534 /* Disabled when ttl workaround is needed, e.g 1535 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1536 */ 1537 esw_warn(dev, 1538 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1539 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1540 } else { 1541 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1542 esw_info(dev, "Supported tc chains and prios offload\n"); 1543 } 1544 1545 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1546 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1547 } 1548 1549 static int 1550 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1551 { 1552 struct mlx5_core_dev *dev = esw->dev; 1553 struct mlx5_flow_table *nf_ft, *ft; 1554 struct mlx5_chains_attr attr = {}; 1555 struct mlx5_fs_chains *chains; 1556 u32 fdb_max; 1557 int err; 1558 1559 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size); 1560 1561 esw_init_chains_offload_flags(esw, &attr.flags); 1562 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1563 attr.max_ft_sz = fdb_max; 1564 attr.max_grp_num = esw->params.large_group_num; 1565 attr.default_ft = miss_fdb; 1566 attr.max_restore_tag = esw_get_max_restore_tag(esw); 1567 1568 chains = mlx5_chains_create(dev, &attr); 1569 if (IS_ERR(chains)) { 1570 err = PTR_ERR(chains); 1571 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1572 return err; 1573 } 1574 1575 esw->fdb_table.offloads.esw_chains_priv = chains; 1576 1577 /* Create tc_end_ft which is the always created ft chain */ 1578 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1579 1, 0); 1580 if (IS_ERR(nf_ft)) { 1581 err = PTR_ERR(nf_ft); 1582 goto nf_ft_err; 1583 } 1584 1585 /* Always open the root for fast path */ 1586 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1587 if (IS_ERR(ft)) { 1588 err = PTR_ERR(ft); 1589 goto level_0_err; 1590 } 1591 1592 /* Open level 1 for split fdb rules now if prios isn't supported */ 1593 if (!mlx5_chains_prios_supported(chains)) { 1594 err = mlx5_esw_vport_tbl_get(esw); 1595 if (err) 1596 goto level_1_err; 1597 } 1598 1599 mlx5_chains_set_end_ft(chains, nf_ft); 1600 1601 return 0; 1602 1603 level_1_err: 1604 mlx5_chains_put_table(chains, 0, 1, 0); 1605 level_0_err: 1606 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1607 nf_ft_err: 1608 mlx5_chains_destroy(chains); 1609 esw->fdb_table.offloads.esw_chains_priv = NULL; 1610 1611 return err; 1612 } 1613 1614 static void 1615 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1616 { 1617 if (!mlx5_chains_prios_supported(chains)) 1618 mlx5_esw_vport_tbl_put(esw); 1619 mlx5_chains_put_table(chains, 0, 1, 0); 1620 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1621 mlx5_chains_destroy(chains); 1622 } 1623 1624 #else /* CONFIG_MLX5_CLS_ACT */ 1625 1626 static int 1627 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1628 { return 0; } 1629 1630 static void 1631 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1632 {} 1633 1634 #endif 1635 1636 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1637 { 1638 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1639 struct mlx5_flow_table_attr ft_attr = {}; 1640 int num_vfs, table_size, ix, err = 0; 1641 struct mlx5_core_dev *dev = esw->dev; 1642 struct mlx5_flow_namespace *root_ns; 1643 struct mlx5_flow_table *fdb = NULL; 1644 u32 flags = 0, *flow_group_in; 1645 struct mlx5_flow_group *g; 1646 void *match_criteria; 1647 u8 *dmac; 1648 1649 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1650 1651 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1652 if (!flow_group_in) 1653 return -ENOMEM; 1654 1655 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1656 if (!root_ns) { 1657 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1658 err = -EOPNOTSUPP; 1659 goto ns_err; 1660 } 1661 esw->fdb_table.offloads.ns = root_ns; 1662 err = mlx5_flow_namespace_set_mode(root_ns, 1663 esw->dev->priv.steering->mode); 1664 if (err) { 1665 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1666 goto ns_err; 1667 } 1668 1669 table_size = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1670 MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs; 1671 1672 /* create the slow path fdb with encap set, so further table instances 1673 * can be created at run time while VFs are probed if the FW allows that. 1674 */ 1675 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1676 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1677 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1678 1679 ft_attr.flags = flags; 1680 ft_attr.max_fte = table_size; 1681 ft_attr.prio = FDB_SLOW_PATH; 1682 1683 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1684 if (IS_ERR(fdb)) { 1685 err = PTR_ERR(fdb); 1686 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1687 goto slow_fdb_err; 1688 } 1689 esw->fdb_table.offloads.slow_fdb = fdb; 1690 1691 err = esw_chains_create(esw, fdb); 1692 if (err) { 1693 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1694 goto fdb_chains_err; 1695 } 1696 1697 /* create send-to-vport group */ 1698 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1699 MLX5_MATCH_MISC_PARAMETERS); 1700 1701 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1702 1703 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1704 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 1705 1706 ix = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ; 1707 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1708 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); 1709 1710 g = mlx5_create_flow_group(fdb, flow_group_in); 1711 if (IS_ERR(g)) { 1712 err = PTR_ERR(g); 1713 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1714 goto send_vport_err; 1715 } 1716 esw->fdb_table.offloads.send_to_vport_grp = g; 1717 1718 /* meta send to vport */ 1719 memset(flow_group_in, 0, inlen); 1720 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1721 MLX5_MATCH_MISC_PARAMETERS_2); 1722 1723 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1724 1725 MLX5_SET(fte_match_param, match_criteria, 1726 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1727 MLX5_SET(fte_match_param, match_criteria, 1728 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1729 1730 num_vfs = esw->esw_funcs.num_vfs; 1731 if (num_vfs) { 1732 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1733 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix + num_vfs - 1); 1734 ix += num_vfs; 1735 1736 g = mlx5_create_flow_group(fdb, flow_group_in); 1737 if (IS_ERR(g)) { 1738 err = PTR_ERR(g); 1739 esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n", 1740 err); 1741 goto send_vport_meta_err; 1742 } 1743 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1744 1745 err = mlx5_eswitch_add_send_to_vport_meta_rules(esw); 1746 if (err) 1747 goto meta_rule_err; 1748 } 1749 1750 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1751 /* create peer esw miss group */ 1752 memset(flow_group_in, 0, inlen); 1753 1754 esw_set_flow_group_source_port(esw, flow_group_in); 1755 1756 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1757 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1758 flow_group_in, 1759 match_criteria); 1760 1761 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1762 misc_parameters.source_eswitch_owner_vhca_id); 1763 1764 MLX5_SET(create_flow_group_in, flow_group_in, 1765 source_eswitch_owner_vhca_id_valid, 1); 1766 } 1767 1768 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1769 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1770 ix + esw->total_vports - 1); 1771 ix += esw->total_vports; 1772 1773 g = mlx5_create_flow_group(fdb, flow_group_in); 1774 if (IS_ERR(g)) { 1775 err = PTR_ERR(g); 1776 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err); 1777 goto peer_miss_err; 1778 } 1779 esw->fdb_table.offloads.peer_miss_grp = g; 1780 } 1781 1782 /* create miss group */ 1783 memset(flow_group_in, 0, inlen); 1784 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1785 MLX5_MATCH_OUTER_HEADERS); 1786 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1787 match_criteria); 1788 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1789 outer_headers.dmac_47_16); 1790 dmac[0] = 0x01; 1791 1792 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1793 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1794 ix + MLX5_ESW_MISS_FLOWS); 1795 1796 g = mlx5_create_flow_group(fdb, flow_group_in); 1797 if (IS_ERR(g)) { 1798 err = PTR_ERR(g); 1799 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err); 1800 goto miss_err; 1801 } 1802 esw->fdb_table.offloads.miss_grp = g; 1803 1804 err = esw_add_fdb_miss_rule(esw); 1805 if (err) 1806 goto miss_rule_err; 1807 1808 kvfree(flow_group_in); 1809 return 0; 1810 1811 miss_rule_err: 1812 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1813 miss_err: 1814 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1815 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1816 peer_miss_err: 1817 mlx5_eswitch_del_send_to_vport_meta_rules(esw); 1818 meta_rule_err: 1819 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1820 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1821 send_vport_meta_err: 1822 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1823 send_vport_err: 1824 esw_chains_destroy(esw, esw_chains(esw)); 1825 fdb_chains_err: 1826 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1827 slow_fdb_err: 1828 /* Holds true only as long as DMFS is the default */ 1829 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1830 ns_err: 1831 kvfree(flow_group_in); 1832 return err; 1833 } 1834 1835 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1836 { 1837 if (!esw->fdb_table.offloads.slow_fdb) 1838 return; 1839 1840 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1841 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1842 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1843 mlx5_eswitch_del_send_to_vport_meta_rules(esw); 1844 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1845 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1846 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1847 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1848 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1849 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1850 1851 esw_chains_destroy(esw, esw_chains(esw)); 1852 1853 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1854 /* Holds true only as long as DMFS is the default */ 1855 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1856 MLX5_FLOW_STEERING_MODE_DMFS); 1857 } 1858 1859 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1860 { 1861 struct mlx5_flow_table_attr ft_attr = {}; 1862 struct mlx5_core_dev *dev = esw->dev; 1863 struct mlx5_flow_table *ft_offloads; 1864 struct mlx5_flow_namespace *ns; 1865 int err = 0; 1866 1867 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1868 if (!ns) { 1869 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1870 return -EOPNOTSUPP; 1871 } 1872 1873 ft_attr.max_fte = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1874 ft_attr.prio = 1; 1875 1876 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 1877 if (IS_ERR(ft_offloads)) { 1878 err = PTR_ERR(ft_offloads); 1879 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 1880 return err; 1881 } 1882 1883 esw->offloads.ft_offloads = ft_offloads; 1884 return 0; 1885 } 1886 1887 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 1888 { 1889 struct mlx5_esw_offload *offloads = &esw->offloads; 1890 1891 mlx5_destroy_flow_table(offloads->ft_offloads); 1892 } 1893 1894 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 1895 { 1896 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1897 struct mlx5_flow_group *g; 1898 u32 *flow_group_in; 1899 int nvports; 1900 int err = 0; 1901 1902 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1903 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1904 if (!flow_group_in) 1905 return -ENOMEM; 1906 1907 /* create vport rx group */ 1908 esw_set_flow_group_source_port(esw, flow_group_in); 1909 1910 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1911 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 1912 1913 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 1914 1915 if (IS_ERR(g)) { 1916 err = PTR_ERR(g); 1917 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 1918 goto out; 1919 } 1920 1921 esw->offloads.vport_rx_group = g; 1922 out: 1923 kvfree(flow_group_in); 1924 return err; 1925 } 1926 1927 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 1928 { 1929 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 1930 } 1931 1932 struct mlx5_flow_handle * 1933 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 1934 struct mlx5_flow_destination *dest) 1935 { 1936 struct mlx5_flow_act flow_act = {0}; 1937 struct mlx5_flow_handle *flow_rule; 1938 struct mlx5_flow_spec *spec; 1939 void *misc; 1940 1941 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1942 if (!spec) { 1943 flow_rule = ERR_PTR(-ENOMEM); 1944 goto out; 1945 } 1946 1947 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1948 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 1949 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1950 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 1951 1952 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 1953 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1954 mlx5_eswitch_get_vport_metadata_mask()); 1955 1956 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1957 } else { 1958 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1959 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1960 1961 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1962 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1963 1964 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1965 } 1966 1967 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1968 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 1969 &flow_act, dest, 1); 1970 if (IS_ERR(flow_rule)) { 1971 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 1972 goto out; 1973 } 1974 1975 out: 1976 kvfree(spec); 1977 return flow_rule; 1978 } 1979 1980 1981 static int mlx5_eswitch_inline_mode_get(const struct mlx5_eswitch *esw, u8 *mode) 1982 { 1983 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 1984 struct mlx5_core_dev *dev = esw->dev; 1985 int vport; 1986 1987 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 1988 return -EOPNOTSUPP; 1989 1990 if (esw->mode == MLX5_ESWITCH_NONE) 1991 return -EOPNOTSUPP; 1992 1993 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 1994 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 1995 mlx5_mode = MLX5_INLINE_MODE_NONE; 1996 goto out; 1997 case MLX5_CAP_INLINE_MODE_L2: 1998 mlx5_mode = MLX5_INLINE_MODE_L2; 1999 goto out; 2000 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2001 goto query_vports; 2002 } 2003 2004 query_vports: 2005 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2006 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) { 2007 mlx5_query_nic_vport_min_inline(dev, vport, &mlx5_mode); 2008 if (prev_mlx5_mode != mlx5_mode) 2009 return -EINVAL; 2010 prev_mlx5_mode = mlx5_mode; 2011 } 2012 2013 out: 2014 *mode = mlx5_mode; 2015 return 0; 2016 } 2017 2018 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2019 { 2020 struct mlx5_esw_offload *offloads = &esw->offloads; 2021 2022 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2023 return; 2024 2025 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2026 mlx5_destroy_flow_group(offloads->restore_group); 2027 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2028 } 2029 2030 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2031 { 2032 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2033 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2034 struct mlx5_flow_table_attr ft_attr = {}; 2035 struct mlx5_core_dev *dev = esw->dev; 2036 struct mlx5_flow_namespace *ns; 2037 struct mlx5_modify_hdr *mod_hdr; 2038 void *match_criteria, *misc; 2039 struct mlx5_flow_table *ft; 2040 struct mlx5_flow_group *g; 2041 u32 *flow_group_in; 2042 int err = 0; 2043 2044 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2045 return 0; 2046 2047 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2048 if (!ns) { 2049 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2050 return -EOPNOTSUPP; 2051 } 2052 2053 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2054 if (!flow_group_in) { 2055 err = -ENOMEM; 2056 goto out_free; 2057 } 2058 2059 ft_attr.max_fte = 1 << ESW_CHAIN_TAG_METADATA_BITS; 2060 ft = mlx5_create_flow_table(ns, &ft_attr); 2061 if (IS_ERR(ft)) { 2062 err = PTR_ERR(ft); 2063 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2064 err); 2065 goto out_free; 2066 } 2067 2068 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2069 match_criteria); 2070 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2071 misc_parameters_2); 2072 2073 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2074 ESW_CHAIN_TAG_METADATA_MASK); 2075 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2076 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2077 ft_attr.max_fte - 1); 2078 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2079 MLX5_MATCH_MISC_PARAMETERS_2); 2080 g = mlx5_create_flow_group(ft, flow_group_in); 2081 if (IS_ERR(g)) { 2082 err = PTR_ERR(g); 2083 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2084 err); 2085 goto err_group; 2086 } 2087 2088 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2089 MLX5_SET(copy_action_in, modact, src_field, 2090 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2091 MLX5_SET(copy_action_in, modact, dst_field, 2092 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2093 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2094 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2095 modact); 2096 if (IS_ERR(mod_hdr)) { 2097 err = PTR_ERR(mod_hdr); 2098 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2099 err); 2100 goto err_mod_hdr; 2101 } 2102 2103 esw->offloads.ft_offloads_restore = ft; 2104 esw->offloads.restore_group = g; 2105 esw->offloads.restore_copy_hdr_id = mod_hdr; 2106 2107 kvfree(flow_group_in); 2108 2109 return 0; 2110 2111 err_mod_hdr: 2112 mlx5_destroy_flow_group(g); 2113 err_group: 2114 mlx5_destroy_flow_table(ft); 2115 out_free: 2116 kvfree(flow_group_in); 2117 2118 return err; 2119 } 2120 2121 static int esw_offloads_start(struct mlx5_eswitch *esw, 2122 struct netlink_ext_ack *extack) 2123 { 2124 int err, err1; 2125 2126 mlx5_eswitch_disable_locked(esw, false); 2127 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, 2128 esw->dev->priv.sriov.num_vfs); 2129 if (err) { 2130 NL_SET_ERR_MSG_MOD(extack, 2131 "Failed setting eswitch to offloads"); 2132 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, 2133 MLX5_ESWITCH_IGNORE_NUM_VFS); 2134 if (err1) { 2135 NL_SET_ERR_MSG_MOD(extack, 2136 "Failed setting eswitch back to legacy"); 2137 } 2138 } 2139 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2140 if (mlx5_eswitch_inline_mode_get(esw, 2141 &esw->offloads.inline_mode)) { 2142 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2143 NL_SET_ERR_MSG_MOD(extack, 2144 "Inline mode is different between vports"); 2145 } 2146 } 2147 return err; 2148 } 2149 2150 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2151 { 2152 kfree(esw->offloads.vport_reps); 2153 } 2154 2155 int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2156 { 2157 int total_vports = esw->total_vports; 2158 struct mlx5_eswitch_rep *rep; 2159 int vport_index; 2160 u8 rep_type; 2161 2162 esw->offloads.vport_reps = kcalloc(total_vports, 2163 sizeof(struct mlx5_eswitch_rep), 2164 GFP_KERNEL); 2165 if (!esw->offloads.vport_reps) 2166 return -ENOMEM; 2167 2168 mlx5_esw_for_all_reps(esw, vport_index, rep) { 2169 rep->vport = mlx5_eswitch_index_to_vport_num(esw, vport_index); 2170 rep->vport_index = vport_index; 2171 2172 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2173 atomic_set(&rep->rep_data[rep_type].state, 2174 REP_UNREGISTERED); 2175 } 2176 2177 return 0; 2178 } 2179 2180 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2181 struct mlx5_eswitch_rep *rep, u8 rep_type) 2182 { 2183 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2184 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2185 esw->offloads.rep_ops[rep_type]->unload(rep); 2186 } 2187 2188 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type) 2189 { 2190 struct mlx5_eswitch_rep *rep; 2191 int i; 2192 2193 mlx5_esw_for_each_sf_rep(esw, i, rep) 2194 __esw_offloads_unload_rep(esw, rep, rep_type); 2195 } 2196 2197 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2198 { 2199 struct mlx5_eswitch_rep *rep; 2200 int i; 2201 2202 __unload_reps_sf_vport(esw, rep_type); 2203 2204 mlx5_esw_for_each_vf_rep_reverse(esw, i, rep, esw->esw_funcs.num_vfs) 2205 __esw_offloads_unload_rep(esw, rep, rep_type); 2206 2207 if (mlx5_ecpf_vport_exists(esw->dev)) { 2208 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF); 2209 __esw_offloads_unload_rep(esw, rep, rep_type); 2210 } 2211 2212 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 2213 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF); 2214 __esw_offloads_unload_rep(esw, rep, rep_type); 2215 } 2216 2217 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 2218 __esw_offloads_unload_rep(esw, rep, rep_type); 2219 } 2220 2221 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2222 { 2223 struct mlx5_eswitch_rep *rep; 2224 int rep_type; 2225 int err; 2226 2227 rep = mlx5_eswitch_get_rep(esw, vport_num); 2228 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2229 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2230 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 2231 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2232 if (err) 2233 goto err_reps; 2234 } 2235 2236 return 0; 2237 2238 err_reps: 2239 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2240 for (--rep_type; rep_type >= 0; rep_type--) 2241 __esw_offloads_unload_rep(esw, rep, rep_type); 2242 return err; 2243 } 2244 2245 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2246 { 2247 struct mlx5_eswitch_rep *rep; 2248 int rep_type; 2249 2250 rep = mlx5_eswitch_get_rep(esw, vport_num); 2251 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2252 __esw_offloads_unload_rep(esw, rep, rep_type); 2253 } 2254 2255 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) 2256 { 2257 int err; 2258 2259 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2260 return 0; 2261 2262 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num); 2263 if (err) 2264 return err; 2265 2266 err = mlx5_esw_offloads_rep_load(esw, vport_num); 2267 if (err) 2268 goto load_err; 2269 return err; 2270 2271 load_err: 2272 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2273 return err; 2274 } 2275 2276 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) 2277 { 2278 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2279 return; 2280 2281 mlx5_esw_offloads_rep_unload(esw, vport_num); 2282 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2283 } 2284 2285 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 2286 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2287 2288 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2289 struct mlx5_eswitch *peer_esw) 2290 { 2291 int err; 2292 2293 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2294 if (err) 2295 return err; 2296 2297 return 0; 2298 } 2299 2300 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) 2301 { 2302 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2303 mlx5e_tc_clean_fdb_peer_flows(esw); 2304 #endif 2305 esw_del_fdb_peer_miss_rules(esw); 2306 } 2307 2308 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2309 struct mlx5_eswitch *peer_esw, 2310 bool pair) 2311 { 2312 struct mlx5_flow_root_namespace *peer_ns; 2313 struct mlx5_flow_root_namespace *ns; 2314 int err; 2315 2316 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2317 ns = esw->dev->priv.steering->fdb_root_ns; 2318 2319 if (pair) { 2320 err = mlx5_flow_namespace_set_peer(ns, peer_ns); 2321 if (err) 2322 return err; 2323 2324 err = mlx5_flow_namespace_set_peer(peer_ns, ns); 2325 if (err) { 2326 mlx5_flow_namespace_set_peer(ns, NULL); 2327 return err; 2328 } 2329 } else { 2330 mlx5_flow_namespace_set_peer(ns, NULL); 2331 mlx5_flow_namespace_set_peer(peer_ns, NULL); 2332 } 2333 2334 return 0; 2335 } 2336 2337 static int mlx5_esw_offloads_devcom_event(int event, 2338 void *my_data, 2339 void *event_data) 2340 { 2341 struct mlx5_eswitch *esw = my_data; 2342 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2343 struct mlx5_eswitch *peer_esw = event_data; 2344 int err; 2345 2346 switch (event) { 2347 case ESW_OFFLOADS_DEVCOM_PAIR: 2348 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2349 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2350 break; 2351 2352 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2353 if (err) 2354 goto err_out; 2355 err = mlx5_esw_offloads_pair(esw, peer_esw); 2356 if (err) 2357 goto err_peer; 2358 2359 err = mlx5_esw_offloads_pair(peer_esw, esw); 2360 if (err) 2361 goto err_pair; 2362 2363 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); 2364 break; 2365 2366 case ESW_OFFLOADS_DEVCOM_UNPAIR: 2367 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) 2368 break; 2369 2370 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); 2371 mlx5_esw_offloads_unpair(peer_esw); 2372 mlx5_esw_offloads_unpair(esw); 2373 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2374 break; 2375 } 2376 2377 return 0; 2378 2379 err_pair: 2380 mlx5_esw_offloads_unpair(esw); 2381 err_peer: 2382 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2383 err_out: 2384 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 2385 event, err); 2386 return err; 2387 } 2388 2389 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw) 2390 { 2391 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2392 2393 INIT_LIST_HEAD(&esw->offloads.peer_flows); 2394 mutex_init(&esw->offloads.peer_mutex); 2395 2396 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2397 return; 2398 2399 mlx5_devcom_register_component(devcom, 2400 MLX5_DEVCOM_ESW_OFFLOADS, 2401 mlx5_esw_offloads_devcom_event, 2402 esw); 2403 2404 mlx5_devcom_send_event(devcom, 2405 MLX5_DEVCOM_ESW_OFFLOADS, 2406 ESW_OFFLOADS_DEVCOM_PAIR, esw); 2407 } 2408 2409 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 2410 { 2411 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2412 2413 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2414 return; 2415 2416 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, 2417 ESW_OFFLOADS_DEVCOM_UNPAIR, esw); 2418 2419 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); 2420 } 2421 2422 static bool 2423 esw_check_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 2424 { 2425 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 2426 return false; 2427 2428 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 2429 MLX5_FDB_TO_VPORT_REG_C_0)) 2430 return false; 2431 2432 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 2433 return false; 2434 2435 if (mlx5_core_is_ecpf_esw_manager(esw->dev) || 2436 mlx5_ecpf_vport_exists(esw->dev)) 2437 return false; 2438 2439 return true; 2440 } 2441 2442 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 2443 { 2444 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 2445 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 1; 2446 u32 pf_num; 2447 int id; 2448 2449 /* Only 4 bits of pf_num */ 2450 pf_num = PCI_FUNC(esw->dev->pdev->devfn); 2451 if (pf_num > max_pf_num) 2452 return 0; 2453 2454 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 2455 /* Use only non-zero vport_id (1-4095) for all PF's */ 2456 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 1, vport_end_ida, GFP_KERNEL); 2457 if (id < 0) 2458 return 0; 2459 id = (pf_num << ESW_VPORT_BITS) | id; 2460 return id; 2461 } 2462 2463 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 2464 { 2465 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 2466 2467 /* Metadata contains only 12 bits of actual ida id */ 2468 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 2469 } 2470 2471 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 2472 struct mlx5_vport *vport) 2473 { 2474 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 2475 vport->metadata = vport->default_metadata; 2476 return vport->metadata ? 0 : -ENOSPC; 2477 } 2478 2479 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 2480 struct mlx5_vport *vport) 2481 { 2482 if (!vport->default_metadata) 2483 return; 2484 2485 WARN_ON(vport->metadata != vport->default_metadata); 2486 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 2487 } 2488 2489 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 2490 { 2491 struct mlx5_vport *vport; 2492 int i; 2493 2494 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2495 return; 2496 2497 mlx5_esw_for_all_vports_reverse(esw, i, vport) 2498 esw_offloads_vport_metadata_cleanup(esw, vport); 2499 } 2500 2501 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 2502 { 2503 struct mlx5_vport *vport; 2504 int err; 2505 int i; 2506 2507 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2508 return 0; 2509 2510 mlx5_esw_for_all_vports(esw, i, vport) { 2511 err = esw_offloads_vport_metadata_setup(esw, vport); 2512 if (err) 2513 goto metadata_err; 2514 } 2515 2516 return 0; 2517 2518 metadata_err: 2519 esw_offloads_metadata_uninit(esw); 2520 return err; 2521 } 2522 2523 int 2524 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 2525 struct mlx5_vport *vport) 2526 { 2527 int err; 2528 2529 err = esw_acl_ingress_ofld_setup(esw, vport); 2530 if (err) 2531 return err; 2532 2533 err = esw_acl_egress_ofld_setup(esw, vport); 2534 if (err) 2535 goto egress_err; 2536 2537 return 0; 2538 2539 egress_err: 2540 esw_acl_ingress_ofld_cleanup(esw, vport); 2541 return err; 2542 } 2543 2544 void 2545 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 2546 struct mlx5_vport *vport) 2547 { 2548 esw_acl_egress_ofld_cleanup(vport); 2549 esw_acl_ingress_ofld_cleanup(esw, vport); 2550 } 2551 2552 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2553 { 2554 struct mlx5_vport *vport; 2555 2556 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2557 return esw_vport_create_offloads_acl_tables(esw, vport); 2558 } 2559 2560 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2561 { 2562 struct mlx5_vport *vport; 2563 2564 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2565 esw_vport_destroy_offloads_acl_tables(esw, vport); 2566 } 2567 2568 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 2569 { 2570 struct mlx5_esw_indir_table *indir; 2571 int err; 2572 2573 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 2574 mutex_init(&esw->fdb_table.offloads.vports.lock); 2575 hash_init(esw->fdb_table.offloads.vports.table); 2576 2577 indir = mlx5_esw_indir_table_init(); 2578 if (IS_ERR(indir)) { 2579 err = PTR_ERR(indir); 2580 goto create_indir_err; 2581 } 2582 esw->fdb_table.offloads.indir = indir; 2583 2584 err = esw_create_uplink_offloads_acl_tables(esw); 2585 if (err) 2586 goto create_acl_err; 2587 2588 err = esw_create_offloads_table(esw); 2589 if (err) 2590 goto create_offloads_err; 2591 2592 err = esw_create_restore_table(esw); 2593 if (err) 2594 goto create_restore_err; 2595 2596 err = esw_create_offloads_fdb_tables(esw); 2597 if (err) 2598 goto create_fdb_err; 2599 2600 err = esw_create_vport_rx_group(esw); 2601 if (err) 2602 goto create_fg_err; 2603 2604 return 0; 2605 2606 create_fg_err: 2607 esw_destroy_offloads_fdb_tables(esw); 2608 create_fdb_err: 2609 esw_destroy_restore_table(esw); 2610 create_restore_err: 2611 esw_destroy_offloads_table(esw); 2612 create_offloads_err: 2613 esw_destroy_uplink_offloads_acl_tables(esw); 2614 create_acl_err: 2615 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 2616 create_indir_err: 2617 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 2618 return err; 2619 } 2620 2621 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 2622 { 2623 esw_destroy_vport_rx_group(esw); 2624 esw_destroy_offloads_fdb_tables(esw); 2625 esw_destroy_restore_table(esw); 2626 esw_destroy_offloads_table(esw); 2627 esw_destroy_uplink_offloads_acl_tables(esw); 2628 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 2629 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 2630 } 2631 2632 static void 2633 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 2634 { 2635 bool host_pf_disabled; 2636 u16 new_num_vfs; 2637 2638 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 2639 host_params_context.host_num_of_vfs); 2640 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 2641 host_params_context.host_pf_disabled); 2642 2643 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 2644 return; 2645 2646 /* Number of VFs can only change from "0 to x" or "x to 0". */ 2647 if (esw->esw_funcs.num_vfs > 0) { 2648 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 2649 } else { 2650 int err; 2651 2652 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 2653 MLX5_VPORT_UC_ADDR_CHANGE); 2654 if (err) 2655 return; 2656 } 2657 esw->esw_funcs.num_vfs = new_num_vfs; 2658 } 2659 2660 static void esw_functions_changed_event_handler(struct work_struct *work) 2661 { 2662 struct mlx5_host_work *host_work; 2663 struct mlx5_eswitch *esw; 2664 const u32 *out; 2665 2666 host_work = container_of(work, struct mlx5_host_work, work); 2667 esw = host_work->esw; 2668 2669 out = mlx5_esw_query_functions(esw->dev); 2670 if (IS_ERR(out)) 2671 goto out; 2672 2673 esw_vfs_changed_event_handler(esw, out); 2674 kvfree(out); 2675 out: 2676 kfree(host_work); 2677 } 2678 2679 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 2680 { 2681 struct mlx5_esw_functions *esw_funcs; 2682 struct mlx5_host_work *host_work; 2683 struct mlx5_eswitch *esw; 2684 2685 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 2686 if (!host_work) 2687 return NOTIFY_DONE; 2688 2689 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 2690 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 2691 2692 host_work->esw = esw; 2693 2694 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 2695 queue_work(esw->work_queue, &host_work->work); 2696 2697 return NOTIFY_OK; 2698 } 2699 2700 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 2701 { 2702 const u32 *query_host_out; 2703 2704 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 2705 return 0; 2706 2707 query_host_out = mlx5_esw_query_functions(esw->dev); 2708 if (IS_ERR(query_host_out)) 2709 return PTR_ERR(query_host_out); 2710 2711 /* Mark non local controller with non zero controller number. */ 2712 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 2713 host_params_context.host_number); 2714 kvfree(query_host_out); 2715 return 0; 2716 } 2717 2718 int esw_offloads_enable(struct mlx5_eswitch *esw) 2719 { 2720 struct mlx5_vport *vport; 2721 int err, i; 2722 2723 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat) && 2724 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, decap)) 2725 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC; 2726 else 2727 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE; 2728 2729 mutex_init(&esw->offloads.termtbl_mutex); 2730 mlx5_rdma_enable_roce(esw->dev); 2731 2732 err = mlx5_esw_host_number_init(esw); 2733 if (err) 2734 goto err_metadata; 2735 2736 if (esw_check_vport_match_metadata_supported(esw)) 2737 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2738 2739 err = esw_offloads_metadata_init(esw); 2740 if (err) 2741 goto err_metadata; 2742 2743 err = esw_set_passing_vport_metadata(esw, true); 2744 if (err) 2745 goto err_vport_metadata; 2746 2747 err = esw_offloads_steering_init(esw); 2748 if (err) 2749 goto err_steering_init; 2750 2751 /* Representor will control the vport link state */ 2752 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 2753 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 2754 2755 /* Uplink vport rep must load first. */ 2756 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); 2757 if (err) 2758 goto err_uplink; 2759 2760 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 2761 if (err) 2762 goto err_vports; 2763 2764 esw_offloads_devcom_init(esw); 2765 2766 return 0; 2767 2768 err_vports: 2769 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 2770 err_uplink: 2771 esw_offloads_steering_cleanup(esw); 2772 err_steering_init: 2773 esw_set_passing_vport_metadata(esw, false); 2774 err_vport_metadata: 2775 esw_offloads_metadata_uninit(esw); 2776 err_metadata: 2777 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2778 mlx5_rdma_disable_roce(esw->dev); 2779 mutex_destroy(&esw->offloads.termtbl_mutex); 2780 return err; 2781 } 2782 2783 static int esw_offloads_stop(struct mlx5_eswitch *esw, 2784 struct netlink_ext_ack *extack) 2785 { 2786 int err, err1; 2787 2788 mlx5_eswitch_disable_locked(esw, false); 2789 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, 2790 MLX5_ESWITCH_IGNORE_NUM_VFS); 2791 if (err) { 2792 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 2793 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, 2794 MLX5_ESWITCH_IGNORE_NUM_VFS); 2795 if (err1) { 2796 NL_SET_ERR_MSG_MOD(extack, 2797 "Failed setting eswitch back to offloads"); 2798 } 2799 } 2800 2801 return err; 2802 } 2803 2804 void esw_offloads_disable(struct mlx5_eswitch *esw) 2805 { 2806 esw_offloads_devcom_cleanup(esw); 2807 mlx5_eswitch_disable_pf_vf_vports(esw); 2808 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 2809 esw_set_passing_vport_metadata(esw, false); 2810 esw_offloads_steering_cleanup(esw); 2811 esw_offloads_metadata_uninit(esw); 2812 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2813 mlx5_rdma_disable_roce(esw->dev); 2814 mutex_destroy(&esw->offloads.termtbl_mutex); 2815 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE; 2816 } 2817 2818 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 2819 { 2820 switch (mode) { 2821 case DEVLINK_ESWITCH_MODE_LEGACY: 2822 *mlx5_mode = MLX5_ESWITCH_LEGACY; 2823 break; 2824 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 2825 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 2826 break; 2827 default: 2828 return -EINVAL; 2829 } 2830 2831 return 0; 2832 } 2833 2834 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 2835 { 2836 switch (mlx5_mode) { 2837 case MLX5_ESWITCH_LEGACY: 2838 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 2839 break; 2840 case MLX5_ESWITCH_OFFLOADS: 2841 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 2842 break; 2843 default: 2844 return -EINVAL; 2845 } 2846 2847 return 0; 2848 } 2849 2850 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 2851 { 2852 switch (mode) { 2853 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 2854 *mlx5_mode = MLX5_INLINE_MODE_NONE; 2855 break; 2856 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 2857 *mlx5_mode = MLX5_INLINE_MODE_L2; 2858 break; 2859 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 2860 *mlx5_mode = MLX5_INLINE_MODE_IP; 2861 break; 2862 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 2863 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 2864 break; 2865 default: 2866 return -EINVAL; 2867 } 2868 2869 return 0; 2870 } 2871 2872 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 2873 { 2874 switch (mlx5_mode) { 2875 case MLX5_INLINE_MODE_NONE: 2876 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 2877 break; 2878 case MLX5_INLINE_MODE_L2: 2879 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 2880 break; 2881 case MLX5_INLINE_MODE_IP: 2882 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 2883 break; 2884 case MLX5_INLINE_MODE_TCP_UDP: 2885 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 2886 break; 2887 default: 2888 return -EINVAL; 2889 } 2890 2891 return 0; 2892 } 2893 2894 static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) 2895 { 2896 /* devlink commands in NONE eswitch mode are currently supported only 2897 * on ECPF. 2898 */ 2899 return (esw->mode == MLX5_ESWITCH_NONE && 2900 !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0; 2901 } 2902 2903 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 2904 struct netlink_ext_ack *extack) 2905 { 2906 u16 cur_mlx5_mode, mlx5_mode = 0; 2907 struct mlx5_eswitch *esw; 2908 int err = 0; 2909 2910 esw = mlx5_devlink_eswitch_get(devlink); 2911 if (IS_ERR(esw)) 2912 return PTR_ERR(esw); 2913 2914 if (esw_mode_from_devlink(mode, &mlx5_mode)) 2915 return -EINVAL; 2916 2917 mutex_lock(&esw->mode_lock); 2918 cur_mlx5_mode = esw->mode; 2919 if (cur_mlx5_mode == mlx5_mode) 2920 goto unlock; 2921 2922 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) 2923 err = esw_offloads_start(esw, extack); 2924 else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) 2925 err = esw_offloads_stop(esw, extack); 2926 else 2927 err = -EINVAL; 2928 2929 unlock: 2930 mutex_unlock(&esw->mode_lock); 2931 return err; 2932 } 2933 2934 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 2935 { 2936 struct mlx5_eswitch *esw; 2937 int err; 2938 2939 esw = mlx5_devlink_eswitch_get(devlink); 2940 if (IS_ERR(esw)) 2941 return PTR_ERR(esw); 2942 2943 mutex_lock(&esw->mode_lock); 2944 err = eswitch_devlink_esw_mode_check(esw); 2945 if (err) 2946 goto unlock; 2947 2948 err = esw_mode_to_devlink(esw->mode, mode); 2949 unlock: 2950 mutex_unlock(&esw->mode_lock); 2951 return err; 2952 } 2953 2954 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 2955 struct netlink_ext_ack *extack) 2956 { 2957 struct mlx5_core_dev *dev = devlink_priv(devlink); 2958 int err, vport, num_vport; 2959 struct mlx5_eswitch *esw; 2960 u8 mlx5_mode; 2961 2962 esw = mlx5_devlink_eswitch_get(devlink); 2963 if (IS_ERR(esw)) 2964 return PTR_ERR(esw); 2965 2966 mutex_lock(&esw->mode_lock); 2967 err = eswitch_devlink_esw_mode_check(esw); 2968 if (err) 2969 goto out; 2970 2971 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2972 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2973 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) 2974 goto out; 2975 fallthrough; 2976 case MLX5_CAP_INLINE_MODE_L2: 2977 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 2978 err = -EOPNOTSUPP; 2979 goto out; 2980 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2981 break; 2982 } 2983 2984 if (atomic64_read(&esw->offloads.num_flows) > 0) { 2985 NL_SET_ERR_MSG_MOD(extack, 2986 "Can't set inline mode when flows are configured"); 2987 err = -EOPNOTSUPP; 2988 goto out; 2989 } 2990 2991 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 2992 if (err) 2993 goto out; 2994 2995 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) { 2996 err = mlx5_modify_nic_vport_min_inline(dev, vport, mlx5_mode); 2997 if (err) { 2998 NL_SET_ERR_MSG_MOD(extack, 2999 "Failed to set min inline on vport"); 3000 goto revert_inline_mode; 3001 } 3002 } 3003 3004 esw->offloads.inline_mode = mlx5_mode; 3005 mutex_unlock(&esw->mode_lock); 3006 return 0; 3007 3008 revert_inline_mode: 3009 num_vport = --vport; 3010 mlx5_esw_for_each_host_func_vport_reverse(esw, vport, num_vport) 3011 mlx5_modify_nic_vport_min_inline(dev, 3012 vport, 3013 esw->offloads.inline_mode); 3014 out: 3015 mutex_unlock(&esw->mode_lock); 3016 return err; 3017 } 3018 3019 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3020 { 3021 struct mlx5_eswitch *esw; 3022 int err; 3023 3024 esw = mlx5_devlink_eswitch_get(devlink); 3025 if (IS_ERR(esw)) 3026 return PTR_ERR(esw); 3027 3028 mutex_lock(&esw->mode_lock); 3029 err = eswitch_devlink_esw_mode_check(esw); 3030 if (err) 3031 goto unlock; 3032 3033 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3034 unlock: 3035 mutex_unlock(&esw->mode_lock); 3036 return err; 3037 } 3038 3039 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3040 enum devlink_eswitch_encap_mode encap, 3041 struct netlink_ext_ack *extack) 3042 { 3043 struct mlx5_core_dev *dev = devlink_priv(devlink); 3044 struct mlx5_eswitch *esw; 3045 int err; 3046 3047 esw = mlx5_devlink_eswitch_get(devlink); 3048 if (IS_ERR(esw)) 3049 return PTR_ERR(esw); 3050 3051 mutex_lock(&esw->mode_lock); 3052 err = eswitch_devlink_esw_mode_check(esw); 3053 if (err) 3054 goto unlock; 3055 3056 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3057 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3058 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3059 err = -EOPNOTSUPP; 3060 goto unlock; 3061 } 3062 3063 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3064 err = -EOPNOTSUPP; 3065 goto unlock; 3066 } 3067 3068 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3069 esw->offloads.encap = encap; 3070 goto unlock; 3071 } 3072 3073 if (esw->offloads.encap == encap) 3074 goto unlock; 3075 3076 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3077 NL_SET_ERR_MSG_MOD(extack, 3078 "Can't set encapsulation when flows are configured"); 3079 err = -EOPNOTSUPP; 3080 goto unlock; 3081 } 3082 3083 esw_destroy_offloads_fdb_tables(esw); 3084 3085 esw->offloads.encap = encap; 3086 3087 err = esw_create_offloads_fdb_tables(esw); 3088 3089 if (err) { 3090 NL_SET_ERR_MSG_MOD(extack, 3091 "Failed re-creating fast FDB table"); 3092 esw->offloads.encap = !encap; 3093 (void)esw_create_offloads_fdb_tables(esw); 3094 } 3095 3096 unlock: 3097 mutex_unlock(&esw->mode_lock); 3098 return err; 3099 } 3100 3101 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 3102 enum devlink_eswitch_encap_mode *encap) 3103 { 3104 struct mlx5_eswitch *esw; 3105 int err; 3106 3107 esw = mlx5_devlink_eswitch_get(devlink); 3108 if (IS_ERR(esw)) 3109 return PTR_ERR(esw); 3110 3111 3112 mutex_lock(&esw->mode_lock); 3113 err = eswitch_devlink_esw_mode_check(esw); 3114 if (err) 3115 goto unlock; 3116 3117 *encap = esw->offloads.encap; 3118 unlock: 3119 mutex_unlock(&esw->mode_lock); 3120 return 0; 3121 } 3122 3123 static bool 3124 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 3125 { 3126 /* Currently, only ECPF based device has representor for host PF. */ 3127 if (vport_num == MLX5_VPORT_PF && 3128 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 3129 return false; 3130 3131 if (vport_num == MLX5_VPORT_ECPF && 3132 !mlx5_ecpf_vport_exists(esw->dev)) 3133 return false; 3134 3135 return true; 3136 } 3137 3138 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 3139 const struct mlx5_eswitch_rep_ops *ops, 3140 u8 rep_type) 3141 { 3142 struct mlx5_eswitch_rep_data *rep_data; 3143 struct mlx5_eswitch_rep *rep; 3144 int i; 3145 3146 esw->offloads.rep_ops[rep_type] = ops; 3147 mlx5_esw_for_all_reps(esw, i, rep) { 3148 if (likely(mlx5_eswitch_vport_has_rep(esw, i))) { 3149 rep_data = &rep->rep_data[rep_type]; 3150 atomic_set(&rep_data->state, REP_REGISTERED); 3151 } 3152 } 3153 } 3154 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 3155 3156 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 3157 { 3158 struct mlx5_eswitch_rep *rep; 3159 int i; 3160 3161 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 3162 __unload_reps_all_vport(esw, rep_type); 3163 3164 mlx5_esw_for_all_reps(esw, i, rep) 3165 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 3166 } 3167 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 3168 3169 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 3170 { 3171 struct mlx5_eswitch_rep *rep; 3172 3173 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3174 return rep->rep_data[rep_type].priv; 3175 } 3176 3177 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 3178 u16 vport, 3179 u8 rep_type) 3180 { 3181 struct mlx5_eswitch_rep *rep; 3182 3183 rep = mlx5_eswitch_get_rep(esw, vport); 3184 3185 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3186 esw->offloads.rep_ops[rep_type]->get_proto_dev) 3187 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 3188 return NULL; 3189 } 3190 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 3191 3192 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 3193 { 3194 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 3195 } 3196 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 3197 3198 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 3199 u16 vport) 3200 { 3201 return mlx5_eswitch_get_rep(esw, vport); 3202 } 3203 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 3204 3205 bool mlx5_eswitch_is_vf_vport(const struct mlx5_eswitch *esw, u16 vport_num) 3206 { 3207 return vport_num >= MLX5_VPORT_FIRST_VF && 3208 vport_num <= esw->dev->priv.sriov.max_vfs; 3209 } 3210 3211 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 3212 { 3213 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 3214 } 3215 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 3216 3217 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 3218 { 3219 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 3220 } 3221 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 3222 3223 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 3224 u16 vport_num) 3225 { 3226 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3227 3228 if (WARN_ON_ONCE(IS_ERR(vport))) 3229 return 0; 3230 3231 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 3232 } 3233 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 3234 3235 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port, 3236 u16 vport_num, u32 sfnum) 3237 { 3238 int err; 3239 3240 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE); 3241 if (err) 3242 return err; 3243 3244 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, sfnum); 3245 if (err) 3246 goto devlink_err; 3247 3248 err = mlx5_esw_offloads_rep_load(esw, vport_num); 3249 if (err) 3250 goto rep_err; 3251 return 0; 3252 3253 rep_err: 3254 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3255 devlink_err: 3256 mlx5_esw_vport_disable(esw, vport_num); 3257 return err; 3258 } 3259 3260 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num) 3261 { 3262 mlx5_esw_offloads_rep_unload(esw, vport_num); 3263 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3264 mlx5_esw_vport_disable(esw, vport_num); 3265 } 3266 3267 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 3268 { 3269 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 3270 void *query_ctx; 3271 void *hca_caps; 3272 int err; 3273 3274 *vhca_id = 0; 3275 if (mlx5_esw_is_manager_vport(esw, vport_num) || 3276 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) 3277 return -EPERM; 3278 3279 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 3280 if (!query_ctx) 3281 return -ENOMEM; 3282 3283 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx); 3284 if (err) 3285 goto out_free; 3286 3287 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 3288 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 3289 3290 out_free: 3291 kfree(query_ctx); 3292 return err; 3293 } 3294 3295 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 3296 { 3297 u16 *old_entry, *vhca_map_entry, vhca_id; 3298 int err; 3299 3300 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3301 if (err) { 3302 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 3303 vport_num, err); 3304 return err; 3305 } 3306 3307 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 3308 if (!vhca_map_entry) 3309 return -ENOMEM; 3310 3311 *vhca_map_entry = vport_num; 3312 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 3313 if (xa_is_err(old_entry)) { 3314 kfree(vhca_map_entry); 3315 return xa_err(old_entry); 3316 } 3317 kfree(old_entry); 3318 return 0; 3319 } 3320 3321 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 3322 { 3323 u16 *vhca_map_entry, vhca_id; 3324 int err; 3325 3326 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3327 if (err) 3328 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 3329 vport_num, err); 3330 3331 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 3332 kfree(vhca_map_entry); 3333 } 3334 3335 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 3336 { 3337 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 3338 3339 if (!res) 3340 return -ENOENT; 3341 3342 *vport_num = *res; 3343 return 0; 3344 } 3345 3346 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 3347 u16 vport_num) 3348 { 3349 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3350 3351 if (WARN_ON_ONCE(IS_ERR(vport))) 3352 return 0; 3353 3354 return vport->metadata; 3355 } 3356 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 3357