1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/indir_table.h" 42 #include "esw/acl/ofld.h" 43 #include "rdma.h" 44 #include "en.h" 45 #include "fs_core.h" 46 #include "lib/devcom.h" 47 #include "lib/eq.h" 48 #include "lib/fs_chains.h" 49 #include "en_tc.h" 50 #include "en/mapping.h" 51 #include "devlink.h" 52 #include "lag/lag.h" 53 54 #define mlx5_esw_for_each_rep(esw, i, rep) \ 55 xa_for_each(&((esw)->offloads.vport_reps), i, rep) 56 57 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \ 58 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF) 59 60 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \ 61 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \ 62 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF) 63 64 /* There are two match-all miss flows, one for unicast dst mac and 65 * one for multicast. 66 */ 67 #define MLX5_ESW_MISS_FLOWS (2) 68 #define UPLINK_REP_INDEX 0 69 70 #define MLX5_ESW_VPORT_TBL_SIZE 128 71 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 72 73 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 74 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 75 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 76 .flags = 0, 77 }; 78 79 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 80 u16 vport_num) 81 { 82 return xa_load(&esw->offloads.vport_reps, vport_num); 83 } 84 85 static void 86 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 87 struct mlx5_flow_spec *spec, 88 struct mlx5_esw_flow_attr *attr) 89 { 90 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 91 return; 92 93 if (attr->int_port) { 94 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 95 96 return; 97 } 98 99 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 100 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 101 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 102 } 103 104 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 105 * are not needed as well in the following process. So clear them all for simplicity. 106 */ 107 void 108 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 109 { 110 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 111 void *misc2; 112 113 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 114 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 115 116 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 117 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 118 119 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 120 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 121 } 122 } 123 124 static void 125 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 126 struct mlx5_flow_spec *spec, 127 struct mlx5_flow_attr *attr, 128 struct mlx5_eswitch *src_esw, 129 u16 vport) 130 { 131 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 132 u32 metadata; 133 void *misc2; 134 void *misc; 135 136 /* Use metadata matching because vport is not represented by single 137 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 138 */ 139 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 140 if (mlx5_esw_indir_table_decap_vport(attr)) 141 vport = mlx5_esw_indir_table_decap_vport(attr); 142 143 if (attr && !attr->chain && esw_attr->int_port) 144 metadata = 145 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 146 else 147 metadata = 148 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 149 150 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 151 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 152 153 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 154 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 155 mlx5_eswitch_get_vport_metadata_mask()); 156 157 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 158 } else { 159 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 160 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 161 162 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 163 MLX5_SET(fte_match_set_misc, misc, 164 source_eswitch_owner_vhca_id, 165 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 166 167 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 168 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 169 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 170 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 171 source_eswitch_owner_vhca_id); 172 173 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 174 } 175 } 176 177 static int 178 esw_setup_decap_indir(struct mlx5_eswitch *esw, 179 struct mlx5_flow_attr *attr, 180 struct mlx5_flow_spec *spec) 181 { 182 struct mlx5_flow_table *ft; 183 184 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 185 return -EOPNOTSUPP; 186 187 ft = mlx5_esw_indir_table_get(esw, attr, spec, 188 mlx5_esw_indir_table_decap_vport(attr), true); 189 return PTR_ERR_OR_ZERO(ft); 190 } 191 192 static void 193 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 194 struct mlx5_flow_attr *attr) 195 { 196 if (mlx5_esw_indir_table_decap_vport(attr)) 197 mlx5_esw_indir_table_put(esw, attr, 198 mlx5_esw_indir_table_decap_vport(attr), 199 true); 200 } 201 202 static int 203 esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 204 struct mlx5_flow_act *flow_act, 205 u32 sampler_id, 206 int i) 207 { 208 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 209 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 210 dest[i].sampler_id = sampler_id; 211 212 return 0; 213 } 214 215 static int 216 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 217 struct mlx5_flow_act *flow_act, 218 struct mlx5_eswitch *esw, 219 struct mlx5_flow_attr *attr, 220 struct mlx5_flow_spec *spec, 221 int i) 222 { 223 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 224 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 225 dest[i].ft = attr->dest_ft; 226 227 if (mlx5_esw_indir_table_decap_vport(attr)) 228 return esw_setup_decap_indir(esw, attr, spec); 229 return 0; 230 } 231 232 static void 233 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 234 struct mlx5_fs_chains *chains, int i) 235 { 236 if (mlx5_chains_ignore_flow_level_supported(chains)) 237 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 238 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 239 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 240 } 241 242 static void 243 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 244 struct mlx5_eswitch *esw, int i) 245 { 246 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 247 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 248 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 249 dest[i].ft = esw->fdb_table.offloads.slow_fdb; 250 } 251 252 static int 253 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 254 struct mlx5_flow_act *flow_act, 255 struct mlx5_fs_chains *chains, 256 u32 chain, u32 prio, u32 level, 257 int i) 258 { 259 struct mlx5_flow_table *ft; 260 261 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 262 ft = mlx5_chains_get_table(chains, chain, prio, level); 263 if (IS_ERR(ft)) 264 return PTR_ERR(ft); 265 266 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 267 dest[i].ft = ft; 268 return 0; 269 } 270 271 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 272 int from, int to) 273 { 274 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 275 struct mlx5_fs_chains *chains = esw_chains(esw); 276 int i; 277 278 for (i = from; i < to; i++) 279 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 280 mlx5_chains_put_table(chains, 0, 1, 0); 281 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 282 esw_attr->dests[i].mdev)) 283 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport, 284 false); 285 } 286 287 static bool 288 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 289 { 290 int i; 291 292 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 293 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 294 return true; 295 return false; 296 } 297 298 static int 299 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 300 struct mlx5_flow_act *flow_act, 301 struct mlx5_eswitch *esw, 302 struct mlx5_fs_chains *chains, 303 struct mlx5_flow_attr *attr, 304 int *i) 305 { 306 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 307 int err; 308 309 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 310 return -EOPNOTSUPP; 311 312 /* flow steering cannot handle more than one dest with the same ft 313 * in a single flow 314 */ 315 if (esw_attr->out_count - esw_attr->split_count > 1) 316 return -EOPNOTSUPP; 317 318 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 319 if (err) 320 return err; 321 322 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 323 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 324 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 325 } 326 (*i)++; 327 328 return 0; 329 } 330 331 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 332 struct mlx5_flow_attr *attr) 333 { 334 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 335 336 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 337 } 338 339 static bool 340 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 341 { 342 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 343 bool result = false; 344 int i; 345 346 /* Indirect table is supported only for flows with in_port uplink 347 * and the destination is vport on the same eswitch as the uplink, 348 * return false in case at least one of destinations doesn't meet 349 * this criteria. 350 */ 351 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 352 if (esw_attr->dests[i].rep && 353 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 354 esw_attr->dests[i].mdev)) { 355 result = true; 356 } else { 357 result = false; 358 break; 359 } 360 } 361 return result; 362 } 363 364 static int 365 esw_setup_indir_table(struct mlx5_flow_destination *dest, 366 struct mlx5_flow_act *flow_act, 367 struct mlx5_eswitch *esw, 368 struct mlx5_flow_attr *attr, 369 struct mlx5_flow_spec *spec, 370 bool ignore_flow_lvl, 371 int *i) 372 { 373 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 374 int j, err; 375 376 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 377 return -EOPNOTSUPP; 378 379 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 380 if (ignore_flow_lvl) 381 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 382 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 383 384 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec, 385 esw_attr->dests[j].rep->vport, false); 386 if (IS_ERR(dest[*i].ft)) { 387 err = PTR_ERR(dest[*i].ft); 388 goto err_indir_tbl_get; 389 } 390 } 391 392 if (mlx5_esw_indir_table_decap_vport(attr)) { 393 err = esw_setup_decap_indir(esw, attr, spec); 394 if (err) 395 goto err_indir_tbl_get; 396 } 397 398 return 0; 399 400 err_indir_tbl_get: 401 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 402 return err; 403 } 404 405 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 406 { 407 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 408 409 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 410 esw_cleanup_decap_indir(esw, attr); 411 } 412 413 static void 414 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 415 { 416 mlx5_chains_put_table(chains, chain, prio, level); 417 } 418 419 static void 420 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 421 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 422 int attr_idx, int dest_idx, bool pkt_reformat) 423 { 424 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 425 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport; 426 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 427 dest[dest_idx].vport.vhca_id = 428 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 429 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 430 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 431 mlx5_lag_mpesw_is_activated(esw->dev)) 432 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 433 } 434 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP) { 435 if (pkt_reformat) { 436 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 437 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 438 } 439 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 440 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 441 } 442 } 443 444 static int 445 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 446 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 447 int i) 448 { 449 int j; 450 451 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 452 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 453 return i; 454 } 455 456 static bool 457 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 458 { 459 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 460 mlx5_eswitch_vport_match_metadata_enabled(esw) && 461 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 462 } 463 464 static int 465 esw_setup_dests(struct mlx5_flow_destination *dest, 466 struct mlx5_flow_act *flow_act, 467 struct mlx5_eswitch *esw, 468 struct mlx5_flow_attr *attr, 469 struct mlx5_flow_spec *spec, 470 int *i) 471 { 472 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 473 struct mlx5_fs_chains *chains = esw_chains(esw); 474 int err = 0; 475 476 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 477 esw_src_port_rewrite_supported(esw)) 478 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 479 480 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE && 481 !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) { 482 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 483 (*i)++; 484 } else if (attr->dest_ft) { 485 esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i); 486 (*i)++; 487 } else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 488 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 489 (*i)++; 490 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 491 esw_setup_accept_dest(dest, flow_act, chains, *i); 492 (*i)++; 493 } else if (attr->dest_chain) { 494 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 495 1, 0, *i); 496 (*i)++; 497 } else if (esw_is_indir_table(esw, attr)) { 498 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i); 499 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 500 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 501 } else { 502 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 503 } 504 505 return err; 506 } 507 508 static void 509 esw_cleanup_dests(struct mlx5_eswitch *esw, 510 struct mlx5_flow_attr *attr) 511 { 512 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 513 struct mlx5_fs_chains *chains = esw_chains(esw); 514 515 if (attr->dest_ft) { 516 esw_cleanup_decap_indir(esw, attr); 517 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 518 if (attr->dest_chain) 519 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 520 else if (esw_is_indir_table(esw, attr)) 521 esw_cleanup_indir_table(esw, attr); 522 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 523 esw_cleanup_chain_src_port_rewrite(esw, attr); 524 } 525 } 526 527 static void 528 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 529 { 530 struct mlx5e_flow_meter_handle *meter; 531 532 meter = attr->meter_attr.meter; 533 flow_act->exe_aso.type = attr->exe_aso_type; 534 flow_act->exe_aso.object_id = meter->obj_id; 535 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 536 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 537 /* use metadata reg 5 for packet color */ 538 flow_act->exe_aso.return_reg_id = 5; 539 } 540 541 struct mlx5_flow_handle * 542 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 543 struct mlx5_flow_spec *spec, 544 struct mlx5_flow_attr *attr) 545 { 546 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 547 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 548 struct mlx5_fs_chains *chains = esw_chains(esw); 549 bool split = !!(esw_attr->split_count); 550 struct mlx5_vport_tbl_attr fwd_attr; 551 struct mlx5_flow_destination *dest; 552 struct mlx5_flow_handle *rule; 553 struct mlx5_flow_table *fdb; 554 int i = 0; 555 556 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 557 return ERR_PTR(-EOPNOTSUPP); 558 559 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 560 if (!dest) 561 return ERR_PTR(-ENOMEM); 562 563 flow_act.action = attr->action; 564 /* if per flow vlan pop/push is emulated, don't set that into the firmware */ 565 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 566 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | 567 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 568 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 569 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 570 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 571 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 572 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 573 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 574 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 575 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 576 } 577 } 578 579 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 580 581 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 582 int err; 583 584 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 585 if (err) { 586 rule = ERR_PTR(err); 587 goto err_create_goto_table; 588 } 589 } 590 591 if (esw_attr->decap_pkt_reformat) 592 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 593 594 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 595 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 596 dest[i].counter_id = mlx5_fc_id(attr->counter); 597 i++; 598 } 599 600 if (attr->outer_match_level != MLX5_MATCH_NONE) 601 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 602 if (attr->inner_match_level != MLX5_MATCH_NONE) 603 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 604 605 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 606 flow_act.modify_hdr = attr->modify_hdr; 607 608 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 609 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 610 esw_setup_meter(attr, &flow_act); 611 612 if (split) { 613 fwd_attr.chain = attr->chain; 614 fwd_attr.prio = attr->prio; 615 fwd_attr.vport = esw_attr->in_rep->vport; 616 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 617 618 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 619 } else { 620 if (attr->chain || attr->prio) 621 fdb = mlx5_chains_get_table(chains, attr->chain, 622 attr->prio, 0); 623 else 624 fdb = attr->ft; 625 626 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 627 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 628 esw_attr->in_mdev->priv.eswitch, 629 esw_attr->in_rep->vport); 630 } 631 if (IS_ERR(fdb)) { 632 rule = ERR_CAST(fdb); 633 goto err_esw_get; 634 } 635 636 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 637 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 638 &flow_act, dest, i); 639 else 640 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 641 if (IS_ERR(rule)) 642 goto err_add_rule; 643 else 644 atomic64_inc(&esw->offloads.num_flows); 645 646 kfree(dest); 647 return rule; 648 649 err_add_rule: 650 if (split) 651 mlx5_esw_vporttbl_put(esw, &fwd_attr); 652 else if (attr->chain || attr->prio) 653 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 654 err_esw_get: 655 esw_cleanup_dests(esw, attr); 656 err_create_goto_table: 657 kfree(dest); 658 return rule; 659 } 660 661 struct mlx5_flow_handle * 662 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 663 struct mlx5_flow_spec *spec, 664 struct mlx5_flow_attr *attr) 665 { 666 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 667 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 668 struct mlx5_fs_chains *chains = esw_chains(esw); 669 struct mlx5_vport_tbl_attr fwd_attr; 670 struct mlx5_flow_destination *dest; 671 struct mlx5_flow_table *fast_fdb; 672 struct mlx5_flow_table *fwd_fdb; 673 struct mlx5_flow_handle *rule; 674 int i, err = 0; 675 676 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 677 if (!dest) 678 return ERR_PTR(-ENOMEM); 679 680 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 681 if (IS_ERR(fast_fdb)) { 682 rule = ERR_CAST(fast_fdb); 683 goto err_get_fast; 684 } 685 686 fwd_attr.chain = attr->chain; 687 fwd_attr.prio = attr->prio; 688 fwd_attr.vport = esw_attr->in_rep->vport; 689 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 690 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 691 if (IS_ERR(fwd_fdb)) { 692 rule = ERR_CAST(fwd_fdb); 693 goto err_get_fwd; 694 } 695 696 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 697 for (i = 0; i < esw_attr->split_count; i++) { 698 if (esw_is_indir_table(esw, attr)) 699 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i); 700 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 701 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr, 702 &i); 703 else 704 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 705 706 if (err) { 707 rule = ERR_PTR(err); 708 goto err_chain_src_rewrite; 709 } 710 } 711 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 712 dest[i].ft = fwd_fdb; 713 i++; 714 715 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 716 esw_attr->in_mdev->priv.eswitch, 717 esw_attr->in_rep->vport); 718 719 if (attr->outer_match_level != MLX5_MATCH_NONE) 720 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 721 722 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 723 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 724 725 if (IS_ERR(rule)) { 726 i = esw_attr->split_count; 727 goto err_chain_src_rewrite; 728 } 729 730 atomic64_inc(&esw->offloads.num_flows); 731 732 kfree(dest); 733 return rule; 734 err_chain_src_rewrite: 735 esw_put_dest_tables_loop(esw, attr, 0, i); 736 mlx5_esw_vporttbl_put(esw, &fwd_attr); 737 err_get_fwd: 738 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 739 err_get_fast: 740 kfree(dest); 741 return rule; 742 } 743 744 static void 745 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 746 struct mlx5_flow_handle *rule, 747 struct mlx5_flow_attr *attr, 748 bool fwd_rule) 749 { 750 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 751 struct mlx5_fs_chains *chains = esw_chains(esw); 752 bool split = (esw_attr->split_count > 0); 753 struct mlx5_vport_tbl_attr fwd_attr; 754 int i; 755 756 mlx5_del_flow_rules(rule); 757 758 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 759 /* unref the term table */ 760 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 761 if (esw_attr->dests[i].termtbl) 762 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 763 } 764 } 765 766 atomic64_dec(&esw->offloads.num_flows); 767 768 if (fwd_rule || split) { 769 fwd_attr.chain = attr->chain; 770 fwd_attr.prio = attr->prio; 771 fwd_attr.vport = esw_attr->in_rep->vport; 772 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 773 } 774 775 if (fwd_rule) { 776 mlx5_esw_vporttbl_put(esw, &fwd_attr); 777 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 778 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count); 779 } else { 780 if (split) 781 mlx5_esw_vporttbl_put(esw, &fwd_attr); 782 else if (attr->chain || attr->prio) 783 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 784 esw_cleanup_dests(esw, attr); 785 } 786 } 787 788 void 789 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 790 struct mlx5_flow_handle *rule, 791 struct mlx5_flow_attr *attr) 792 { 793 __mlx5_eswitch_del_rule(esw, rule, attr, false); 794 } 795 796 void 797 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 798 struct mlx5_flow_handle *rule, 799 struct mlx5_flow_attr *attr) 800 { 801 __mlx5_eswitch_del_rule(esw, rule, attr, true); 802 } 803 804 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) 805 { 806 struct mlx5_eswitch_rep *rep; 807 unsigned long i; 808 int err = 0; 809 810 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); 811 mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) { 812 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 813 continue; 814 815 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); 816 if (err) 817 goto out; 818 } 819 820 out: 821 return err; 822 } 823 824 static struct mlx5_eswitch_rep * 825 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) 826 { 827 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; 828 829 in_rep = attr->in_rep; 830 out_rep = attr->dests[0].rep; 831 832 if (push) 833 vport = in_rep; 834 else if (pop) 835 vport = out_rep; 836 else 837 vport = in_rep; 838 839 return vport; 840 } 841 842 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, 843 bool push, bool pop, bool fwd) 844 { 845 struct mlx5_eswitch_rep *in_rep, *out_rep; 846 847 if ((push || pop) && !fwd) 848 goto out_notsupp; 849 850 in_rep = attr->in_rep; 851 out_rep = attr->dests[0].rep; 852 853 if (push && in_rep->vport == MLX5_VPORT_UPLINK) 854 goto out_notsupp; 855 856 if (pop && out_rep->vport == MLX5_VPORT_UPLINK) 857 goto out_notsupp; 858 859 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ 860 if (!push && !pop && fwd) 861 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK) 862 goto out_notsupp; 863 864 /* protects against (1) setting rules with different vlans to push and 865 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) 866 */ 867 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0])) 868 goto out_notsupp; 869 870 return 0; 871 872 out_notsupp: 873 return -EOPNOTSUPP; 874 } 875 876 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, 877 struct mlx5_flow_attr *attr) 878 { 879 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 880 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 881 struct mlx5_eswitch_rep *vport = NULL; 882 bool push, pop, fwd; 883 int err = 0; 884 885 /* nop if we're on the vlan push/pop non emulation mode */ 886 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 887 return 0; 888 889 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 890 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 891 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && 892 !attr->dest_chain); 893 894 mutex_lock(&esw->state_lock); 895 896 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd); 897 if (err) 898 goto unlock; 899 900 attr->flags &= ~MLX5_ATTR_FLAG_VLAN_HANDLED; 901 902 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 903 904 if (!push && !pop && fwd) { 905 /* tracks VF --> wire rules without vlan push action */ 906 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) { 907 vport->vlan_refcount++; 908 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED; 909 } 910 911 goto unlock; 912 } 913 914 if (!push && !pop) 915 goto unlock; 916 917 if (!(offloads->vlan_push_pop_refcount)) { 918 /* it's the 1st vlan rule, apply global vlan pop policy */ 919 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); 920 if (err) 921 goto out; 922 } 923 offloads->vlan_push_pop_refcount++; 924 925 if (push) { 926 if (vport->vlan_refcount) 927 goto skip_set_push; 928 929 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0], 930 0, SET_VLAN_INSERT | SET_VLAN_STRIP); 931 if (err) 932 goto out; 933 vport->vlan = esw_attr->vlan_vid[0]; 934 skip_set_push: 935 vport->vlan_refcount++; 936 } 937 out: 938 if (!err) 939 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED; 940 unlock: 941 mutex_unlock(&esw->state_lock); 942 return err; 943 } 944 945 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, 946 struct mlx5_flow_attr *attr) 947 { 948 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 949 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 950 struct mlx5_eswitch_rep *vport = NULL; 951 bool push, pop, fwd; 952 int err = 0; 953 954 /* nop if we're on the vlan push/pop non emulation mode */ 955 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 956 return 0; 957 958 if (!(attr->flags & MLX5_ATTR_FLAG_VLAN_HANDLED)) 959 return 0; 960 961 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 962 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 963 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 964 965 mutex_lock(&esw->state_lock); 966 967 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 968 969 if (!push && !pop && fwd) { 970 /* tracks VF --> wire rules without vlan push action */ 971 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) 972 vport->vlan_refcount--; 973 974 goto out; 975 } 976 977 if (push) { 978 vport->vlan_refcount--; 979 if (vport->vlan_refcount) 980 goto skip_unset_push; 981 982 vport->vlan = 0; 983 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, 984 0, 0, SET_VLAN_STRIP); 985 if (err) 986 goto out; 987 } 988 989 skip_unset_push: 990 offloads->vlan_push_pop_refcount--; 991 if (offloads->vlan_push_pop_refcount) 992 goto out; 993 994 /* no more vlan rules, stop global vlan pop policy */ 995 err = esw_set_global_vlan_pop(esw, 0); 996 997 out: 998 mutex_unlock(&esw->state_lock); 999 return err; 1000 } 1001 1002 struct mlx5_flow_handle * 1003 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 1004 struct mlx5_eswitch *from_esw, 1005 struct mlx5_eswitch_rep *rep, 1006 u32 sqn) 1007 { 1008 struct mlx5_flow_act flow_act = {0}; 1009 struct mlx5_flow_destination dest = {}; 1010 struct mlx5_flow_handle *flow_rule; 1011 struct mlx5_flow_spec *spec; 1012 void *misc; 1013 1014 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1015 if (!spec) { 1016 flow_rule = ERR_PTR(-ENOMEM); 1017 goto out; 1018 } 1019 1020 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1021 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 1022 /* source vport is the esw manager */ 1023 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport); 1024 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 1025 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1026 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 1027 1028 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1029 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 1030 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1031 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 1032 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1033 source_eswitch_owner_vhca_id); 1034 1035 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1036 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1037 dest.vport.num = rep->vport; 1038 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1039 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1040 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1041 1042 if (rep->vport == MLX5_VPORT_UPLINK) 1043 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1044 1045 flow_rule = mlx5_add_flow_rules(on_esw->fdb_table.offloads.slow_fdb, 1046 spec, &flow_act, &dest, 1); 1047 if (IS_ERR(flow_rule)) 1048 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", 1049 PTR_ERR(flow_rule)); 1050 out: 1051 kvfree(spec); 1052 return flow_rule; 1053 } 1054 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1055 1056 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1057 { 1058 mlx5_del_flow_rules(rule); 1059 } 1060 1061 static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) 1062 { 1063 struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules; 1064 int i = 0, num_vfs = esw->esw_funcs.num_vfs; 1065 1066 if (!num_vfs || !flows) 1067 return; 1068 1069 for (i = 0; i < num_vfs; i++) 1070 mlx5_del_flow_rules(flows[i]); 1071 1072 kvfree(flows); 1073 /* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0, 1074 * meta rules could be freed again. So set it to NULL. 1075 */ 1076 esw->fdb_table.offloads.send_to_vport_meta_rules = NULL; 1077 } 1078 1079 void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) 1080 { 1081 mlx5_eswitch_del_send_to_vport_meta_rules(esw); 1082 } 1083 1084 static int 1085 mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw) 1086 { 1087 struct mlx5_flow_destination dest = {}; 1088 struct mlx5_flow_act flow_act = {0}; 1089 int num_vfs, rule_idx = 0, err = 0; 1090 struct mlx5_flow_handle *flow_rule; 1091 struct mlx5_flow_handle **flows; 1092 struct mlx5_flow_spec *spec; 1093 struct mlx5_vport *vport; 1094 unsigned long i; 1095 u16 vport_num; 1096 1097 num_vfs = esw->esw_funcs.num_vfs; 1098 flows = kvcalloc(num_vfs, sizeof(*flows), GFP_KERNEL); 1099 if (!flows) 1100 return -ENOMEM; 1101 1102 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1103 if (!spec) { 1104 err = -ENOMEM; 1105 goto alloc_err; 1106 } 1107 1108 MLX5_SET(fte_match_param, spec->match_criteria, 1109 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1110 MLX5_SET(fte_match_param, spec->match_criteria, 1111 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1112 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1113 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1114 1115 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1116 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1117 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1118 1119 mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) { 1120 vport_num = vport->vport; 1121 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1122 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1123 dest.vport.num = vport_num; 1124 1125 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1126 spec, &flow_act, &dest, 1); 1127 if (IS_ERR(flow_rule)) { 1128 err = PTR_ERR(flow_rule); 1129 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n", 1130 rule_idx, PTR_ERR(flow_rule)); 1131 goto rule_err; 1132 } 1133 flows[rule_idx++] = flow_rule; 1134 } 1135 1136 esw->fdb_table.offloads.send_to_vport_meta_rules = flows; 1137 kvfree(spec); 1138 return 0; 1139 1140 rule_err: 1141 while (--rule_idx >= 0) 1142 mlx5_del_flow_rules(flows[rule_idx]); 1143 kvfree(spec); 1144 alloc_err: 1145 kvfree(flows); 1146 return err; 1147 } 1148 1149 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1150 { 1151 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1152 MLX5_FDB_TO_VPORT_REG_C_1; 1153 } 1154 1155 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1156 { 1157 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1158 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1159 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1160 u8 curr, wanted; 1161 int err; 1162 1163 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1164 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1165 return 0; 1166 1167 MLX5_SET(query_esw_vport_context_in, in, opcode, 1168 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1169 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1170 if (err) 1171 return err; 1172 1173 curr = MLX5_GET(query_esw_vport_context_out, out, 1174 esw_vport_context.fdb_to_vport_reg_c_id); 1175 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1176 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1177 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1178 1179 if (enable) 1180 curr |= wanted; 1181 else 1182 curr &= ~wanted; 1183 1184 MLX5_SET(modify_esw_vport_context_in, min, 1185 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1186 MLX5_SET(modify_esw_vport_context_in, min, 1187 field_select.fdb_to_vport_reg_c_id, 1); 1188 1189 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1190 if (!err) { 1191 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1192 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1193 else 1194 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1195 } 1196 1197 return err; 1198 } 1199 1200 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1201 struct mlx5_core_dev *peer_dev, 1202 struct mlx5_flow_spec *spec, 1203 struct mlx5_flow_destination *dest) 1204 { 1205 void *misc; 1206 1207 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1208 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1209 misc_parameters_2); 1210 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1211 mlx5_eswitch_get_vport_metadata_mask()); 1212 1213 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1214 } else { 1215 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1216 misc_parameters); 1217 1218 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1219 MLX5_CAP_GEN(peer_dev, vhca_id)); 1220 1221 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1222 1223 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1224 misc_parameters); 1225 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1226 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1227 source_eswitch_owner_vhca_id); 1228 } 1229 1230 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1231 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1232 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1233 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1234 } 1235 1236 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1237 struct mlx5_eswitch *peer_esw, 1238 struct mlx5_flow_spec *spec, 1239 u16 vport) 1240 { 1241 void *misc; 1242 1243 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1244 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1245 misc_parameters_2); 1246 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1247 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1248 vport)); 1249 } else { 1250 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1251 misc_parameters); 1252 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1253 } 1254 } 1255 1256 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1257 struct mlx5_core_dev *peer_dev) 1258 { 1259 struct mlx5_flow_destination dest = {}; 1260 struct mlx5_flow_act flow_act = {0}; 1261 struct mlx5_flow_handle **flows; 1262 /* total vports is the same for both e-switches */ 1263 int nvports = esw->total_vports; 1264 struct mlx5_flow_handle *flow; 1265 struct mlx5_flow_spec *spec; 1266 struct mlx5_vport *vport; 1267 unsigned long i; 1268 void *misc; 1269 int err; 1270 1271 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1272 if (!spec) 1273 return -ENOMEM; 1274 1275 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1276 1277 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); 1278 if (!flows) { 1279 err = -ENOMEM; 1280 goto alloc_flows_err; 1281 } 1282 1283 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1284 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1285 misc_parameters); 1286 1287 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1288 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1289 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1290 spec, MLX5_VPORT_PF); 1291 1292 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1293 spec, &flow_act, &dest, 1); 1294 if (IS_ERR(flow)) { 1295 err = PTR_ERR(flow); 1296 goto add_pf_flow_err; 1297 } 1298 flows[vport->index] = flow; 1299 } 1300 1301 if (mlx5_ecpf_vport_exists(esw->dev)) { 1302 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1303 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1304 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1305 spec, &flow_act, &dest, 1); 1306 if (IS_ERR(flow)) { 1307 err = PTR_ERR(flow); 1308 goto add_ecpf_flow_err; 1309 } 1310 flows[vport->index] = flow; 1311 } 1312 1313 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1314 esw_set_peer_miss_rule_source_port(esw, 1315 peer_dev->priv.eswitch, 1316 spec, vport->vport); 1317 1318 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1319 spec, &flow_act, &dest, 1); 1320 if (IS_ERR(flow)) { 1321 err = PTR_ERR(flow); 1322 goto add_vf_flow_err; 1323 } 1324 flows[vport->index] = flow; 1325 } 1326 1327 esw->fdb_table.offloads.peer_miss_rules = flows; 1328 1329 kvfree(spec); 1330 return 0; 1331 1332 add_vf_flow_err: 1333 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1334 if (!flows[vport->index]) 1335 continue; 1336 mlx5_del_flow_rules(flows[vport->index]); 1337 } 1338 if (mlx5_ecpf_vport_exists(esw->dev)) { 1339 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1340 mlx5_del_flow_rules(flows[vport->index]); 1341 } 1342 add_ecpf_flow_err: 1343 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1344 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1345 mlx5_del_flow_rules(flows[vport->index]); 1346 } 1347 add_pf_flow_err: 1348 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1349 kvfree(flows); 1350 alloc_flows_err: 1351 kvfree(spec); 1352 return err; 1353 } 1354 1355 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) 1356 { 1357 struct mlx5_flow_handle **flows; 1358 struct mlx5_vport *vport; 1359 unsigned long i; 1360 1361 flows = esw->fdb_table.offloads.peer_miss_rules; 1362 1363 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) 1364 mlx5_del_flow_rules(flows[vport->index]); 1365 1366 if (mlx5_ecpf_vport_exists(esw->dev)) { 1367 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1368 mlx5_del_flow_rules(flows[vport->index]); 1369 } 1370 1371 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1372 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1373 mlx5_del_flow_rules(flows[vport->index]); 1374 } 1375 kvfree(flows); 1376 } 1377 1378 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1379 { 1380 struct mlx5_flow_act flow_act = {0}; 1381 struct mlx5_flow_destination dest = {}; 1382 struct mlx5_flow_handle *flow_rule = NULL; 1383 struct mlx5_flow_spec *spec; 1384 void *headers_c; 1385 void *headers_v; 1386 int err = 0; 1387 u8 *dmac_c; 1388 u8 *dmac_v; 1389 1390 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1391 if (!spec) { 1392 err = -ENOMEM; 1393 goto out; 1394 } 1395 1396 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1397 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1398 outer_headers); 1399 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1400 outer_headers.dmac_47_16); 1401 dmac_c[0] = 0x01; 1402 1403 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1404 dest.vport.num = esw->manager_vport; 1405 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1406 1407 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1408 spec, &flow_act, &dest, 1); 1409 if (IS_ERR(flow_rule)) { 1410 err = PTR_ERR(flow_rule); 1411 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1412 goto out; 1413 } 1414 1415 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1416 1417 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1418 outer_headers); 1419 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1420 outer_headers.dmac_47_16); 1421 dmac_v[0] = 0x01; 1422 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1423 spec, &flow_act, &dest, 1); 1424 if (IS_ERR(flow_rule)) { 1425 err = PTR_ERR(flow_rule); 1426 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1427 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1428 goto out; 1429 } 1430 1431 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1432 1433 out: 1434 kvfree(spec); 1435 return err; 1436 } 1437 1438 struct mlx5_flow_handle * 1439 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1440 { 1441 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1442 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1443 struct mlx5_flow_context *flow_context; 1444 struct mlx5_flow_handle *flow_rule; 1445 struct mlx5_flow_destination dest; 1446 struct mlx5_flow_spec *spec; 1447 void *misc; 1448 1449 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1450 return ERR_PTR(-EOPNOTSUPP); 1451 1452 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1453 if (!spec) 1454 return ERR_PTR(-ENOMEM); 1455 1456 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1457 misc_parameters_2); 1458 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1459 ESW_REG_C0_USER_DATA_METADATA_MASK); 1460 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1461 misc_parameters_2); 1462 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1463 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1464 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1465 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1466 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1467 1468 flow_context = &spec->flow_context; 1469 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1470 flow_context->flow_tag = tag; 1471 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1472 dest.ft = esw->offloads.ft_offloads; 1473 1474 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1475 kvfree(spec); 1476 1477 if (IS_ERR(flow_rule)) 1478 esw_warn(esw->dev, 1479 "Failed to create restore rule for tag: %d, err(%d)\n", 1480 tag, (int)PTR_ERR(flow_rule)); 1481 1482 return flow_rule; 1483 } 1484 1485 #define MAX_PF_SQ 256 1486 #define MAX_SQ_NVPORTS 32 1487 1488 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1489 u32 *flow_group_in) 1490 { 1491 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1492 flow_group_in, 1493 match_criteria); 1494 1495 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1496 MLX5_SET(create_flow_group_in, flow_group_in, 1497 match_criteria_enable, 1498 MLX5_MATCH_MISC_PARAMETERS_2); 1499 1500 MLX5_SET(fte_match_param, match_criteria, 1501 misc_parameters_2.metadata_reg_c_0, 1502 mlx5_eswitch_get_vport_metadata_mask()); 1503 } else { 1504 MLX5_SET(create_flow_group_in, flow_group_in, 1505 match_criteria_enable, 1506 MLX5_MATCH_MISC_PARAMETERS); 1507 1508 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1509 misc_parameters.source_port); 1510 } 1511 } 1512 1513 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1514 static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1515 { 1516 struct mlx5_vport_tbl_attr attr; 1517 struct mlx5_vport *vport; 1518 unsigned long i; 1519 1520 attr.chain = 0; 1521 attr.prio = 1; 1522 mlx5_esw_for_each_vport(esw, i, vport) { 1523 attr.vport = vport->vport; 1524 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1525 mlx5_esw_vporttbl_put(esw, &attr); 1526 } 1527 } 1528 1529 static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1530 { 1531 struct mlx5_vport_tbl_attr attr; 1532 struct mlx5_flow_table *fdb; 1533 struct mlx5_vport *vport; 1534 unsigned long i; 1535 1536 attr.chain = 0; 1537 attr.prio = 1; 1538 mlx5_esw_for_each_vport(esw, i, vport) { 1539 attr.vport = vport->vport; 1540 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1541 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1542 if (IS_ERR(fdb)) 1543 goto out; 1544 } 1545 return 0; 1546 1547 out: 1548 esw_vport_tbl_put(esw); 1549 return PTR_ERR(fdb); 1550 } 1551 1552 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1553 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1554 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1555 { 1556 struct mlx5_core_dev *dev = esw->dev; 1557 1558 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1559 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1560 1561 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1562 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1563 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1564 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1565 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1566 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1567 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1568 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1569 /* Disabled when ttl workaround is needed, e.g 1570 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1571 */ 1572 esw_warn(dev, 1573 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1574 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1575 } else { 1576 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1577 esw_info(dev, "Supported tc chains and prios offload\n"); 1578 } 1579 1580 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1581 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1582 } 1583 1584 static int 1585 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1586 { 1587 struct mlx5_core_dev *dev = esw->dev; 1588 struct mlx5_flow_table *nf_ft, *ft; 1589 struct mlx5_chains_attr attr = {}; 1590 struct mlx5_fs_chains *chains; 1591 u32 fdb_max; 1592 int err; 1593 1594 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size); 1595 1596 esw_init_chains_offload_flags(esw, &attr.flags); 1597 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1598 attr.max_ft_sz = fdb_max; 1599 attr.max_grp_num = esw->params.large_group_num; 1600 attr.default_ft = miss_fdb; 1601 attr.mapping = esw->offloads.reg_c0_obj_pool; 1602 1603 chains = mlx5_chains_create(dev, &attr); 1604 if (IS_ERR(chains)) { 1605 err = PTR_ERR(chains); 1606 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1607 return err; 1608 } 1609 1610 esw->fdb_table.offloads.esw_chains_priv = chains; 1611 1612 /* Create tc_end_ft which is the always created ft chain */ 1613 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1614 1, 0); 1615 if (IS_ERR(nf_ft)) { 1616 err = PTR_ERR(nf_ft); 1617 goto nf_ft_err; 1618 } 1619 1620 /* Always open the root for fast path */ 1621 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1622 if (IS_ERR(ft)) { 1623 err = PTR_ERR(ft); 1624 goto level_0_err; 1625 } 1626 1627 /* Open level 1 for split fdb rules now if prios isn't supported */ 1628 if (!mlx5_chains_prios_supported(chains)) { 1629 err = esw_vport_tbl_get(esw); 1630 if (err) 1631 goto level_1_err; 1632 } 1633 1634 mlx5_chains_set_end_ft(chains, nf_ft); 1635 1636 return 0; 1637 1638 level_1_err: 1639 mlx5_chains_put_table(chains, 0, 1, 0); 1640 level_0_err: 1641 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1642 nf_ft_err: 1643 mlx5_chains_destroy(chains); 1644 esw->fdb_table.offloads.esw_chains_priv = NULL; 1645 1646 return err; 1647 } 1648 1649 static void 1650 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1651 { 1652 if (!mlx5_chains_prios_supported(chains)) 1653 esw_vport_tbl_put(esw); 1654 mlx5_chains_put_table(chains, 0, 1, 0); 1655 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1656 mlx5_chains_destroy(chains); 1657 } 1658 1659 #else /* CONFIG_MLX5_CLS_ACT */ 1660 1661 static int 1662 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1663 { return 0; } 1664 1665 static void 1666 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1667 {} 1668 1669 #endif 1670 1671 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1672 { 1673 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1674 struct mlx5_flow_table_attr ft_attr = {}; 1675 int num_vfs, table_size, ix, err = 0; 1676 struct mlx5_core_dev *dev = esw->dev; 1677 struct mlx5_flow_namespace *root_ns; 1678 struct mlx5_flow_table *fdb = NULL; 1679 u32 flags = 0, *flow_group_in; 1680 struct mlx5_flow_group *g; 1681 void *match_criteria; 1682 u8 *dmac; 1683 1684 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1685 1686 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1687 if (!flow_group_in) 1688 return -ENOMEM; 1689 1690 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1691 if (!root_ns) { 1692 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1693 err = -EOPNOTSUPP; 1694 goto ns_err; 1695 } 1696 esw->fdb_table.offloads.ns = root_ns; 1697 err = mlx5_flow_namespace_set_mode(root_ns, 1698 esw->dev->priv.steering->mode); 1699 if (err) { 1700 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1701 goto ns_err; 1702 } 1703 1704 /* To be strictly correct: 1705 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1706 * should be: 1707 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1708 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1709 * but as the peer device might not be in switchdev mode it's not 1710 * possible. We use the fact that by default FW sets max vfs and max sfs 1711 * to the same value on both devices. If it needs to be changed in the future note 1712 * the peer miss group should also be created based on the number of 1713 * total vports of the peer (currently is also uses esw->total_vports). 1714 */ 1715 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1716 MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs; 1717 1718 /* create the slow path fdb with encap set, so further table instances 1719 * can be created at run time while VFs are probed if the FW allows that. 1720 */ 1721 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1722 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1723 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1724 1725 ft_attr.flags = flags; 1726 ft_attr.max_fte = table_size; 1727 ft_attr.prio = FDB_SLOW_PATH; 1728 1729 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1730 if (IS_ERR(fdb)) { 1731 err = PTR_ERR(fdb); 1732 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1733 goto slow_fdb_err; 1734 } 1735 esw->fdb_table.offloads.slow_fdb = fdb; 1736 1737 /* Create empty TC-miss managed table. This allows plugging in following 1738 * priorities without directly exposing their level 0 table to 1739 * eswitch_offloads and passing it as miss_fdb to following call to 1740 * esw_chains_create(). 1741 */ 1742 memset(&ft_attr, 0, sizeof(ft_attr)); 1743 ft_attr.prio = FDB_TC_MISS; 1744 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1745 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1746 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1747 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1748 goto tc_miss_table_err; 1749 } 1750 1751 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1752 if (err) { 1753 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1754 goto fdb_chains_err; 1755 } 1756 1757 /* create send-to-vport group */ 1758 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1759 MLX5_MATCH_MISC_PARAMETERS); 1760 1761 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1762 1763 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1764 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 1765 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1766 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1767 misc_parameters.source_eswitch_owner_vhca_id); 1768 MLX5_SET(create_flow_group_in, flow_group_in, 1769 source_eswitch_owner_vhca_id_valid, 1); 1770 } 1771 1772 /* See comment above table_size calculation */ 1773 ix = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1774 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1775 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); 1776 1777 g = mlx5_create_flow_group(fdb, flow_group_in); 1778 if (IS_ERR(g)) { 1779 err = PTR_ERR(g); 1780 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1781 goto send_vport_err; 1782 } 1783 esw->fdb_table.offloads.send_to_vport_grp = g; 1784 1785 if (esw_src_port_rewrite_supported(esw)) { 1786 /* meta send to vport */ 1787 memset(flow_group_in, 0, inlen); 1788 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1789 MLX5_MATCH_MISC_PARAMETERS_2); 1790 1791 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1792 1793 MLX5_SET(fte_match_param, match_criteria, 1794 misc_parameters_2.metadata_reg_c_0, 1795 mlx5_eswitch_get_vport_metadata_mask()); 1796 MLX5_SET(fte_match_param, match_criteria, 1797 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1798 1799 num_vfs = esw->esw_funcs.num_vfs; 1800 if (num_vfs) { 1801 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1802 MLX5_SET(create_flow_group_in, flow_group_in, 1803 end_flow_index, ix + num_vfs - 1); 1804 ix += num_vfs; 1805 1806 g = mlx5_create_flow_group(fdb, flow_group_in); 1807 if (IS_ERR(g)) { 1808 err = PTR_ERR(g); 1809 esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n", 1810 err); 1811 goto send_vport_meta_err; 1812 } 1813 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1814 1815 err = mlx5_eswitch_add_send_to_vport_meta_rules(esw); 1816 if (err) 1817 goto meta_rule_err; 1818 } 1819 } 1820 1821 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1822 /* create peer esw miss group */ 1823 memset(flow_group_in, 0, inlen); 1824 1825 esw_set_flow_group_source_port(esw, flow_group_in); 1826 1827 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1828 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1829 flow_group_in, 1830 match_criteria); 1831 1832 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1833 misc_parameters.source_eswitch_owner_vhca_id); 1834 1835 MLX5_SET(create_flow_group_in, flow_group_in, 1836 source_eswitch_owner_vhca_id_valid, 1); 1837 } 1838 1839 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1840 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1841 ix + esw->total_vports - 1); 1842 ix += esw->total_vports; 1843 1844 g = mlx5_create_flow_group(fdb, flow_group_in); 1845 if (IS_ERR(g)) { 1846 err = PTR_ERR(g); 1847 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err); 1848 goto peer_miss_err; 1849 } 1850 esw->fdb_table.offloads.peer_miss_grp = g; 1851 } 1852 1853 /* create miss group */ 1854 memset(flow_group_in, 0, inlen); 1855 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1856 MLX5_MATCH_OUTER_HEADERS); 1857 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1858 match_criteria); 1859 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1860 outer_headers.dmac_47_16); 1861 dmac[0] = 0x01; 1862 1863 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1864 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1865 ix + MLX5_ESW_MISS_FLOWS); 1866 1867 g = mlx5_create_flow_group(fdb, flow_group_in); 1868 if (IS_ERR(g)) { 1869 err = PTR_ERR(g); 1870 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err); 1871 goto miss_err; 1872 } 1873 esw->fdb_table.offloads.miss_grp = g; 1874 1875 err = esw_add_fdb_miss_rule(esw); 1876 if (err) 1877 goto miss_rule_err; 1878 1879 kvfree(flow_group_in); 1880 return 0; 1881 1882 miss_rule_err: 1883 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1884 miss_err: 1885 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1886 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1887 peer_miss_err: 1888 mlx5_eswitch_del_send_to_vport_meta_rules(esw); 1889 meta_rule_err: 1890 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1891 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1892 send_vport_meta_err: 1893 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1894 send_vport_err: 1895 esw_chains_destroy(esw, esw_chains(esw)); 1896 fdb_chains_err: 1897 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1898 tc_miss_table_err: 1899 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1900 slow_fdb_err: 1901 /* Holds true only as long as DMFS is the default */ 1902 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1903 ns_err: 1904 kvfree(flow_group_in); 1905 return err; 1906 } 1907 1908 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1909 { 1910 if (!esw->fdb_table.offloads.slow_fdb) 1911 return; 1912 1913 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1914 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1915 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1916 mlx5_eswitch_del_send_to_vport_meta_rules(esw); 1917 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1918 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1919 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1920 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1921 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1922 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1923 1924 esw_chains_destroy(esw, esw_chains(esw)); 1925 1926 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1927 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1928 /* Holds true only as long as DMFS is the default */ 1929 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1930 MLX5_FLOW_STEERING_MODE_DMFS); 1931 atomic64_set(&esw->user_count, 0); 1932 } 1933 1934 static int esw_get_offloads_ft_size(struct mlx5_eswitch *esw) 1935 { 1936 int nvports; 1937 1938 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1939 if (mlx5e_tc_int_port_supported(esw)) 1940 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 1941 1942 return nvports; 1943 } 1944 1945 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1946 { 1947 struct mlx5_flow_table_attr ft_attr = {}; 1948 struct mlx5_core_dev *dev = esw->dev; 1949 struct mlx5_flow_table *ft_offloads; 1950 struct mlx5_flow_namespace *ns; 1951 int err = 0; 1952 1953 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1954 if (!ns) { 1955 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1956 return -EOPNOTSUPP; 1957 } 1958 1959 ft_attr.max_fte = esw_get_offloads_ft_size(esw); 1960 ft_attr.prio = 1; 1961 1962 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 1963 if (IS_ERR(ft_offloads)) { 1964 err = PTR_ERR(ft_offloads); 1965 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 1966 return err; 1967 } 1968 1969 esw->offloads.ft_offloads = ft_offloads; 1970 return 0; 1971 } 1972 1973 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 1974 { 1975 struct mlx5_esw_offload *offloads = &esw->offloads; 1976 1977 mlx5_destroy_flow_table(offloads->ft_offloads); 1978 } 1979 1980 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 1981 { 1982 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1983 struct mlx5_flow_group *g; 1984 u32 *flow_group_in; 1985 int nvports; 1986 int err = 0; 1987 1988 nvports = esw_get_offloads_ft_size(esw); 1989 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1990 if (!flow_group_in) 1991 return -ENOMEM; 1992 1993 /* create vport rx group */ 1994 esw_set_flow_group_source_port(esw, flow_group_in); 1995 1996 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1997 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 1998 1999 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2000 2001 if (IS_ERR(g)) { 2002 err = PTR_ERR(g); 2003 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 2004 goto out; 2005 } 2006 2007 esw->offloads.vport_rx_group = g; 2008 out: 2009 kvfree(flow_group_in); 2010 return err; 2011 } 2012 2013 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2014 { 2015 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2016 } 2017 2018 struct mlx5_flow_handle * 2019 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2020 struct mlx5_flow_destination *dest) 2021 { 2022 struct mlx5_flow_act flow_act = {0}; 2023 struct mlx5_flow_handle *flow_rule; 2024 struct mlx5_flow_spec *spec; 2025 void *misc; 2026 2027 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2028 if (!spec) { 2029 flow_rule = ERR_PTR(-ENOMEM); 2030 goto out; 2031 } 2032 2033 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2034 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2035 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2036 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2037 2038 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2039 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2040 mlx5_eswitch_get_vport_metadata_mask()); 2041 2042 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2043 } else { 2044 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2045 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2046 2047 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2048 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2049 2050 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2051 } 2052 2053 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2054 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2055 &flow_act, dest, 1); 2056 if (IS_ERR(flow_rule)) { 2057 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 2058 goto out; 2059 } 2060 2061 out: 2062 kvfree(spec); 2063 return flow_rule; 2064 } 2065 2066 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2067 { 2068 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2069 struct mlx5_core_dev *dev = esw->dev; 2070 struct mlx5_vport *vport; 2071 unsigned long i; 2072 2073 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2074 return -EOPNOTSUPP; 2075 2076 if (!mlx5_esw_is_fdb_created(esw)) 2077 return -EOPNOTSUPP; 2078 2079 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2080 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2081 mlx5_mode = MLX5_INLINE_MODE_NONE; 2082 goto out; 2083 case MLX5_CAP_INLINE_MODE_L2: 2084 mlx5_mode = MLX5_INLINE_MODE_L2; 2085 goto out; 2086 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2087 goto query_vports; 2088 } 2089 2090 query_vports: 2091 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2092 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2093 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2094 if (prev_mlx5_mode != mlx5_mode) 2095 return -EINVAL; 2096 prev_mlx5_mode = mlx5_mode; 2097 } 2098 2099 out: 2100 *mode = mlx5_mode; 2101 return 0; 2102 } 2103 2104 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2105 { 2106 struct mlx5_esw_offload *offloads = &esw->offloads; 2107 2108 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2109 return; 2110 2111 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2112 mlx5_destroy_flow_group(offloads->restore_group); 2113 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2114 } 2115 2116 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2117 { 2118 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2119 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2120 struct mlx5_flow_table_attr ft_attr = {}; 2121 struct mlx5_core_dev *dev = esw->dev; 2122 struct mlx5_flow_namespace *ns; 2123 struct mlx5_modify_hdr *mod_hdr; 2124 void *match_criteria, *misc; 2125 struct mlx5_flow_table *ft; 2126 struct mlx5_flow_group *g; 2127 u32 *flow_group_in; 2128 int err = 0; 2129 2130 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2131 return 0; 2132 2133 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2134 if (!ns) { 2135 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2136 return -EOPNOTSUPP; 2137 } 2138 2139 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2140 if (!flow_group_in) { 2141 err = -ENOMEM; 2142 goto out_free; 2143 } 2144 2145 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2146 ft = mlx5_create_flow_table(ns, &ft_attr); 2147 if (IS_ERR(ft)) { 2148 err = PTR_ERR(ft); 2149 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2150 err); 2151 goto out_free; 2152 } 2153 2154 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2155 match_criteria); 2156 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2157 misc_parameters_2); 2158 2159 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2160 ESW_REG_C0_USER_DATA_METADATA_MASK); 2161 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2162 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2163 ft_attr.max_fte - 1); 2164 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2165 MLX5_MATCH_MISC_PARAMETERS_2); 2166 g = mlx5_create_flow_group(ft, flow_group_in); 2167 if (IS_ERR(g)) { 2168 err = PTR_ERR(g); 2169 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2170 err); 2171 goto err_group; 2172 } 2173 2174 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2175 MLX5_SET(copy_action_in, modact, src_field, 2176 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2177 MLX5_SET(copy_action_in, modact, dst_field, 2178 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2179 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2180 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2181 modact); 2182 if (IS_ERR(mod_hdr)) { 2183 err = PTR_ERR(mod_hdr); 2184 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2185 err); 2186 goto err_mod_hdr; 2187 } 2188 2189 esw->offloads.ft_offloads_restore = ft; 2190 esw->offloads.restore_group = g; 2191 esw->offloads.restore_copy_hdr_id = mod_hdr; 2192 2193 kvfree(flow_group_in); 2194 2195 return 0; 2196 2197 err_mod_hdr: 2198 mlx5_destroy_flow_group(g); 2199 err_group: 2200 mlx5_destroy_flow_table(ft); 2201 out_free: 2202 kvfree(flow_group_in); 2203 2204 return err; 2205 } 2206 2207 static int esw_offloads_start(struct mlx5_eswitch *esw, 2208 struct netlink_ext_ack *extack) 2209 { 2210 int err, err1; 2211 2212 esw->mode = MLX5_ESWITCH_OFFLOADS; 2213 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2214 if (err) { 2215 NL_SET_ERR_MSG_MOD(extack, 2216 "Failed setting eswitch to offloads"); 2217 esw->mode = MLX5_ESWITCH_LEGACY; 2218 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 2219 if (err1) { 2220 NL_SET_ERR_MSG_MOD(extack, 2221 "Failed setting eswitch back to legacy"); 2222 } 2223 mlx5_rescan_drivers(esw->dev); 2224 } 2225 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2226 if (mlx5_eswitch_inline_mode_get(esw, 2227 &esw->offloads.inline_mode)) { 2228 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2229 NL_SET_ERR_MSG_MOD(extack, 2230 "Inline mode is different between vports"); 2231 } 2232 } 2233 return err; 2234 } 2235 2236 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw, 2237 struct mlx5_eswitch_rep *rep, 2238 xa_mark_t mark) 2239 { 2240 bool mark_set; 2241 2242 /* Copy the mark from vport to its rep */ 2243 mark_set = xa_get_mark(&esw->vports, rep->vport, mark); 2244 if (mark_set) 2245 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark); 2246 } 2247 2248 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) 2249 { 2250 struct mlx5_eswitch_rep *rep; 2251 int rep_type; 2252 int err; 2253 2254 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 2255 if (!rep) 2256 return -ENOMEM; 2257 2258 rep->vport = vport->vport; 2259 rep->vport_index = vport->index; 2260 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2261 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2262 2263 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2264 if (err) 2265 goto insert_err; 2266 2267 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN); 2268 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF); 2269 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF); 2270 return 0; 2271 2272 insert_err: 2273 kfree(rep); 2274 return err; 2275 } 2276 2277 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2278 struct mlx5_eswitch_rep *rep) 2279 { 2280 xa_erase(&esw->offloads.vport_reps, rep->vport); 2281 kfree(rep); 2282 } 2283 2284 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2285 { 2286 struct mlx5_eswitch_rep *rep; 2287 unsigned long i; 2288 2289 mlx5_esw_for_each_rep(esw, i, rep) 2290 mlx5_esw_offloads_rep_cleanup(esw, rep); 2291 xa_destroy(&esw->offloads.vport_reps); 2292 } 2293 2294 int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2295 { 2296 struct mlx5_vport *vport; 2297 unsigned long i; 2298 int err; 2299 2300 xa_init(&esw->offloads.vport_reps); 2301 2302 mlx5_esw_for_each_vport(esw, i, vport) { 2303 err = mlx5_esw_offloads_rep_init(esw, vport); 2304 if (err) 2305 goto err; 2306 } 2307 return 0; 2308 2309 err: 2310 esw_offloads_cleanup_reps(esw); 2311 return err; 2312 } 2313 2314 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2315 struct mlx5_eswitch_rep *rep, u8 rep_type) 2316 { 2317 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2318 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2319 esw->offloads.rep_ops[rep_type]->unload(rep); 2320 } 2321 2322 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type) 2323 { 2324 struct mlx5_eswitch_rep *rep; 2325 unsigned long i; 2326 2327 mlx5_esw_for_each_sf_rep(esw, i, rep) 2328 __esw_offloads_unload_rep(esw, rep, rep_type); 2329 } 2330 2331 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2332 { 2333 struct mlx5_eswitch_rep *rep; 2334 unsigned long i; 2335 2336 __unload_reps_sf_vport(esw, rep_type); 2337 2338 mlx5_esw_for_each_vf_rep(esw, i, rep) 2339 __esw_offloads_unload_rep(esw, rep, rep_type); 2340 2341 if (mlx5_ecpf_vport_exists(esw->dev)) { 2342 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF); 2343 __esw_offloads_unload_rep(esw, rep, rep_type); 2344 } 2345 2346 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 2347 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF); 2348 __esw_offloads_unload_rep(esw, rep, rep_type); 2349 } 2350 2351 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 2352 __esw_offloads_unload_rep(esw, rep, rep_type); 2353 } 2354 2355 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2356 { 2357 struct mlx5_eswitch_rep *rep; 2358 int rep_type; 2359 int err; 2360 2361 rep = mlx5_eswitch_get_rep(esw, vport_num); 2362 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2363 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2364 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 2365 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2366 if (err) 2367 goto err_reps; 2368 } 2369 2370 return 0; 2371 2372 err_reps: 2373 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2374 for (--rep_type; rep_type >= 0; rep_type--) 2375 __esw_offloads_unload_rep(esw, rep, rep_type); 2376 return err; 2377 } 2378 2379 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2380 { 2381 struct mlx5_eswitch_rep *rep; 2382 int rep_type; 2383 2384 rep = mlx5_eswitch_get_rep(esw, vport_num); 2385 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2386 __esw_offloads_unload_rep(esw, rep, rep_type); 2387 } 2388 2389 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) 2390 { 2391 int err; 2392 2393 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2394 return 0; 2395 2396 if (vport_num != MLX5_VPORT_UPLINK) { 2397 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num); 2398 if (err) 2399 return err; 2400 } 2401 2402 err = mlx5_esw_offloads_rep_load(esw, vport_num); 2403 if (err) 2404 goto load_err; 2405 return err; 2406 2407 load_err: 2408 if (vport_num != MLX5_VPORT_UPLINK) 2409 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2410 return err; 2411 } 2412 2413 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) 2414 { 2415 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2416 return; 2417 2418 mlx5_esw_offloads_rep_unload(esw, vport_num); 2419 2420 if (vport_num != MLX5_VPORT_UPLINK) 2421 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2422 } 2423 2424 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2425 struct mlx5_core_dev *slave) 2426 { 2427 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2428 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2429 struct mlx5_flow_root_namespace *root; 2430 struct mlx5_flow_namespace *ns; 2431 int err; 2432 2433 MLX5_SET(set_flow_table_root_in, in, opcode, 2434 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2435 MLX5_SET(set_flow_table_root_in, in, table_type, 2436 FS_FT_FDB); 2437 2438 if (master) { 2439 ns = mlx5_get_flow_namespace(master, 2440 MLX5_FLOW_NAMESPACE_FDB); 2441 root = find_root(&ns->node); 2442 mutex_lock(&root->chain_lock); 2443 MLX5_SET(set_flow_table_root_in, in, 2444 table_eswitch_owner_vhca_id_valid, 1); 2445 MLX5_SET(set_flow_table_root_in, in, 2446 table_eswitch_owner_vhca_id, 2447 MLX5_CAP_GEN(master, vhca_id)); 2448 MLX5_SET(set_flow_table_root_in, in, table_id, 2449 root->root_ft->id); 2450 } else { 2451 ns = mlx5_get_flow_namespace(slave, 2452 MLX5_FLOW_NAMESPACE_FDB); 2453 root = find_root(&ns->node); 2454 mutex_lock(&root->chain_lock); 2455 MLX5_SET(set_flow_table_root_in, in, table_id, 2456 root->root_ft->id); 2457 } 2458 2459 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2460 mutex_unlock(&root->chain_lock); 2461 2462 return err; 2463 } 2464 2465 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2466 struct mlx5_core_dev *slave, 2467 struct mlx5_vport *vport, 2468 struct mlx5_flow_table *acl) 2469 { 2470 struct mlx5_flow_handle *flow_rule = NULL; 2471 struct mlx5_flow_destination dest = {}; 2472 struct mlx5_flow_act flow_act = {}; 2473 struct mlx5_flow_spec *spec; 2474 int err = 0; 2475 void *misc; 2476 2477 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2478 if (!spec) 2479 return -ENOMEM; 2480 2481 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2482 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2483 misc_parameters); 2484 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2485 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 2486 MLX5_CAP_GEN(slave, vhca_id)); 2487 2488 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2489 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2490 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2491 source_eswitch_owner_vhca_id); 2492 2493 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2494 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2495 dest.vport.num = slave->priv.eswitch->manager_vport; 2496 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2497 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2498 2499 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2500 &dest, 1); 2501 if (IS_ERR(flow_rule)) 2502 err = PTR_ERR(flow_rule); 2503 else 2504 vport->egress.offloads.bounce_rule = flow_rule; 2505 2506 kvfree(spec); 2507 return err; 2508 } 2509 2510 static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 2511 struct mlx5_core_dev *slave) 2512 { 2513 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2514 struct mlx5_eswitch *esw = master->priv.eswitch; 2515 struct mlx5_flow_table_attr ft_attr = { 2516 .max_fte = 1, .prio = 0, .level = 0, 2517 .flags = MLX5_FLOW_TABLE_OTHER_VPORT, 2518 }; 2519 struct mlx5_flow_namespace *egress_ns; 2520 struct mlx5_flow_table *acl; 2521 struct mlx5_flow_group *g; 2522 struct mlx5_vport *vport; 2523 void *match_criteria; 2524 u32 *flow_group_in; 2525 int err; 2526 2527 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 2528 if (IS_ERR(vport)) 2529 return PTR_ERR(vport); 2530 2531 egress_ns = mlx5_get_flow_vport_acl_namespace(master, 2532 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 2533 vport->index); 2534 if (!egress_ns) 2535 return -EINVAL; 2536 2537 if (vport->egress.acl) 2538 return -EINVAL; 2539 2540 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2541 if (!flow_group_in) 2542 return -ENOMEM; 2543 2544 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2545 if (IS_ERR(acl)) { 2546 err = PTR_ERR(acl); 2547 goto out; 2548 } 2549 2550 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2551 match_criteria); 2552 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2553 misc_parameters.source_port); 2554 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2555 misc_parameters.source_eswitch_owner_vhca_id); 2556 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2557 MLX5_MATCH_MISC_PARAMETERS); 2558 2559 MLX5_SET(create_flow_group_in, flow_group_in, 2560 source_eswitch_owner_vhca_id_valid, 1); 2561 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2562 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0); 2563 2564 g = mlx5_create_flow_group(acl, flow_group_in); 2565 if (IS_ERR(g)) { 2566 err = PTR_ERR(g); 2567 goto err_group; 2568 } 2569 2570 err = __esw_set_master_egress_rule(master, slave, vport, acl); 2571 if (err) 2572 goto err_rule; 2573 2574 vport->egress.acl = acl; 2575 vport->egress.offloads.bounce_grp = g; 2576 2577 kvfree(flow_group_in); 2578 2579 return 0; 2580 2581 err_rule: 2582 mlx5_destroy_flow_group(g); 2583 err_group: 2584 mlx5_destroy_flow_table(acl); 2585 out: 2586 kvfree(flow_group_in); 2587 return err; 2588 } 2589 2590 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev) 2591 { 2592 struct mlx5_vport *vport; 2593 2594 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 2595 dev->priv.eswitch->manager_vport); 2596 2597 esw_acl_egress_ofld_cleanup(vport); 2598 } 2599 2600 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw, 2601 struct mlx5_eswitch *slave_esw) 2602 { 2603 int err; 2604 2605 err = esw_set_slave_root_fdb(master_esw->dev, 2606 slave_esw->dev); 2607 if (err) 2608 return err; 2609 2610 err = esw_set_master_egress_rule(master_esw->dev, 2611 slave_esw->dev); 2612 if (err) 2613 goto err_acl; 2614 2615 return err; 2616 2617 err_acl: 2618 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2619 2620 return err; 2621 } 2622 2623 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw, 2624 struct mlx5_eswitch *slave_esw) 2625 { 2626 esw_unset_master_egress_rule(master_esw->dev); 2627 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2628 } 2629 2630 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 2631 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2632 2633 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw) 2634 { 2635 const struct mlx5_eswitch_rep_ops *ops; 2636 struct mlx5_eswitch_rep *rep; 2637 unsigned long i; 2638 u8 rep_type; 2639 2640 mlx5_esw_for_each_rep(esw, i, rep) { 2641 rep_type = NUM_REP_TYPES; 2642 while (rep_type--) { 2643 ops = esw->offloads.rep_ops[rep_type]; 2644 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2645 ops->event) 2646 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL); 2647 } 2648 } 2649 } 2650 2651 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) 2652 { 2653 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2654 mlx5e_tc_clean_fdb_peer_flows(esw); 2655 #endif 2656 mlx5_esw_offloads_rep_event_unpair(esw); 2657 esw_del_fdb_peer_miss_rules(esw); 2658 } 2659 2660 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2661 struct mlx5_eswitch *peer_esw) 2662 { 2663 const struct mlx5_eswitch_rep_ops *ops; 2664 struct mlx5_eswitch_rep *rep; 2665 unsigned long i; 2666 u8 rep_type; 2667 int err; 2668 2669 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2670 if (err) 2671 return err; 2672 2673 mlx5_esw_for_each_rep(esw, i, rep) { 2674 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2675 ops = esw->offloads.rep_ops[rep_type]; 2676 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2677 ops->event) { 2678 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 2679 if (err) 2680 goto err_out; 2681 } 2682 } 2683 } 2684 2685 return 0; 2686 2687 err_out: 2688 mlx5_esw_offloads_unpair(esw); 2689 return err; 2690 } 2691 2692 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2693 struct mlx5_eswitch *peer_esw, 2694 bool pair) 2695 { 2696 struct mlx5_flow_root_namespace *peer_ns; 2697 struct mlx5_flow_root_namespace *ns; 2698 int err; 2699 2700 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2701 ns = esw->dev->priv.steering->fdb_root_ns; 2702 2703 if (pair) { 2704 err = mlx5_flow_namespace_set_peer(ns, peer_ns); 2705 if (err) 2706 return err; 2707 2708 err = mlx5_flow_namespace_set_peer(peer_ns, ns); 2709 if (err) { 2710 mlx5_flow_namespace_set_peer(ns, NULL); 2711 return err; 2712 } 2713 } else { 2714 mlx5_flow_namespace_set_peer(ns, NULL); 2715 mlx5_flow_namespace_set_peer(peer_ns, NULL); 2716 } 2717 2718 return 0; 2719 } 2720 2721 static int mlx5_esw_offloads_devcom_event(int event, 2722 void *my_data, 2723 void *event_data) 2724 { 2725 struct mlx5_eswitch *esw = my_data; 2726 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2727 struct mlx5_eswitch *peer_esw = event_data; 2728 int err; 2729 2730 switch (event) { 2731 case ESW_OFFLOADS_DEVCOM_PAIR: 2732 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2733 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2734 break; 2735 2736 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2737 if (err) 2738 goto err_out; 2739 err = mlx5_esw_offloads_pair(esw, peer_esw); 2740 if (err) 2741 goto err_peer; 2742 2743 err = mlx5_esw_offloads_pair(peer_esw, esw); 2744 if (err) 2745 goto err_pair; 2746 2747 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); 2748 break; 2749 2750 case ESW_OFFLOADS_DEVCOM_UNPAIR: 2751 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) 2752 break; 2753 2754 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); 2755 mlx5_esw_offloads_unpair(peer_esw); 2756 mlx5_esw_offloads_unpair(esw); 2757 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2758 break; 2759 } 2760 2761 return 0; 2762 2763 err_pair: 2764 mlx5_esw_offloads_unpair(esw); 2765 err_peer: 2766 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2767 err_out: 2768 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 2769 event, err); 2770 return err; 2771 } 2772 2773 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw) 2774 { 2775 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2776 2777 INIT_LIST_HEAD(&esw->offloads.peer_flows); 2778 mutex_init(&esw->offloads.peer_mutex); 2779 2780 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2781 return; 2782 2783 if (!mlx5_is_lag_supported(esw->dev)) 2784 return; 2785 2786 mlx5_devcom_register_component(devcom, 2787 MLX5_DEVCOM_ESW_OFFLOADS, 2788 mlx5_esw_offloads_devcom_event, 2789 esw); 2790 2791 mlx5_devcom_send_event(devcom, 2792 MLX5_DEVCOM_ESW_OFFLOADS, 2793 ESW_OFFLOADS_DEVCOM_PAIR, esw); 2794 } 2795 2796 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 2797 { 2798 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2799 2800 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2801 return; 2802 2803 if (!mlx5_is_lag_supported(esw->dev)) 2804 return; 2805 2806 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, 2807 ESW_OFFLOADS_DEVCOM_UNPAIR, esw); 2808 2809 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); 2810 } 2811 2812 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 2813 { 2814 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 2815 return false; 2816 2817 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 2818 MLX5_FDB_TO_VPORT_REG_C_0)) 2819 return false; 2820 2821 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 2822 return false; 2823 2824 return true; 2825 } 2826 2827 #define MLX5_ESW_METADATA_RSVD_UPLINK 1 2828 2829 /* Share the same metadata for uplink's. This is fine because: 2830 * (a) In shared FDB mode (LAG) both uplink's are treated the 2831 * same and tagged with the same metadata. 2832 * (b) In non shared FDB mode, packets from physical port0 2833 * cannot hit eswitch of PF1 and vice versa. 2834 */ 2835 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 2836 { 2837 return MLX5_ESW_METADATA_RSVD_UPLINK; 2838 } 2839 2840 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 2841 { 2842 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 2843 /* Reserve 0xf for internal port offload */ 2844 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 2845 u32 pf_num; 2846 int id; 2847 2848 /* Only 4 bits of pf_num */ 2849 pf_num = mlx5_get_dev_index(esw->dev); 2850 if (pf_num > max_pf_num) 2851 return 0; 2852 2853 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 2854 /* Use only non-zero vport_id (2-4095) for all PF's */ 2855 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 2856 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 2857 vport_end_ida, GFP_KERNEL); 2858 if (id < 0) 2859 return 0; 2860 id = (pf_num << ESW_VPORT_BITS) | id; 2861 return id; 2862 } 2863 2864 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 2865 { 2866 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 2867 2868 /* Metadata contains only 12 bits of actual ida id */ 2869 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 2870 } 2871 2872 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 2873 struct mlx5_vport *vport) 2874 { 2875 if (vport->vport == MLX5_VPORT_UPLINK) 2876 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 2877 else 2878 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 2879 2880 vport->metadata = vport->default_metadata; 2881 return vport->metadata ? 0 : -ENOSPC; 2882 } 2883 2884 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 2885 struct mlx5_vport *vport) 2886 { 2887 if (!vport->default_metadata) 2888 return; 2889 2890 if (vport->vport == MLX5_VPORT_UPLINK) 2891 return; 2892 2893 WARN_ON(vport->metadata != vport->default_metadata); 2894 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 2895 } 2896 2897 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 2898 { 2899 struct mlx5_vport *vport; 2900 unsigned long i; 2901 2902 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2903 return; 2904 2905 mlx5_esw_for_each_vport(esw, i, vport) 2906 esw_offloads_vport_metadata_cleanup(esw, vport); 2907 } 2908 2909 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 2910 { 2911 struct mlx5_vport *vport; 2912 unsigned long i; 2913 int err; 2914 2915 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2916 return 0; 2917 2918 mlx5_esw_for_each_vport(esw, i, vport) { 2919 err = esw_offloads_vport_metadata_setup(esw, vport); 2920 if (err) 2921 goto metadata_err; 2922 } 2923 2924 return 0; 2925 2926 metadata_err: 2927 esw_offloads_metadata_uninit(esw); 2928 return err; 2929 } 2930 2931 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable) 2932 { 2933 int err = 0; 2934 2935 down_write(&esw->mode_lock); 2936 if (mlx5_esw_is_fdb_created(esw)) { 2937 err = -EBUSY; 2938 goto done; 2939 } 2940 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2941 err = -EOPNOTSUPP; 2942 goto done; 2943 } 2944 if (enable) 2945 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2946 else 2947 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2948 done: 2949 up_write(&esw->mode_lock); 2950 return err; 2951 } 2952 2953 int 2954 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 2955 struct mlx5_vport *vport) 2956 { 2957 int err; 2958 2959 err = esw_acl_ingress_ofld_setup(esw, vport); 2960 if (err) 2961 return err; 2962 2963 err = esw_acl_egress_ofld_setup(esw, vport); 2964 if (err) 2965 goto egress_err; 2966 2967 return 0; 2968 2969 egress_err: 2970 esw_acl_ingress_ofld_cleanup(esw, vport); 2971 return err; 2972 } 2973 2974 void 2975 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 2976 struct mlx5_vport *vport) 2977 { 2978 esw_acl_egress_ofld_cleanup(vport); 2979 esw_acl_ingress_ofld_cleanup(esw, vport); 2980 } 2981 2982 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2983 { 2984 struct mlx5_vport *vport; 2985 2986 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2987 if (IS_ERR(vport)) 2988 return PTR_ERR(vport); 2989 2990 return esw_vport_create_offloads_acl_tables(esw, vport); 2991 } 2992 2993 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2994 { 2995 struct mlx5_vport *vport; 2996 2997 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2998 if (IS_ERR(vport)) 2999 return; 3000 3001 esw_vport_destroy_offloads_acl_tables(esw, vport); 3002 } 3003 3004 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) 3005 { 3006 struct mlx5_eswitch_rep *rep; 3007 unsigned long i; 3008 int ret; 3009 3010 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3011 return 0; 3012 3013 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3014 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3015 return 0; 3016 3017 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3018 if (ret) 3019 return ret; 3020 3021 mlx5_esw_for_each_rep(esw, i, rep) { 3022 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3023 mlx5_esw_offloads_rep_load(esw, rep->vport); 3024 } 3025 3026 return 0; 3027 } 3028 3029 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3030 { 3031 struct mlx5_esw_indir_table *indir; 3032 int err; 3033 3034 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3035 mutex_init(&esw->fdb_table.offloads.vports.lock); 3036 hash_init(esw->fdb_table.offloads.vports.table); 3037 atomic64_set(&esw->user_count, 0); 3038 3039 indir = mlx5_esw_indir_table_init(); 3040 if (IS_ERR(indir)) { 3041 err = PTR_ERR(indir); 3042 goto create_indir_err; 3043 } 3044 esw->fdb_table.offloads.indir = indir; 3045 3046 err = esw_create_uplink_offloads_acl_tables(esw); 3047 if (err) 3048 goto create_acl_err; 3049 3050 err = esw_create_offloads_table(esw); 3051 if (err) 3052 goto create_offloads_err; 3053 3054 err = esw_create_restore_table(esw); 3055 if (err) 3056 goto create_restore_err; 3057 3058 err = esw_create_offloads_fdb_tables(esw); 3059 if (err) 3060 goto create_fdb_err; 3061 3062 err = esw_create_vport_rx_group(esw); 3063 if (err) 3064 goto create_fg_err; 3065 3066 return 0; 3067 3068 create_fg_err: 3069 esw_destroy_offloads_fdb_tables(esw); 3070 create_fdb_err: 3071 esw_destroy_restore_table(esw); 3072 create_restore_err: 3073 esw_destroy_offloads_table(esw); 3074 create_offloads_err: 3075 esw_destroy_uplink_offloads_acl_tables(esw); 3076 create_acl_err: 3077 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3078 create_indir_err: 3079 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3080 return err; 3081 } 3082 3083 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3084 { 3085 esw_destroy_vport_rx_group(esw); 3086 esw_destroy_offloads_fdb_tables(esw); 3087 esw_destroy_restore_table(esw); 3088 esw_destroy_offloads_table(esw); 3089 esw_destroy_uplink_offloads_acl_tables(esw); 3090 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3091 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3092 } 3093 3094 static void 3095 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 3096 { 3097 struct devlink *devlink; 3098 bool host_pf_disabled; 3099 u16 new_num_vfs; 3100 3101 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3102 host_params_context.host_num_of_vfs); 3103 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3104 host_params_context.host_pf_disabled); 3105 3106 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3107 return; 3108 3109 devlink = priv_to_devlink(esw->dev); 3110 devl_lock(devlink); 3111 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3112 if (esw->esw_funcs.num_vfs > 0) { 3113 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3114 } else { 3115 int err; 3116 3117 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3118 MLX5_VPORT_UC_ADDR_CHANGE); 3119 if (err) { 3120 devl_unlock(devlink); 3121 return; 3122 } 3123 } 3124 esw->esw_funcs.num_vfs = new_num_vfs; 3125 devl_unlock(devlink); 3126 } 3127 3128 static void esw_functions_changed_event_handler(struct work_struct *work) 3129 { 3130 struct mlx5_host_work *host_work; 3131 struct mlx5_eswitch *esw; 3132 const u32 *out; 3133 3134 host_work = container_of(work, struct mlx5_host_work, work); 3135 esw = host_work->esw; 3136 3137 out = mlx5_esw_query_functions(esw->dev); 3138 if (IS_ERR(out)) 3139 goto out; 3140 3141 esw_vfs_changed_event_handler(esw, out); 3142 kvfree(out); 3143 out: 3144 kfree(host_work); 3145 } 3146 3147 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3148 { 3149 struct mlx5_esw_functions *esw_funcs; 3150 struct mlx5_host_work *host_work; 3151 struct mlx5_eswitch *esw; 3152 3153 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 3154 if (!host_work) 3155 return NOTIFY_DONE; 3156 3157 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3158 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3159 3160 host_work->esw = esw; 3161 3162 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3163 queue_work(esw->work_queue, &host_work->work); 3164 3165 return NOTIFY_OK; 3166 } 3167 3168 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3169 { 3170 const u32 *query_host_out; 3171 3172 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3173 return 0; 3174 3175 query_host_out = mlx5_esw_query_functions(esw->dev); 3176 if (IS_ERR(query_host_out)) 3177 return PTR_ERR(query_host_out); 3178 3179 /* Mark non local controller with non zero controller number. */ 3180 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3181 host_params_context.host_number); 3182 kvfree(query_host_out); 3183 return 0; 3184 } 3185 3186 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3187 { 3188 /* Local controller is always valid */ 3189 if (controller == 0) 3190 return true; 3191 3192 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3193 return false; 3194 3195 /* External host number starts with zero in device */ 3196 return (controller == esw->offloads.host_number + 1); 3197 } 3198 3199 int esw_offloads_enable(struct mlx5_eswitch *esw) 3200 { 3201 struct mapping_ctx *reg_c0_obj_pool; 3202 struct mlx5_vport *vport; 3203 unsigned long i; 3204 u64 mapping_id; 3205 int err; 3206 3207 mutex_init(&esw->offloads.termtbl_mutex); 3208 mlx5_rdma_enable_roce(esw->dev); 3209 3210 err = mlx5_esw_host_number_init(esw); 3211 if (err) 3212 goto err_metadata; 3213 3214 err = esw_offloads_metadata_init(esw); 3215 if (err) 3216 goto err_metadata; 3217 3218 err = esw_set_passing_vport_metadata(esw, true); 3219 if (err) 3220 goto err_vport_metadata; 3221 3222 mapping_id = mlx5_query_nic_system_image_guid(esw->dev); 3223 3224 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN, 3225 sizeof(struct mlx5_mapped_obj), 3226 ESW_REG_C0_USER_DATA_METADATA_MASK, 3227 true); 3228 3229 if (IS_ERR(reg_c0_obj_pool)) { 3230 err = PTR_ERR(reg_c0_obj_pool); 3231 goto err_pool; 3232 } 3233 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3234 3235 err = esw_offloads_steering_init(esw); 3236 if (err) 3237 goto err_steering_init; 3238 3239 /* Representor will control the vport link state */ 3240 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3241 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3242 3243 /* Uplink vport rep must load first. */ 3244 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); 3245 if (err) 3246 goto err_uplink; 3247 3248 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3249 if (err) 3250 goto err_vports; 3251 3252 esw_offloads_devcom_init(esw); 3253 3254 return 0; 3255 3256 err_vports: 3257 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 3258 err_uplink: 3259 esw_offloads_steering_cleanup(esw); 3260 err_steering_init: 3261 mapping_destroy(reg_c0_obj_pool); 3262 err_pool: 3263 esw_set_passing_vport_metadata(esw, false); 3264 err_vport_metadata: 3265 esw_offloads_metadata_uninit(esw); 3266 err_metadata: 3267 mlx5_rdma_disable_roce(esw->dev); 3268 mutex_destroy(&esw->offloads.termtbl_mutex); 3269 return err; 3270 } 3271 3272 static int esw_offloads_stop(struct mlx5_eswitch *esw, 3273 struct netlink_ext_ack *extack) 3274 { 3275 int err, err1; 3276 3277 esw->mode = MLX5_ESWITCH_LEGACY; 3278 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3279 if (err) { 3280 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3281 esw->mode = MLX5_ESWITCH_OFFLOADS; 3282 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3283 if (err1) { 3284 NL_SET_ERR_MSG_MOD(extack, 3285 "Failed setting eswitch back to offloads"); 3286 } 3287 } 3288 3289 return err; 3290 } 3291 3292 void esw_offloads_disable(struct mlx5_eswitch *esw) 3293 { 3294 esw_offloads_devcom_cleanup(esw); 3295 mlx5_eswitch_disable_pf_vf_vports(esw); 3296 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 3297 esw_set_passing_vport_metadata(esw, false); 3298 esw_offloads_steering_cleanup(esw); 3299 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3300 esw_offloads_metadata_uninit(esw); 3301 mlx5_rdma_disable_roce(esw->dev); 3302 mutex_destroy(&esw->offloads.termtbl_mutex); 3303 } 3304 3305 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3306 { 3307 switch (mode) { 3308 case DEVLINK_ESWITCH_MODE_LEGACY: 3309 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3310 break; 3311 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3312 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3313 break; 3314 default: 3315 return -EINVAL; 3316 } 3317 3318 return 0; 3319 } 3320 3321 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 3322 { 3323 switch (mlx5_mode) { 3324 case MLX5_ESWITCH_LEGACY: 3325 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3326 break; 3327 case MLX5_ESWITCH_OFFLOADS: 3328 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3329 break; 3330 default: 3331 return -EINVAL; 3332 } 3333 3334 return 0; 3335 } 3336 3337 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3338 { 3339 switch (mode) { 3340 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3341 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3342 break; 3343 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3344 *mlx5_mode = MLX5_INLINE_MODE_L2; 3345 break; 3346 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3347 *mlx5_mode = MLX5_INLINE_MODE_IP; 3348 break; 3349 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3350 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3351 break; 3352 default: 3353 return -EINVAL; 3354 } 3355 3356 return 0; 3357 } 3358 3359 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3360 { 3361 switch (mlx5_mode) { 3362 case MLX5_INLINE_MODE_NONE: 3363 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3364 break; 3365 case MLX5_INLINE_MODE_L2: 3366 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3367 break; 3368 case MLX5_INLINE_MODE_IP: 3369 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3370 break; 3371 case MLX5_INLINE_MODE_TCP_UDP: 3372 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3373 break; 3374 default: 3375 return -EINVAL; 3376 } 3377 3378 return 0; 3379 } 3380 3381 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 3382 struct netlink_ext_ack *extack) 3383 { 3384 u16 cur_mlx5_mode, mlx5_mode = 0; 3385 struct mlx5_eswitch *esw; 3386 int err = 0; 3387 3388 esw = mlx5_devlink_eswitch_get(devlink); 3389 if (IS_ERR(esw)) 3390 return PTR_ERR(esw); 3391 3392 if (esw_mode_from_devlink(mode, &mlx5_mode)) 3393 return -EINVAL; 3394 3395 mlx5_lag_disable_change(esw->dev); 3396 err = mlx5_esw_try_lock(esw); 3397 if (err < 0) { 3398 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 3399 goto enable_lag; 3400 } 3401 cur_mlx5_mode = err; 3402 err = 0; 3403 3404 if (cur_mlx5_mode == mlx5_mode) 3405 goto unlock; 3406 3407 mlx5_eswitch_disable_locked(esw); 3408 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { 3409 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 3410 NL_SET_ERR_MSG_MOD(extack, 3411 "Can't change mode while devlink traps are active"); 3412 err = -EOPNOTSUPP; 3413 goto unlock; 3414 } 3415 err = esw_offloads_start(esw, extack); 3416 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { 3417 err = esw_offloads_stop(esw, extack); 3418 mlx5_rescan_drivers(esw->dev); 3419 } else { 3420 err = -EINVAL; 3421 } 3422 3423 unlock: 3424 mlx5_esw_unlock(esw); 3425 enable_lag: 3426 mlx5_lag_enable_change(esw->dev); 3427 return err; 3428 } 3429 3430 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 3431 { 3432 struct mlx5_eswitch *esw; 3433 int err; 3434 3435 esw = mlx5_devlink_eswitch_get(devlink); 3436 if (IS_ERR(esw)) 3437 return PTR_ERR(esw); 3438 3439 down_write(&esw->mode_lock); 3440 err = esw_mode_to_devlink(esw->mode, mode); 3441 up_write(&esw->mode_lock); 3442 return err; 3443 } 3444 3445 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 3446 struct netlink_ext_ack *extack) 3447 { 3448 struct mlx5_core_dev *dev = esw->dev; 3449 struct mlx5_vport *vport; 3450 u16 err_vport_num = 0; 3451 unsigned long i; 3452 int err = 0; 3453 3454 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3455 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3456 if (err) { 3457 err_vport_num = vport->vport; 3458 NL_SET_ERR_MSG_MOD(extack, 3459 "Failed to set min inline on vport"); 3460 goto revert_inline_mode; 3461 } 3462 } 3463 return 0; 3464 3465 revert_inline_mode: 3466 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3467 if (vport->vport == err_vport_num) 3468 break; 3469 mlx5_modify_nic_vport_min_inline(dev, 3470 vport->vport, 3471 esw->offloads.inline_mode); 3472 } 3473 return err; 3474 } 3475 3476 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 3477 struct netlink_ext_ack *extack) 3478 { 3479 struct mlx5_core_dev *dev = devlink_priv(devlink); 3480 struct mlx5_eswitch *esw; 3481 u8 mlx5_mode; 3482 int err; 3483 3484 esw = mlx5_devlink_eswitch_get(devlink); 3485 if (IS_ERR(esw)) 3486 return PTR_ERR(esw); 3487 3488 down_write(&esw->mode_lock); 3489 3490 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 3491 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 3492 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 3493 err = 0; 3494 goto out; 3495 } 3496 3497 fallthrough; 3498 case MLX5_CAP_INLINE_MODE_L2: 3499 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 3500 err = -EOPNOTSUPP; 3501 goto out; 3502 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 3503 break; 3504 } 3505 3506 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3507 NL_SET_ERR_MSG_MOD(extack, 3508 "Can't set inline mode when flows are configured"); 3509 err = -EOPNOTSUPP; 3510 goto out; 3511 } 3512 3513 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 3514 if (err) 3515 goto out; 3516 3517 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 3518 if (err) 3519 goto out; 3520 3521 esw->offloads.inline_mode = mlx5_mode; 3522 up_write(&esw->mode_lock); 3523 return 0; 3524 3525 out: 3526 up_write(&esw->mode_lock); 3527 return err; 3528 } 3529 3530 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3531 { 3532 struct mlx5_eswitch *esw; 3533 int err; 3534 3535 esw = mlx5_devlink_eswitch_get(devlink); 3536 if (IS_ERR(esw)) 3537 return PTR_ERR(esw); 3538 3539 down_write(&esw->mode_lock); 3540 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3541 up_write(&esw->mode_lock); 3542 return err; 3543 } 3544 3545 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3546 enum devlink_eswitch_encap_mode encap, 3547 struct netlink_ext_ack *extack) 3548 { 3549 struct mlx5_core_dev *dev = devlink_priv(devlink); 3550 struct mlx5_eswitch *esw; 3551 int err = 0; 3552 3553 esw = mlx5_devlink_eswitch_get(devlink); 3554 if (IS_ERR(esw)) 3555 return PTR_ERR(esw); 3556 3557 down_write(&esw->mode_lock); 3558 3559 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3560 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3561 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3562 err = -EOPNOTSUPP; 3563 goto unlock; 3564 } 3565 3566 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3567 err = -EOPNOTSUPP; 3568 goto unlock; 3569 } 3570 3571 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3572 esw->offloads.encap = encap; 3573 goto unlock; 3574 } 3575 3576 if (esw->offloads.encap == encap) 3577 goto unlock; 3578 3579 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3580 NL_SET_ERR_MSG_MOD(extack, 3581 "Can't set encapsulation when flows are configured"); 3582 err = -EOPNOTSUPP; 3583 goto unlock; 3584 } 3585 3586 esw_destroy_offloads_fdb_tables(esw); 3587 3588 esw->offloads.encap = encap; 3589 3590 err = esw_create_offloads_fdb_tables(esw); 3591 3592 if (err) { 3593 NL_SET_ERR_MSG_MOD(extack, 3594 "Failed re-creating fast FDB table"); 3595 esw->offloads.encap = !encap; 3596 (void)esw_create_offloads_fdb_tables(esw); 3597 } 3598 3599 unlock: 3600 up_write(&esw->mode_lock); 3601 return err; 3602 } 3603 3604 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 3605 enum devlink_eswitch_encap_mode *encap) 3606 { 3607 struct mlx5_eswitch *esw; 3608 3609 esw = mlx5_devlink_eswitch_get(devlink); 3610 if (IS_ERR(esw)) 3611 return PTR_ERR(esw); 3612 3613 down_write(&esw->mode_lock); 3614 *encap = esw->offloads.encap; 3615 up_write(&esw->mode_lock); 3616 return 0; 3617 } 3618 3619 static bool 3620 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 3621 { 3622 /* Currently, only ECPF based device has representor for host PF. */ 3623 if (vport_num == MLX5_VPORT_PF && 3624 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 3625 return false; 3626 3627 if (vport_num == MLX5_VPORT_ECPF && 3628 !mlx5_ecpf_vport_exists(esw->dev)) 3629 return false; 3630 3631 return true; 3632 } 3633 3634 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 3635 const struct mlx5_eswitch_rep_ops *ops, 3636 u8 rep_type) 3637 { 3638 struct mlx5_eswitch_rep_data *rep_data; 3639 struct mlx5_eswitch_rep *rep; 3640 unsigned long i; 3641 3642 esw->offloads.rep_ops[rep_type] = ops; 3643 mlx5_esw_for_each_rep(esw, i, rep) { 3644 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 3645 rep->esw = esw; 3646 rep_data = &rep->rep_data[rep_type]; 3647 atomic_set(&rep_data->state, REP_REGISTERED); 3648 } 3649 } 3650 } 3651 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 3652 3653 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 3654 { 3655 struct mlx5_eswitch_rep *rep; 3656 unsigned long i; 3657 3658 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 3659 __unload_reps_all_vport(esw, rep_type); 3660 3661 mlx5_esw_for_each_rep(esw, i, rep) 3662 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 3663 } 3664 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 3665 3666 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 3667 { 3668 struct mlx5_eswitch_rep *rep; 3669 3670 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3671 return rep->rep_data[rep_type].priv; 3672 } 3673 3674 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 3675 u16 vport, 3676 u8 rep_type) 3677 { 3678 struct mlx5_eswitch_rep *rep; 3679 3680 rep = mlx5_eswitch_get_rep(esw, vport); 3681 3682 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3683 esw->offloads.rep_ops[rep_type]->get_proto_dev) 3684 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 3685 return NULL; 3686 } 3687 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 3688 3689 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 3690 { 3691 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 3692 } 3693 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 3694 3695 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 3696 u16 vport) 3697 { 3698 return mlx5_eswitch_get_rep(esw, vport); 3699 } 3700 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 3701 3702 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 3703 { 3704 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 3705 } 3706 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 3707 3708 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 3709 { 3710 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 3711 } 3712 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 3713 3714 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 3715 u16 vport_num) 3716 { 3717 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3718 3719 if (WARN_ON_ONCE(IS_ERR(vport))) 3720 return 0; 3721 3722 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 3723 } 3724 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 3725 3726 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port, 3727 u16 vport_num, u32 controller, u32 sfnum) 3728 { 3729 int err; 3730 3731 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE); 3732 if (err) 3733 return err; 3734 3735 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum); 3736 if (err) 3737 goto devlink_err; 3738 3739 mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum); 3740 err = mlx5_esw_offloads_rep_load(esw, vport_num); 3741 if (err) 3742 goto rep_err; 3743 return 0; 3744 3745 rep_err: 3746 mlx5_esw_vport_debugfs_destroy(esw, vport_num); 3747 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3748 devlink_err: 3749 mlx5_esw_vport_disable(esw, vport_num); 3750 return err; 3751 } 3752 3753 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num) 3754 { 3755 mlx5_esw_offloads_rep_unload(esw, vport_num); 3756 mlx5_esw_vport_debugfs_destroy(esw, vport_num); 3757 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3758 mlx5_esw_vport_disable(esw, vport_num); 3759 } 3760 3761 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 3762 { 3763 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 3764 void *query_ctx; 3765 void *hca_caps; 3766 int err; 3767 3768 *vhca_id = 0; 3769 if (mlx5_esw_is_manager_vport(esw, vport_num) || 3770 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) 3771 return -EPERM; 3772 3773 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 3774 if (!query_ctx) 3775 return -ENOMEM; 3776 3777 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx); 3778 if (err) 3779 goto out_free; 3780 3781 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 3782 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 3783 3784 out_free: 3785 kfree(query_ctx); 3786 return err; 3787 } 3788 3789 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 3790 { 3791 u16 *old_entry, *vhca_map_entry, vhca_id; 3792 int err; 3793 3794 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3795 if (err) { 3796 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 3797 vport_num, err); 3798 return err; 3799 } 3800 3801 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 3802 if (!vhca_map_entry) 3803 return -ENOMEM; 3804 3805 *vhca_map_entry = vport_num; 3806 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 3807 if (xa_is_err(old_entry)) { 3808 kfree(vhca_map_entry); 3809 return xa_err(old_entry); 3810 } 3811 kfree(old_entry); 3812 return 0; 3813 } 3814 3815 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 3816 { 3817 u16 *vhca_map_entry, vhca_id; 3818 int err; 3819 3820 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3821 if (err) 3822 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 3823 vport_num, err); 3824 3825 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 3826 kfree(vhca_map_entry); 3827 } 3828 3829 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 3830 { 3831 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 3832 3833 if (!res) 3834 return -ENOENT; 3835 3836 *vport_num = *res; 3837 return 0; 3838 } 3839 3840 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 3841 u16 vport_num) 3842 { 3843 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3844 3845 if (WARN_ON_ONCE(IS_ERR(vport))) 3846 return 0; 3847 3848 return vport->metadata; 3849 } 3850 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 3851 3852 static bool 3853 is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num) 3854 { 3855 return vport_num == MLX5_VPORT_PF || 3856 mlx5_eswitch_is_vf_vport(esw, vport_num) || 3857 mlx5_esw_is_sf_vport(esw, vport_num); 3858 } 3859 3860 int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port, 3861 u8 *hw_addr, int *hw_addr_len, 3862 struct netlink_ext_ack *extack) 3863 { 3864 struct mlx5_eswitch *esw; 3865 struct mlx5_vport *vport; 3866 u16 vport_num; 3867 3868 esw = mlx5_devlink_eswitch_get(port->devlink); 3869 if (IS_ERR(esw)) 3870 return PTR_ERR(esw); 3871 3872 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 3873 if (!is_port_function_supported(esw, vport_num)) 3874 return -EOPNOTSUPP; 3875 3876 vport = mlx5_eswitch_get_vport(esw, vport_num); 3877 if (IS_ERR(vport)) { 3878 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 3879 return PTR_ERR(vport); 3880 } 3881 3882 mutex_lock(&esw->state_lock); 3883 ether_addr_copy(hw_addr, vport->info.mac); 3884 *hw_addr_len = ETH_ALEN; 3885 mutex_unlock(&esw->state_lock); 3886 return 0; 3887 } 3888 3889 int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, 3890 const u8 *hw_addr, int hw_addr_len, 3891 struct netlink_ext_ack *extack) 3892 { 3893 struct mlx5_eswitch *esw; 3894 u16 vport_num; 3895 3896 esw = mlx5_devlink_eswitch_get(port->devlink); 3897 if (IS_ERR(esw)) { 3898 NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr"); 3899 return PTR_ERR(esw); 3900 } 3901 3902 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 3903 if (!is_port_function_supported(esw, vport_num)) { 3904 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr"); 3905 return -EINVAL; 3906 } 3907 3908 return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr); 3909 } 3910