1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/indir_table.h" 42 #include "esw/acl/ofld.h" 43 #include "rdma.h" 44 #include "en.h" 45 #include "fs_core.h" 46 #include "lib/devcom.h" 47 #include "lib/eq.h" 48 #include "lib/fs_chains.h" 49 #include "en_tc.h" 50 #include "en/mapping.h" 51 #include "devlink.h" 52 #include "lag/lag.h" 53 #include "en/tc/post_meter.h" 54 55 #define mlx5_esw_for_each_rep(esw, i, rep) \ 56 xa_for_each(&((esw)->offloads.vport_reps), i, rep) 57 58 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \ 59 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF) 60 61 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \ 62 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \ 63 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF) 64 65 /* There are two match-all miss flows, one for unicast dst mac and 66 * one for multicast. 67 */ 68 #define MLX5_ESW_MISS_FLOWS (2) 69 #define UPLINK_REP_INDEX 0 70 71 #define MLX5_ESW_VPORT_TBL_SIZE 128 72 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 73 74 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 75 76 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 77 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 78 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 79 .flags = 0, 80 }; 81 82 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 83 u16 vport_num) 84 { 85 return xa_load(&esw->offloads.vport_reps, vport_num); 86 } 87 88 static void 89 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 90 struct mlx5_flow_spec *spec, 91 struct mlx5_esw_flow_attr *attr) 92 { 93 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 94 return; 95 96 if (attr->int_port) { 97 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 98 99 return; 100 } 101 102 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 103 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 104 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 105 } 106 107 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 108 * are not needed as well in the following process. So clear them all for simplicity. 109 */ 110 void 111 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 112 { 113 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 114 void *misc2; 115 116 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 117 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 118 119 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 120 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 121 122 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 123 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 124 } 125 } 126 127 static void 128 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 129 struct mlx5_flow_spec *spec, 130 struct mlx5_flow_attr *attr, 131 struct mlx5_eswitch *src_esw, 132 u16 vport) 133 { 134 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 135 u32 metadata; 136 void *misc2; 137 void *misc; 138 139 /* Use metadata matching because vport is not represented by single 140 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 141 */ 142 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 143 if (mlx5_esw_indir_table_decap_vport(attr)) 144 vport = mlx5_esw_indir_table_decap_vport(attr); 145 146 if (attr && !attr->chain && esw_attr->int_port) 147 metadata = 148 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 149 else 150 metadata = 151 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 152 153 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 154 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 155 156 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 157 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 158 mlx5_eswitch_get_vport_metadata_mask()); 159 160 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 161 } else { 162 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 163 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 164 165 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 166 MLX5_SET(fte_match_set_misc, misc, 167 source_eswitch_owner_vhca_id, 168 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 169 170 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 171 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 172 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 173 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 174 source_eswitch_owner_vhca_id); 175 176 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 177 } 178 } 179 180 static int 181 esw_setup_decap_indir(struct mlx5_eswitch *esw, 182 struct mlx5_flow_attr *attr, 183 struct mlx5_flow_spec *spec) 184 { 185 struct mlx5_flow_table *ft; 186 187 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 188 return -EOPNOTSUPP; 189 190 ft = mlx5_esw_indir_table_get(esw, attr, spec, 191 mlx5_esw_indir_table_decap_vport(attr), true); 192 return PTR_ERR_OR_ZERO(ft); 193 } 194 195 static void 196 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 197 struct mlx5_flow_attr *attr) 198 { 199 if (mlx5_esw_indir_table_decap_vport(attr)) 200 mlx5_esw_indir_table_put(esw, attr, 201 mlx5_esw_indir_table_decap_vport(attr), 202 true); 203 } 204 205 static int 206 esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 207 struct mlx5e_meter_attr *meter, 208 int i) 209 { 210 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 211 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 212 dest[i].range.min = 0; 213 dest[i].range.max = meter->params.mtu; 214 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 215 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 216 217 return 0; 218 } 219 220 static int 221 esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 222 struct mlx5_flow_act *flow_act, 223 u32 sampler_id, 224 int i) 225 { 226 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 227 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 228 dest[i].sampler_id = sampler_id; 229 230 return 0; 231 } 232 233 static int 234 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 235 struct mlx5_flow_act *flow_act, 236 struct mlx5_eswitch *esw, 237 struct mlx5_flow_attr *attr, 238 struct mlx5_flow_spec *spec, 239 int i) 240 { 241 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 242 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 243 dest[i].ft = attr->dest_ft; 244 245 if (mlx5_esw_indir_table_decap_vport(attr)) 246 return esw_setup_decap_indir(esw, attr, spec); 247 return 0; 248 } 249 250 static void 251 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 252 struct mlx5_fs_chains *chains, int i) 253 { 254 if (mlx5_chains_ignore_flow_level_supported(chains)) 255 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 256 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 257 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 258 } 259 260 static void 261 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 262 struct mlx5_eswitch *esw, int i) 263 { 264 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 265 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 266 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 267 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 268 } 269 270 static int 271 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 272 struct mlx5_flow_act *flow_act, 273 struct mlx5_fs_chains *chains, 274 u32 chain, u32 prio, u32 level, 275 int i) 276 { 277 struct mlx5_flow_table *ft; 278 279 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 280 ft = mlx5_chains_get_table(chains, chain, prio, level); 281 if (IS_ERR(ft)) 282 return PTR_ERR(ft); 283 284 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 285 dest[i].ft = ft; 286 return 0; 287 } 288 289 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 290 int from, int to) 291 { 292 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 293 struct mlx5_fs_chains *chains = esw_chains(esw); 294 int i; 295 296 for (i = from; i < to; i++) 297 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 298 mlx5_chains_put_table(chains, 0, 1, 0); 299 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 300 esw_attr->dests[i].mdev)) 301 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport, 302 false); 303 } 304 305 static bool 306 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 307 { 308 int i; 309 310 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 311 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 312 return true; 313 return false; 314 } 315 316 static int 317 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 318 struct mlx5_flow_act *flow_act, 319 struct mlx5_eswitch *esw, 320 struct mlx5_fs_chains *chains, 321 struct mlx5_flow_attr *attr, 322 int *i) 323 { 324 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 325 int err; 326 327 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 328 return -EOPNOTSUPP; 329 330 /* flow steering cannot handle more than one dest with the same ft 331 * in a single flow 332 */ 333 if (esw_attr->out_count - esw_attr->split_count > 1) 334 return -EOPNOTSUPP; 335 336 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 337 if (err) 338 return err; 339 340 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 341 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 342 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 343 } 344 (*i)++; 345 346 return 0; 347 } 348 349 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 350 struct mlx5_flow_attr *attr) 351 { 352 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 353 354 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 355 } 356 357 static bool 358 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 359 { 360 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 361 bool result = false; 362 int i; 363 364 /* Indirect table is supported only for flows with in_port uplink 365 * and the destination is vport on the same eswitch as the uplink, 366 * return false in case at least one of destinations doesn't meet 367 * this criteria. 368 */ 369 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 370 if (esw_attr->dests[i].rep && 371 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 372 esw_attr->dests[i].mdev)) { 373 result = true; 374 } else { 375 result = false; 376 break; 377 } 378 } 379 return result; 380 } 381 382 static int 383 esw_setup_indir_table(struct mlx5_flow_destination *dest, 384 struct mlx5_flow_act *flow_act, 385 struct mlx5_eswitch *esw, 386 struct mlx5_flow_attr *attr, 387 struct mlx5_flow_spec *spec, 388 bool ignore_flow_lvl, 389 int *i) 390 { 391 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 392 int j, err; 393 394 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 395 return -EOPNOTSUPP; 396 397 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 398 if (ignore_flow_lvl) 399 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 400 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 401 402 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec, 403 esw_attr->dests[j].rep->vport, false); 404 if (IS_ERR(dest[*i].ft)) { 405 err = PTR_ERR(dest[*i].ft); 406 goto err_indir_tbl_get; 407 } 408 } 409 410 if (mlx5_esw_indir_table_decap_vport(attr)) { 411 err = esw_setup_decap_indir(esw, attr, spec); 412 if (err) 413 goto err_indir_tbl_get; 414 } 415 416 return 0; 417 418 err_indir_tbl_get: 419 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 420 return err; 421 } 422 423 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 424 { 425 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 426 427 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 428 esw_cleanup_decap_indir(esw, attr); 429 } 430 431 static void 432 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 433 { 434 mlx5_chains_put_table(chains, chain, prio, level); 435 } 436 437 static void 438 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 439 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 440 int attr_idx, int dest_idx, bool pkt_reformat) 441 { 442 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 443 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport; 444 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 445 dest[dest_idx].vport.vhca_id = 446 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 447 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 448 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 449 mlx5_lag_mpesw_is_activated(esw->dev)) 450 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 451 } 452 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 453 if (pkt_reformat) { 454 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 455 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 456 } 457 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 458 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 459 } 460 } 461 462 static int 463 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 464 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 465 int i) 466 { 467 int j; 468 469 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 470 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 471 return i; 472 } 473 474 static bool 475 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 476 { 477 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 478 mlx5_eswitch_vport_match_metadata_enabled(esw) && 479 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 480 } 481 482 static int 483 esw_setup_dests(struct mlx5_flow_destination *dest, 484 struct mlx5_flow_act *flow_act, 485 struct mlx5_eswitch *esw, 486 struct mlx5_flow_attr *attr, 487 struct mlx5_flow_spec *spec, 488 int *i) 489 { 490 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 491 struct mlx5_fs_chains *chains = esw_chains(esw); 492 int err = 0; 493 494 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 495 esw_src_port_rewrite_supported(esw)) 496 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 497 498 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 499 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 500 (*i)++; 501 goto out; 502 } 503 504 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 505 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 506 (*i)++; 507 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 508 esw_setup_accept_dest(dest, flow_act, chains, *i); 509 (*i)++; 510 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 511 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 512 (*i)++; 513 } else if (esw_is_indir_table(esw, attr)) { 514 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i); 515 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 516 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 517 } else { 518 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 519 520 if (attr->dest_ft) { 521 err = esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i); 522 (*i)++; 523 } else if (attr->dest_chain) { 524 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 525 1, 0, *i); 526 (*i)++; 527 } 528 } 529 530 out: 531 return err; 532 } 533 534 static void 535 esw_cleanup_dests(struct mlx5_eswitch *esw, 536 struct mlx5_flow_attr *attr) 537 { 538 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 539 struct mlx5_fs_chains *chains = esw_chains(esw); 540 541 if (attr->dest_ft) { 542 esw_cleanup_decap_indir(esw, attr); 543 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 544 if (attr->dest_chain) 545 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 546 else if (esw_is_indir_table(esw, attr)) 547 esw_cleanup_indir_table(esw, attr); 548 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 549 esw_cleanup_chain_src_port_rewrite(esw, attr); 550 } 551 } 552 553 static void 554 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 555 { 556 struct mlx5e_flow_meter_handle *meter; 557 558 meter = attr->meter_attr.meter; 559 flow_act->exe_aso.type = attr->exe_aso_type; 560 flow_act->exe_aso.object_id = meter->obj_id; 561 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 562 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 563 /* use metadata reg 5 for packet color */ 564 flow_act->exe_aso.return_reg_id = 5; 565 } 566 567 struct mlx5_flow_handle * 568 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 569 struct mlx5_flow_spec *spec, 570 struct mlx5_flow_attr *attr) 571 { 572 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 573 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 574 struct mlx5_fs_chains *chains = esw_chains(esw); 575 bool split = !!(esw_attr->split_count); 576 struct mlx5_vport_tbl_attr fwd_attr; 577 struct mlx5_flow_destination *dest; 578 struct mlx5_flow_handle *rule; 579 struct mlx5_flow_table *fdb; 580 int i = 0; 581 582 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 583 return ERR_PTR(-EOPNOTSUPP); 584 585 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 586 if (!dest) 587 return ERR_PTR(-ENOMEM); 588 589 flow_act.action = attr->action; 590 /* if per flow vlan pop/push is emulated, don't set that into the firmware */ 591 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 592 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | 593 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 594 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 595 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 596 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 597 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 598 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 599 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 600 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 601 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 602 } 603 } 604 605 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 606 607 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 608 int err; 609 610 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 611 if (err) { 612 rule = ERR_PTR(err); 613 goto err_create_goto_table; 614 } 615 } 616 617 if (esw_attr->decap_pkt_reformat) 618 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 619 620 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 621 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 622 dest[i].counter_id = mlx5_fc_id(attr->counter); 623 i++; 624 } 625 626 if (attr->outer_match_level != MLX5_MATCH_NONE) 627 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 628 if (attr->inner_match_level != MLX5_MATCH_NONE) 629 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 630 631 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 632 flow_act.modify_hdr = attr->modify_hdr; 633 634 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 635 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 636 esw_setup_meter(attr, &flow_act); 637 638 if (split) { 639 fwd_attr.chain = attr->chain; 640 fwd_attr.prio = attr->prio; 641 fwd_attr.vport = esw_attr->in_rep->vport; 642 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 643 644 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 645 } else { 646 if (attr->chain || attr->prio) 647 fdb = mlx5_chains_get_table(chains, attr->chain, 648 attr->prio, 0); 649 else 650 fdb = attr->ft; 651 652 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 653 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 654 esw_attr->in_mdev->priv.eswitch, 655 esw_attr->in_rep->vport); 656 } 657 if (IS_ERR(fdb)) { 658 rule = ERR_CAST(fdb); 659 goto err_esw_get; 660 } 661 662 if (!i) { 663 kfree(dest); 664 dest = NULL; 665 } 666 667 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 668 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 669 &flow_act, dest, i); 670 else 671 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 672 if (IS_ERR(rule)) 673 goto err_add_rule; 674 else 675 atomic64_inc(&esw->offloads.num_flows); 676 677 kfree(dest); 678 return rule; 679 680 err_add_rule: 681 if (split) 682 mlx5_esw_vporttbl_put(esw, &fwd_attr); 683 else if (attr->chain || attr->prio) 684 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 685 err_esw_get: 686 esw_cleanup_dests(esw, attr); 687 err_create_goto_table: 688 kfree(dest); 689 return rule; 690 } 691 692 struct mlx5_flow_handle * 693 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 694 struct mlx5_flow_spec *spec, 695 struct mlx5_flow_attr *attr) 696 { 697 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 698 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 699 struct mlx5_fs_chains *chains = esw_chains(esw); 700 struct mlx5_vport_tbl_attr fwd_attr; 701 struct mlx5_flow_destination *dest; 702 struct mlx5_flow_table *fast_fdb; 703 struct mlx5_flow_table *fwd_fdb; 704 struct mlx5_flow_handle *rule; 705 int i, err = 0; 706 707 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 708 if (!dest) 709 return ERR_PTR(-ENOMEM); 710 711 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 712 if (IS_ERR(fast_fdb)) { 713 rule = ERR_CAST(fast_fdb); 714 goto err_get_fast; 715 } 716 717 fwd_attr.chain = attr->chain; 718 fwd_attr.prio = attr->prio; 719 fwd_attr.vport = esw_attr->in_rep->vport; 720 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 721 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 722 if (IS_ERR(fwd_fdb)) { 723 rule = ERR_CAST(fwd_fdb); 724 goto err_get_fwd; 725 } 726 727 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 728 for (i = 0; i < esw_attr->split_count; i++) { 729 if (esw_is_indir_table(esw, attr)) 730 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i); 731 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 732 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr, 733 &i); 734 else 735 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 736 737 if (err) { 738 rule = ERR_PTR(err); 739 goto err_chain_src_rewrite; 740 } 741 } 742 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 743 dest[i].ft = fwd_fdb; 744 i++; 745 746 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 747 esw_attr->in_mdev->priv.eswitch, 748 esw_attr->in_rep->vport); 749 750 if (attr->outer_match_level != MLX5_MATCH_NONE) 751 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 752 753 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 754 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 755 756 if (IS_ERR(rule)) { 757 i = esw_attr->split_count; 758 goto err_chain_src_rewrite; 759 } 760 761 atomic64_inc(&esw->offloads.num_flows); 762 763 kfree(dest); 764 return rule; 765 err_chain_src_rewrite: 766 esw_put_dest_tables_loop(esw, attr, 0, i); 767 mlx5_esw_vporttbl_put(esw, &fwd_attr); 768 err_get_fwd: 769 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 770 err_get_fast: 771 kfree(dest); 772 return rule; 773 } 774 775 static void 776 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 777 struct mlx5_flow_handle *rule, 778 struct mlx5_flow_attr *attr, 779 bool fwd_rule) 780 { 781 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 782 struct mlx5_fs_chains *chains = esw_chains(esw); 783 bool split = (esw_attr->split_count > 0); 784 struct mlx5_vport_tbl_attr fwd_attr; 785 int i; 786 787 mlx5_del_flow_rules(rule); 788 789 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 790 /* unref the term table */ 791 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 792 if (esw_attr->dests[i].termtbl) 793 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 794 } 795 } 796 797 atomic64_dec(&esw->offloads.num_flows); 798 799 if (fwd_rule || split) { 800 fwd_attr.chain = attr->chain; 801 fwd_attr.prio = attr->prio; 802 fwd_attr.vport = esw_attr->in_rep->vport; 803 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 804 } 805 806 if (fwd_rule) { 807 mlx5_esw_vporttbl_put(esw, &fwd_attr); 808 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 809 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count); 810 } else { 811 if (split) 812 mlx5_esw_vporttbl_put(esw, &fwd_attr); 813 else if (attr->chain || attr->prio) 814 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 815 esw_cleanup_dests(esw, attr); 816 } 817 } 818 819 void 820 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 821 struct mlx5_flow_handle *rule, 822 struct mlx5_flow_attr *attr) 823 { 824 __mlx5_eswitch_del_rule(esw, rule, attr, false); 825 } 826 827 void 828 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 829 struct mlx5_flow_handle *rule, 830 struct mlx5_flow_attr *attr) 831 { 832 __mlx5_eswitch_del_rule(esw, rule, attr, true); 833 } 834 835 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) 836 { 837 struct mlx5_eswitch_rep *rep; 838 unsigned long i; 839 int err = 0; 840 841 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); 842 mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) { 843 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 844 continue; 845 846 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); 847 if (err) 848 goto out; 849 } 850 851 out: 852 return err; 853 } 854 855 static struct mlx5_eswitch_rep * 856 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) 857 { 858 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; 859 860 in_rep = attr->in_rep; 861 out_rep = attr->dests[0].rep; 862 863 if (push) 864 vport = in_rep; 865 else if (pop) 866 vport = out_rep; 867 else 868 vport = in_rep; 869 870 return vport; 871 } 872 873 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, 874 bool push, bool pop, bool fwd) 875 { 876 struct mlx5_eswitch_rep *in_rep, *out_rep; 877 878 if ((push || pop) && !fwd) 879 goto out_notsupp; 880 881 in_rep = attr->in_rep; 882 out_rep = attr->dests[0].rep; 883 884 if (push && in_rep->vport == MLX5_VPORT_UPLINK) 885 goto out_notsupp; 886 887 if (pop && out_rep->vport == MLX5_VPORT_UPLINK) 888 goto out_notsupp; 889 890 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ 891 if (!push && !pop && fwd) 892 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK) 893 goto out_notsupp; 894 895 /* protects against (1) setting rules with different vlans to push and 896 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) 897 */ 898 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0])) 899 goto out_notsupp; 900 901 return 0; 902 903 out_notsupp: 904 return -EOPNOTSUPP; 905 } 906 907 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, 908 struct mlx5_flow_attr *attr) 909 { 910 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 911 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 912 struct mlx5_eswitch_rep *vport = NULL; 913 bool push, pop, fwd; 914 int err = 0; 915 916 /* nop if we're on the vlan push/pop non emulation mode */ 917 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 918 return 0; 919 920 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 921 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 922 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && 923 !attr->dest_chain); 924 925 mutex_lock(&esw->state_lock); 926 927 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd); 928 if (err) 929 goto unlock; 930 931 attr->flags &= ~MLX5_ATTR_FLAG_VLAN_HANDLED; 932 933 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 934 935 if (!push && !pop && fwd) { 936 /* tracks VF --> wire rules without vlan push action */ 937 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) { 938 vport->vlan_refcount++; 939 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED; 940 } 941 942 goto unlock; 943 } 944 945 if (!push && !pop) 946 goto unlock; 947 948 if (!(offloads->vlan_push_pop_refcount)) { 949 /* it's the 1st vlan rule, apply global vlan pop policy */ 950 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); 951 if (err) 952 goto out; 953 } 954 offloads->vlan_push_pop_refcount++; 955 956 if (push) { 957 if (vport->vlan_refcount) 958 goto skip_set_push; 959 960 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0], 961 0, SET_VLAN_INSERT | SET_VLAN_STRIP); 962 if (err) 963 goto out; 964 vport->vlan = esw_attr->vlan_vid[0]; 965 skip_set_push: 966 vport->vlan_refcount++; 967 } 968 out: 969 if (!err) 970 attr->flags |= MLX5_ATTR_FLAG_VLAN_HANDLED; 971 unlock: 972 mutex_unlock(&esw->state_lock); 973 return err; 974 } 975 976 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, 977 struct mlx5_flow_attr *attr) 978 { 979 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 980 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 981 struct mlx5_eswitch_rep *vport = NULL; 982 bool push, pop, fwd; 983 int err = 0; 984 985 /* nop if we're on the vlan push/pop non emulation mode */ 986 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 987 return 0; 988 989 if (!(attr->flags & MLX5_ATTR_FLAG_VLAN_HANDLED)) 990 return 0; 991 992 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 993 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 994 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 995 996 mutex_lock(&esw->state_lock); 997 998 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 999 1000 if (!push && !pop && fwd) { 1001 /* tracks VF --> wire rules without vlan push action */ 1002 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) 1003 vport->vlan_refcount--; 1004 1005 goto out; 1006 } 1007 1008 if (push) { 1009 vport->vlan_refcount--; 1010 if (vport->vlan_refcount) 1011 goto skip_unset_push; 1012 1013 vport->vlan = 0; 1014 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, 1015 0, 0, SET_VLAN_STRIP); 1016 if (err) 1017 goto out; 1018 } 1019 1020 skip_unset_push: 1021 offloads->vlan_push_pop_refcount--; 1022 if (offloads->vlan_push_pop_refcount) 1023 goto out; 1024 1025 /* no more vlan rules, stop global vlan pop policy */ 1026 err = esw_set_global_vlan_pop(esw, 0); 1027 1028 out: 1029 mutex_unlock(&esw->state_lock); 1030 return err; 1031 } 1032 1033 struct mlx5_flow_handle * 1034 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 1035 struct mlx5_eswitch *from_esw, 1036 struct mlx5_eswitch_rep *rep, 1037 u32 sqn) 1038 { 1039 struct mlx5_flow_act flow_act = {0}; 1040 struct mlx5_flow_destination dest = {}; 1041 struct mlx5_flow_handle *flow_rule; 1042 struct mlx5_flow_spec *spec; 1043 void *misc; 1044 1045 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1046 if (!spec) { 1047 flow_rule = ERR_PTR(-ENOMEM); 1048 goto out; 1049 } 1050 1051 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1052 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 1053 /* source vport is the esw manager */ 1054 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport); 1055 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 1056 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1057 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 1058 1059 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1060 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 1061 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1062 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 1063 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1064 source_eswitch_owner_vhca_id); 1065 1066 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1067 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1068 dest.vport.num = rep->vport; 1069 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 1070 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1071 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1072 1073 if (rep->vport == MLX5_VPORT_UPLINK) 1074 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1075 1076 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 1077 spec, &flow_act, &dest, 1); 1078 if (IS_ERR(flow_rule)) 1079 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", 1080 PTR_ERR(flow_rule)); 1081 out: 1082 kvfree(spec); 1083 return flow_rule; 1084 } 1085 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1086 1087 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1088 { 1089 mlx5_del_flow_rules(rule); 1090 } 1091 1092 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 1093 { 1094 if (rule) 1095 mlx5_del_flow_rules(rule); 1096 } 1097 1098 struct mlx5_flow_handle * 1099 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 1100 { 1101 struct mlx5_flow_destination dest = {}; 1102 struct mlx5_flow_act flow_act = {0}; 1103 struct mlx5_flow_handle *flow_rule; 1104 struct mlx5_flow_spec *spec; 1105 1106 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1107 if (!spec) 1108 return ERR_PTR(-ENOMEM); 1109 1110 MLX5_SET(fte_match_param, spec->match_criteria, 1111 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1112 MLX5_SET(fte_match_param, spec->match_criteria, 1113 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1114 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1115 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1116 1117 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1118 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1119 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1120 1121 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1122 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1123 dest.vport.num = vport_num; 1124 1125 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1126 spec, &flow_act, &dest, 1); 1127 if (IS_ERR(flow_rule)) 1128 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", 1129 vport_num, PTR_ERR(flow_rule)); 1130 1131 kvfree(spec); 1132 return flow_rule; 1133 } 1134 1135 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1136 { 1137 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1138 MLX5_FDB_TO_VPORT_REG_C_1; 1139 } 1140 1141 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1142 { 1143 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1144 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1145 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1146 u8 curr, wanted; 1147 int err; 1148 1149 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1150 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1151 return 0; 1152 1153 MLX5_SET(query_esw_vport_context_in, in, opcode, 1154 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1155 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1156 if (err) 1157 return err; 1158 1159 curr = MLX5_GET(query_esw_vport_context_out, out, 1160 esw_vport_context.fdb_to_vport_reg_c_id); 1161 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1162 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1163 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1164 1165 if (enable) 1166 curr |= wanted; 1167 else 1168 curr &= ~wanted; 1169 1170 MLX5_SET(modify_esw_vport_context_in, min, 1171 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1172 MLX5_SET(modify_esw_vport_context_in, min, 1173 field_select.fdb_to_vport_reg_c_id, 1); 1174 1175 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1176 if (!err) { 1177 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1178 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1179 else 1180 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1181 } 1182 1183 return err; 1184 } 1185 1186 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1187 struct mlx5_core_dev *peer_dev, 1188 struct mlx5_flow_spec *spec, 1189 struct mlx5_flow_destination *dest) 1190 { 1191 void *misc; 1192 1193 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1194 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1195 misc_parameters_2); 1196 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1197 mlx5_eswitch_get_vport_metadata_mask()); 1198 1199 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1200 } else { 1201 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1202 misc_parameters); 1203 1204 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1205 MLX5_CAP_GEN(peer_dev, vhca_id)); 1206 1207 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1208 1209 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1210 misc_parameters); 1211 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1212 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1213 source_eswitch_owner_vhca_id); 1214 } 1215 1216 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1217 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1218 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1219 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1220 } 1221 1222 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1223 struct mlx5_eswitch *peer_esw, 1224 struct mlx5_flow_spec *spec, 1225 u16 vport) 1226 { 1227 void *misc; 1228 1229 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1230 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1231 misc_parameters_2); 1232 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1233 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1234 vport)); 1235 } else { 1236 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1237 misc_parameters); 1238 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1239 } 1240 } 1241 1242 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1243 struct mlx5_core_dev *peer_dev) 1244 { 1245 struct mlx5_flow_destination dest = {}; 1246 struct mlx5_flow_act flow_act = {0}; 1247 struct mlx5_flow_handle **flows; 1248 /* total vports is the same for both e-switches */ 1249 int nvports = esw->total_vports; 1250 struct mlx5_flow_handle *flow; 1251 struct mlx5_flow_spec *spec; 1252 struct mlx5_vport *vport; 1253 unsigned long i; 1254 void *misc; 1255 int err; 1256 1257 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1258 if (!spec) 1259 return -ENOMEM; 1260 1261 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1262 1263 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); 1264 if (!flows) { 1265 err = -ENOMEM; 1266 goto alloc_flows_err; 1267 } 1268 1269 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1270 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1271 misc_parameters); 1272 1273 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1274 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1275 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1276 spec, MLX5_VPORT_PF); 1277 1278 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1279 spec, &flow_act, &dest, 1); 1280 if (IS_ERR(flow)) { 1281 err = PTR_ERR(flow); 1282 goto add_pf_flow_err; 1283 } 1284 flows[vport->index] = flow; 1285 } 1286 1287 if (mlx5_ecpf_vport_exists(esw->dev)) { 1288 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1289 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1290 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1291 spec, &flow_act, &dest, 1); 1292 if (IS_ERR(flow)) { 1293 err = PTR_ERR(flow); 1294 goto add_ecpf_flow_err; 1295 } 1296 flows[vport->index] = flow; 1297 } 1298 1299 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1300 esw_set_peer_miss_rule_source_port(esw, 1301 peer_dev->priv.eswitch, 1302 spec, vport->vport); 1303 1304 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1305 spec, &flow_act, &dest, 1); 1306 if (IS_ERR(flow)) { 1307 err = PTR_ERR(flow); 1308 goto add_vf_flow_err; 1309 } 1310 flows[vport->index] = flow; 1311 } 1312 1313 esw->fdb_table.offloads.peer_miss_rules = flows; 1314 1315 kvfree(spec); 1316 return 0; 1317 1318 add_vf_flow_err: 1319 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1320 if (!flows[vport->index]) 1321 continue; 1322 mlx5_del_flow_rules(flows[vport->index]); 1323 } 1324 if (mlx5_ecpf_vport_exists(esw->dev)) { 1325 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1326 mlx5_del_flow_rules(flows[vport->index]); 1327 } 1328 add_ecpf_flow_err: 1329 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1330 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1331 mlx5_del_flow_rules(flows[vport->index]); 1332 } 1333 add_pf_flow_err: 1334 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1335 kvfree(flows); 1336 alloc_flows_err: 1337 kvfree(spec); 1338 return err; 1339 } 1340 1341 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) 1342 { 1343 struct mlx5_flow_handle **flows; 1344 struct mlx5_vport *vport; 1345 unsigned long i; 1346 1347 flows = esw->fdb_table.offloads.peer_miss_rules; 1348 1349 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) 1350 mlx5_del_flow_rules(flows[vport->index]); 1351 1352 if (mlx5_ecpf_vport_exists(esw->dev)) { 1353 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1354 mlx5_del_flow_rules(flows[vport->index]); 1355 } 1356 1357 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1358 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1359 mlx5_del_flow_rules(flows[vport->index]); 1360 } 1361 kvfree(flows); 1362 } 1363 1364 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1365 { 1366 struct mlx5_flow_act flow_act = {0}; 1367 struct mlx5_flow_destination dest = {}; 1368 struct mlx5_flow_handle *flow_rule = NULL; 1369 struct mlx5_flow_spec *spec; 1370 void *headers_c; 1371 void *headers_v; 1372 int err = 0; 1373 u8 *dmac_c; 1374 u8 *dmac_v; 1375 1376 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1377 if (!spec) { 1378 err = -ENOMEM; 1379 goto out; 1380 } 1381 1382 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1383 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1384 outer_headers); 1385 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1386 outer_headers.dmac_47_16); 1387 dmac_c[0] = 0x01; 1388 1389 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1390 dest.vport.num = esw->manager_vport; 1391 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1392 1393 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1394 spec, &flow_act, &dest, 1); 1395 if (IS_ERR(flow_rule)) { 1396 err = PTR_ERR(flow_rule); 1397 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1398 goto out; 1399 } 1400 1401 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1402 1403 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1404 outer_headers); 1405 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1406 outer_headers.dmac_47_16); 1407 dmac_v[0] = 0x01; 1408 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1409 spec, &flow_act, &dest, 1); 1410 if (IS_ERR(flow_rule)) { 1411 err = PTR_ERR(flow_rule); 1412 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1413 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1414 goto out; 1415 } 1416 1417 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1418 1419 out: 1420 kvfree(spec); 1421 return err; 1422 } 1423 1424 struct mlx5_flow_handle * 1425 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1426 { 1427 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1428 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1429 struct mlx5_flow_context *flow_context; 1430 struct mlx5_flow_handle *flow_rule; 1431 struct mlx5_flow_destination dest; 1432 struct mlx5_flow_spec *spec; 1433 void *misc; 1434 1435 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1436 return ERR_PTR(-EOPNOTSUPP); 1437 1438 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1439 if (!spec) 1440 return ERR_PTR(-ENOMEM); 1441 1442 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1443 misc_parameters_2); 1444 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1445 ESW_REG_C0_USER_DATA_METADATA_MASK); 1446 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1447 misc_parameters_2); 1448 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1449 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1450 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1451 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1452 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1453 1454 flow_context = &spec->flow_context; 1455 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1456 flow_context->flow_tag = tag; 1457 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1458 dest.ft = esw->offloads.ft_offloads; 1459 1460 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1461 kvfree(spec); 1462 1463 if (IS_ERR(flow_rule)) 1464 esw_warn(esw->dev, 1465 "Failed to create restore rule for tag: %d, err(%d)\n", 1466 tag, (int)PTR_ERR(flow_rule)); 1467 1468 return flow_rule; 1469 } 1470 1471 #define MAX_PF_SQ 256 1472 #define MAX_SQ_NVPORTS 32 1473 1474 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1475 u32 *flow_group_in) 1476 { 1477 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1478 flow_group_in, 1479 match_criteria); 1480 1481 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1482 MLX5_SET(create_flow_group_in, flow_group_in, 1483 match_criteria_enable, 1484 MLX5_MATCH_MISC_PARAMETERS_2); 1485 1486 MLX5_SET(fte_match_param, match_criteria, 1487 misc_parameters_2.metadata_reg_c_0, 1488 mlx5_eswitch_get_vport_metadata_mask()); 1489 } else { 1490 MLX5_SET(create_flow_group_in, flow_group_in, 1491 match_criteria_enable, 1492 MLX5_MATCH_MISC_PARAMETERS); 1493 1494 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1495 misc_parameters.source_port); 1496 } 1497 } 1498 1499 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1500 static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1501 { 1502 struct mlx5_vport_tbl_attr attr; 1503 struct mlx5_vport *vport; 1504 unsigned long i; 1505 1506 attr.chain = 0; 1507 attr.prio = 1; 1508 mlx5_esw_for_each_vport(esw, i, vport) { 1509 attr.vport = vport->vport; 1510 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1511 mlx5_esw_vporttbl_put(esw, &attr); 1512 } 1513 } 1514 1515 static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1516 { 1517 struct mlx5_vport_tbl_attr attr; 1518 struct mlx5_flow_table *fdb; 1519 struct mlx5_vport *vport; 1520 unsigned long i; 1521 1522 attr.chain = 0; 1523 attr.prio = 1; 1524 mlx5_esw_for_each_vport(esw, i, vport) { 1525 attr.vport = vport->vport; 1526 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1527 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1528 if (IS_ERR(fdb)) 1529 goto out; 1530 } 1531 return 0; 1532 1533 out: 1534 esw_vport_tbl_put(esw); 1535 return PTR_ERR(fdb); 1536 } 1537 1538 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1539 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1540 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1541 { 1542 struct mlx5_core_dev *dev = esw->dev; 1543 1544 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1545 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1546 1547 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1548 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1549 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1550 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1551 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1552 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1553 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1554 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1555 /* Disabled when ttl workaround is needed, e.g 1556 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1557 */ 1558 esw_warn(dev, 1559 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1560 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1561 } else { 1562 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1563 esw_info(dev, "Supported tc chains and prios offload\n"); 1564 } 1565 1566 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1567 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1568 } 1569 1570 static int 1571 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1572 { 1573 struct mlx5_core_dev *dev = esw->dev; 1574 struct mlx5_flow_table *nf_ft, *ft; 1575 struct mlx5_chains_attr attr = {}; 1576 struct mlx5_fs_chains *chains; 1577 u32 fdb_max; 1578 int err; 1579 1580 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size); 1581 1582 esw_init_chains_offload_flags(esw, &attr.flags); 1583 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1584 attr.max_ft_sz = fdb_max; 1585 attr.max_grp_num = esw->params.large_group_num; 1586 attr.default_ft = miss_fdb; 1587 attr.mapping = esw->offloads.reg_c0_obj_pool; 1588 1589 chains = mlx5_chains_create(dev, &attr); 1590 if (IS_ERR(chains)) { 1591 err = PTR_ERR(chains); 1592 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1593 return err; 1594 } 1595 1596 esw->fdb_table.offloads.esw_chains_priv = chains; 1597 1598 /* Create tc_end_ft which is the always created ft chain */ 1599 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1600 1, 0); 1601 if (IS_ERR(nf_ft)) { 1602 err = PTR_ERR(nf_ft); 1603 goto nf_ft_err; 1604 } 1605 1606 /* Always open the root for fast path */ 1607 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1608 if (IS_ERR(ft)) { 1609 err = PTR_ERR(ft); 1610 goto level_0_err; 1611 } 1612 1613 /* Open level 1 for split fdb rules now if prios isn't supported */ 1614 if (!mlx5_chains_prios_supported(chains)) { 1615 err = esw_vport_tbl_get(esw); 1616 if (err) 1617 goto level_1_err; 1618 } 1619 1620 mlx5_chains_set_end_ft(chains, nf_ft); 1621 1622 return 0; 1623 1624 level_1_err: 1625 mlx5_chains_put_table(chains, 0, 1, 0); 1626 level_0_err: 1627 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1628 nf_ft_err: 1629 mlx5_chains_destroy(chains); 1630 esw->fdb_table.offloads.esw_chains_priv = NULL; 1631 1632 return err; 1633 } 1634 1635 static void 1636 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1637 { 1638 if (!mlx5_chains_prios_supported(chains)) 1639 esw_vport_tbl_put(esw); 1640 mlx5_chains_put_table(chains, 0, 1, 0); 1641 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1642 mlx5_chains_destroy(chains); 1643 } 1644 1645 #else /* CONFIG_MLX5_CLS_ACT */ 1646 1647 static int 1648 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1649 { return 0; } 1650 1651 static void 1652 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1653 {} 1654 1655 #endif 1656 1657 static int 1658 esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1659 struct mlx5_flow_table *fdb, 1660 u32 *flow_group_in, 1661 int *ix) 1662 { 1663 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1664 struct mlx5_flow_group *g; 1665 void *match_criteria; 1666 int count, err = 0; 1667 1668 memset(flow_group_in, 0, inlen); 1669 1670 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1671 MLX5_MATCH_MISC_PARAMETERS); 1672 1673 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1674 1675 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1676 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 1677 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1678 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1679 misc_parameters.source_eswitch_owner_vhca_id); 1680 MLX5_SET(create_flow_group_in, flow_group_in, 1681 source_eswitch_owner_vhca_id_valid, 1); 1682 } 1683 1684 /* See comment at table_size calculation */ 1685 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1686 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1687 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1688 *ix += count; 1689 1690 g = mlx5_create_flow_group(fdb, flow_group_in); 1691 if (IS_ERR(g)) { 1692 err = PTR_ERR(g); 1693 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1694 goto out; 1695 } 1696 esw->fdb_table.offloads.send_to_vport_grp = g; 1697 1698 out: 1699 return err; 1700 } 1701 1702 static int 1703 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1704 struct mlx5_flow_table *fdb, 1705 u32 *flow_group_in, 1706 int *ix) 1707 { 1708 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1709 struct mlx5_flow_group *g; 1710 void *match_criteria; 1711 int err = 0; 1712 1713 if (!esw_src_port_rewrite_supported(esw)) 1714 return 0; 1715 1716 memset(flow_group_in, 0, inlen); 1717 1718 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1719 MLX5_MATCH_MISC_PARAMETERS_2); 1720 1721 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1722 1723 MLX5_SET(fte_match_param, match_criteria, 1724 misc_parameters_2.metadata_reg_c_0, 1725 mlx5_eswitch_get_vport_metadata_mask()); 1726 MLX5_SET(fte_match_param, match_criteria, 1727 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1728 1729 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1730 MLX5_SET(create_flow_group_in, flow_group_in, 1731 end_flow_index, *ix + esw->total_vports - 1); 1732 *ix += esw->total_vports; 1733 1734 g = mlx5_create_flow_group(fdb, flow_group_in); 1735 if (IS_ERR(g)) { 1736 err = PTR_ERR(g); 1737 esw_warn(esw->dev, 1738 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1739 goto send_vport_meta_err; 1740 } 1741 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1742 1743 return 0; 1744 1745 send_vport_meta_err: 1746 return err; 1747 } 1748 1749 static int 1750 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1751 struct mlx5_flow_table *fdb, 1752 u32 *flow_group_in, 1753 int *ix) 1754 { 1755 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1756 struct mlx5_flow_group *g; 1757 void *match_criteria; 1758 int err = 0; 1759 1760 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1761 return 0; 1762 1763 memset(flow_group_in, 0, inlen); 1764 1765 esw_set_flow_group_source_port(esw, flow_group_in); 1766 1767 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1768 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1769 flow_group_in, 1770 match_criteria); 1771 1772 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1773 misc_parameters.source_eswitch_owner_vhca_id); 1774 1775 MLX5_SET(create_flow_group_in, flow_group_in, 1776 source_eswitch_owner_vhca_id_valid, 1); 1777 } 1778 1779 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1780 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1781 *ix + esw->total_vports - 1); 1782 *ix += esw->total_vports; 1783 1784 g = mlx5_create_flow_group(fdb, flow_group_in); 1785 if (IS_ERR(g)) { 1786 err = PTR_ERR(g); 1787 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1788 goto out; 1789 } 1790 esw->fdb_table.offloads.peer_miss_grp = g; 1791 1792 out: 1793 return err; 1794 } 1795 1796 static int 1797 esw_create_miss_group(struct mlx5_eswitch *esw, 1798 struct mlx5_flow_table *fdb, 1799 u32 *flow_group_in, 1800 int *ix) 1801 { 1802 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1803 struct mlx5_flow_group *g; 1804 void *match_criteria; 1805 int err = 0; 1806 u8 *dmac; 1807 1808 memset(flow_group_in, 0, inlen); 1809 1810 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1811 MLX5_MATCH_OUTER_HEADERS); 1812 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1813 match_criteria); 1814 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1815 outer_headers.dmac_47_16); 1816 dmac[0] = 0x01; 1817 1818 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1819 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1820 *ix + MLX5_ESW_MISS_FLOWS); 1821 1822 g = mlx5_create_flow_group(fdb, flow_group_in); 1823 if (IS_ERR(g)) { 1824 err = PTR_ERR(g); 1825 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1826 goto miss_err; 1827 } 1828 esw->fdb_table.offloads.miss_grp = g; 1829 1830 err = esw_add_fdb_miss_rule(esw); 1831 if (err) 1832 goto miss_rule_err; 1833 1834 return 0; 1835 1836 miss_rule_err: 1837 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1838 miss_err: 1839 return err; 1840 } 1841 1842 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1843 { 1844 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1845 struct mlx5_flow_table_attr ft_attr = {}; 1846 struct mlx5_core_dev *dev = esw->dev; 1847 struct mlx5_flow_namespace *root_ns; 1848 struct mlx5_flow_table *fdb = NULL; 1849 int table_size, ix = 0, err = 0; 1850 u32 flags = 0, *flow_group_in; 1851 1852 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1853 1854 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1855 if (!flow_group_in) 1856 return -ENOMEM; 1857 1858 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1859 if (!root_ns) { 1860 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1861 err = -EOPNOTSUPP; 1862 goto ns_err; 1863 } 1864 esw->fdb_table.offloads.ns = root_ns; 1865 err = mlx5_flow_namespace_set_mode(root_ns, 1866 esw->dev->priv.steering->mode); 1867 if (err) { 1868 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1869 goto ns_err; 1870 } 1871 1872 /* To be strictly correct: 1873 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1874 * should be: 1875 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1876 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1877 * but as the peer device might not be in switchdev mode it's not 1878 * possible. We use the fact that by default FW sets max vfs and max sfs 1879 * to the same value on both devices. If it needs to be changed in the future note 1880 * the peer miss group should also be created based on the number of 1881 * total vports of the peer (currently is also uses esw->total_vports). 1882 */ 1883 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1884 esw->total_vports * 2 + MLX5_ESW_MISS_FLOWS; 1885 1886 /* create the slow path fdb with encap set, so further table instances 1887 * can be created at run time while VFs are probed if the FW allows that. 1888 */ 1889 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1890 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1891 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1892 1893 ft_attr.flags = flags; 1894 ft_attr.max_fte = table_size; 1895 ft_attr.prio = FDB_SLOW_PATH; 1896 1897 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1898 if (IS_ERR(fdb)) { 1899 err = PTR_ERR(fdb); 1900 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1901 goto slow_fdb_err; 1902 } 1903 esw->fdb_table.offloads.slow_fdb = fdb; 1904 1905 /* Create empty TC-miss managed table. This allows plugging in following 1906 * priorities without directly exposing their level 0 table to 1907 * eswitch_offloads and passing it as miss_fdb to following call to 1908 * esw_chains_create(). 1909 */ 1910 memset(&ft_attr, 0, sizeof(ft_attr)); 1911 ft_attr.prio = FDB_TC_MISS; 1912 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1913 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1914 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1915 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1916 goto tc_miss_table_err; 1917 } 1918 1919 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1920 if (err) { 1921 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1922 goto fdb_chains_err; 1923 } 1924 1925 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1926 if (err) 1927 goto send_vport_err; 1928 1929 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1930 if (err) 1931 goto send_vport_meta_err; 1932 1933 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 1934 if (err) 1935 goto peer_miss_err; 1936 1937 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 1938 if (err) 1939 goto miss_err; 1940 1941 kvfree(flow_group_in); 1942 return 0; 1943 1944 miss_err: 1945 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1946 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1947 peer_miss_err: 1948 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1949 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1950 send_vport_meta_err: 1951 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1952 send_vport_err: 1953 esw_chains_destroy(esw, esw_chains(esw)); 1954 fdb_chains_err: 1955 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1956 tc_miss_table_err: 1957 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1958 slow_fdb_err: 1959 /* Holds true only as long as DMFS is the default */ 1960 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1961 ns_err: 1962 kvfree(flow_group_in); 1963 return err; 1964 } 1965 1966 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1967 { 1968 if (!mlx5_eswitch_get_slow_fdb(esw)) 1969 return; 1970 1971 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1972 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1973 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1974 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1975 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1976 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1977 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1978 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1979 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1980 1981 esw_chains_destroy(esw, esw_chains(esw)); 1982 1983 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1984 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1985 /* Holds true only as long as DMFS is the default */ 1986 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1987 MLX5_FLOW_STEERING_MODE_DMFS); 1988 atomic64_set(&esw->user_count, 0); 1989 } 1990 1991 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 1992 { 1993 int nvports; 1994 1995 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1996 if (mlx5e_tc_int_port_supported(esw)) 1997 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 1998 1999 return nvports; 2000 } 2001 2002 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 2003 { 2004 struct mlx5_flow_table_attr ft_attr = {}; 2005 struct mlx5_core_dev *dev = esw->dev; 2006 struct mlx5_flow_table *ft_offloads; 2007 struct mlx5_flow_namespace *ns; 2008 int err = 0; 2009 2010 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2011 if (!ns) { 2012 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2013 return -EOPNOTSUPP; 2014 } 2015 2016 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 2017 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 2018 ft_attr.prio = 1; 2019 2020 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 2021 if (IS_ERR(ft_offloads)) { 2022 err = PTR_ERR(ft_offloads); 2023 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 2024 return err; 2025 } 2026 2027 esw->offloads.ft_offloads = ft_offloads; 2028 return 0; 2029 } 2030 2031 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 2032 { 2033 struct mlx5_esw_offload *offloads = &esw->offloads; 2034 2035 mlx5_destroy_flow_table(offloads->ft_offloads); 2036 } 2037 2038 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 2039 { 2040 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2041 struct mlx5_flow_group *g; 2042 u32 *flow_group_in; 2043 int nvports; 2044 int err = 0; 2045 2046 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 2047 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2048 if (!flow_group_in) 2049 return -ENOMEM; 2050 2051 /* create vport rx group */ 2052 esw_set_flow_group_source_port(esw, flow_group_in); 2053 2054 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2055 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 2056 2057 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2058 2059 if (IS_ERR(g)) { 2060 err = PTR_ERR(g); 2061 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 2062 goto out; 2063 } 2064 2065 esw->offloads.vport_rx_group = g; 2066 out: 2067 kvfree(flow_group_in); 2068 return err; 2069 } 2070 2071 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2072 { 2073 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2074 } 2075 2076 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 2077 { 2078 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 2079 * for the drop rule, which is placed at the end of the table. 2080 * So return the total of vport and int_port as rule index. 2081 */ 2082 return esw_get_nr_ft_offloads_steering_src_ports(esw); 2083 } 2084 2085 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 2086 { 2087 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2088 struct mlx5_flow_group *g; 2089 u32 *flow_group_in; 2090 int flow_index; 2091 int err = 0; 2092 2093 flow_index = esw_create_vport_rx_drop_rule_index(esw); 2094 2095 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2096 if (!flow_group_in) 2097 return -ENOMEM; 2098 2099 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 2100 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 2101 2102 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2103 2104 if (IS_ERR(g)) { 2105 err = PTR_ERR(g); 2106 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); 2107 goto out; 2108 } 2109 2110 esw->offloads.vport_rx_drop_group = g; 2111 out: 2112 kvfree(flow_group_in); 2113 return err; 2114 } 2115 2116 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 2117 { 2118 if (esw->offloads.vport_rx_drop_group) 2119 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 2120 } 2121 2122 struct mlx5_flow_handle * 2123 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2124 struct mlx5_flow_destination *dest) 2125 { 2126 struct mlx5_flow_act flow_act = {0}; 2127 struct mlx5_flow_handle *flow_rule; 2128 struct mlx5_flow_spec *spec; 2129 void *misc; 2130 2131 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2132 if (!spec) { 2133 flow_rule = ERR_PTR(-ENOMEM); 2134 goto out; 2135 } 2136 2137 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2138 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2139 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2140 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2141 2142 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2143 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2144 mlx5_eswitch_get_vport_metadata_mask()); 2145 2146 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2147 } else { 2148 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2149 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2150 2151 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2152 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2153 2154 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2155 } 2156 2157 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2158 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2159 &flow_act, dest, 1); 2160 if (IS_ERR(flow_rule)) { 2161 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 2162 goto out; 2163 } 2164 2165 out: 2166 kvfree(spec); 2167 return flow_rule; 2168 } 2169 2170 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2171 { 2172 struct mlx5_flow_act flow_act = {}; 2173 struct mlx5_flow_handle *flow_rule; 2174 2175 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2176 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 2177 &flow_act, NULL, 0); 2178 if (IS_ERR(flow_rule)) { 2179 esw_warn(esw->dev, 2180 "fs offloads: Failed to add vport rx drop rule err %ld\n", 2181 PTR_ERR(flow_rule)); 2182 return PTR_ERR(flow_rule); 2183 } 2184 2185 esw->offloads.vport_rx_drop_rule = flow_rule; 2186 2187 return 0; 2188 } 2189 2190 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2191 { 2192 if (esw->offloads.vport_rx_drop_rule) 2193 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 2194 } 2195 2196 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2197 { 2198 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2199 struct mlx5_core_dev *dev = esw->dev; 2200 struct mlx5_vport *vport; 2201 unsigned long i; 2202 2203 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2204 return -EOPNOTSUPP; 2205 2206 if (!mlx5_esw_is_fdb_created(esw)) 2207 return -EOPNOTSUPP; 2208 2209 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2210 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2211 mlx5_mode = MLX5_INLINE_MODE_NONE; 2212 goto out; 2213 case MLX5_CAP_INLINE_MODE_L2: 2214 mlx5_mode = MLX5_INLINE_MODE_L2; 2215 goto out; 2216 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2217 goto query_vports; 2218 } 2219 2220 query_vports: 2221 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2222 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2223 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2224 if (prev_mlx5_mode != mlx5_mode) 2225 return -EINVAL; 2226 prev_mlx5_mode = mlx5_mode; 2227 } 2228 2229 out: 2230 *mode = mlx5_mode; 2231 return 0; 2232 } 2233 2234 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2235 { 2236 struct mlx5_esw_offload *offloads = &esw->offloads; 2237 2238 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2239 return; 2240 2241 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2242 mlx5_destroy_flow_group(offloads->restore_group); 2243 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2244 } 2245 2246 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2247 { 2248 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2249 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2250 struct mlx5_flow_table_attr ft_attr = {}; 2251 struct mlx5_core_dev *dev = esw->dev; 2252 struct mlx5_flow_namespace *ns; 2253 struct mlx5_modify_hdr *mod_hdr; 2254 void *match_criteria, *misc; 2255 struct mlx5_flow_table *ft; 2256 struct mlx5_flow_group *g; 2257 u32 *flow_group_in; 2258 int err = 0; 2259 2260 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2261 return 0; 2262 2263 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2264 if (!ns) { 2265 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2266 return -EOPNOTSUPP; 2267 } 2268 2269 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2270 if (!flow_group_in) { 2271 err = -ENOMEM; 2272 goto out_free; 2273 } 2274 2275 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2276 ft = mlx5_create_flow_table(ns, &ft_attr); 2277 if (IS_ERR(ft)) { 2278 err = PTR_ERR(ft); 2279 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2280 err); 2281 goto out_free; 2282 } 2283 2284 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2285 match_criteria); 2286 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2287 misc_parameters_2); 2288 2289 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2290 ESW_REG_C0_USER_DATA_METADATA_MASK); 2291 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2292 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2293 ft_attr.max_fte - 1); 2294 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2295 MLX5_MATCH_MISC_PARAMETERS_2); 2296 g = mlx5_create_flow_group(ft, flow_group_in); 2297 if (IS_ERR(g)) { 2298 err = PTR_ERR(g); 2299 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2300 err); 2301 goto err_group; 2302 } 2303 2304 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2305 MLX5_SET(copy_action_in, modact, src_field, 2306 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2307 MLX5_SET(copy_action_in, modact, dst_field, 2308 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2309 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2310 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2311 modact); 2312 if (IS_ERR(mod_hdr)) { 2313 err = PTR_ERR(mod_hdr); 2314 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2315 err); 2316 goto err_mod_hdr; 2317 } 2318 2319 esw->offloads.ft_offloads_restore = ft; 2320 esw->offloads.restore_group = g; 2321 esw->offloads.restore_copy_hdr_id = mod_hdr; 2322 2323 kvfree(flow_group_in); 2324 2325 return 0; 2326 2327 err_mod_hdr: 2328 mlx5_destroy_flow_group(g); 2329 err_group: 2330 mlx5_destroy_flow_table(ft); 2331 out_free: 2332 kvfree(flow_group_in); 2333 2334 return err; 2335 } 2336 2337 static int esw_offloads_start(struct mlx5_eswitch *esw, 2338 struct netlink_ext_ack *extack) 2339 { 2340 int err; 2341 2342 esw->mode = MLX5_ESWITCH_OFFLOADS; 2343 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2344 if (err) { 2345 NL_SET_ERR_MSG_MOD(extack, 2346 "Failed setting eswitch to offloads"); 2347 esw->mode = MLX5_ESWITCH_LEGACY; 2348 mlx5_rescan_drivers(esw->dev); 2349 } 2350 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2351 if (mlx5_eswitch_inline_mode_get(esw, 2352 &esw->offloads.inline_mode)) { 2353 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2354 NL_SET_ERR_MSG_MOD(extack, 2355 "Inline mode is different between vports"); 2356 } 2357 } 2358 return err; 2359 } 2360 2361 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw, 2362 struct mlx5_eswitch_rep *rep, 2363 xa_mark_t mark) 2364 { 2365 bool mark_set; 2366 2367 /* Copy the mark from vport to its rep */ 2368 mark_set = xa_get_mark(&esw->vports, rep->vport, mark); 2369 if (mark_set) 2370 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark); 2371 } 2372 2373 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) 2374 { 2375 struct mlx5_eswitch_rep *rep; 2376 int rep_type; 2377 int err; 2378 2379 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 2380 if (!rep) 2381 return -ENOMEM; 2382 2383 rep->vport = vport->vport; 2384 rep->vport_index = vport->index; 2385 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2386 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2387 2388 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2389 if (err) 2390 goto insert_err; 2391 2392 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN); 2393 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF); 2394 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF); 2395 return 0; 2396 2397 insert_err: 2398 kfree(rep); 2399 return err; 2400 } 2401 2402 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2403 struct mlx5_eswitch_rep *rep) 2404 { 2405 xa_erase(&esw->offloads.vport_reps, rep->vport); 2406 kfree(rep); 2407 } 2408 2409 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2410 { 2411 struct mlx5_eswitch_rep *rep; 2412 unsigned long i; 2413 2414 mlx5_esw_for_each_rep(esw, i, rep) 2415 mlx5_esw_offloads_rep_cleanup(esw, rep); 2416 xa_destroy(&esw->offloads.vport_reps); 2417 } 2418 2419 int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2420 { 2421 struct mlx5_vport *vport; 2422 unsigned long i; 2423 int err; 2424 2425 xa_init(&esw->offloads.vport_reps); 2426 2427 mlx5_esw_for_each_vport(esw, i, vport) { 2428 err = mlx5_esw_offloads_rep_init(esw, vport); 2429 if (err) 2430 goto err; 2431 } 2432 return 0; 2433 2434 err: 2435 esw_offloads_cleanup_reps(esw); 2436 return err; 2437 } 2438 2439 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2440 struct mlx5_eswitch_rep *rep, u8 rep_type) 2441 { 2442 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2443 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2444 esw->offloads.rep_ops[rep_type]->unload(rep); 2445 } 2446 2447 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type) 2448 { 2449 struct mlx5_eswitch_rep *rep; 2450 unsigned long i; 2451 2452 mlx5_esw_for_each_sf_rep(esw, i, rep) 2453 __esw_offloads_unload_rep(esw, rep, rep_type); 2454 } 2455 2456 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2457 { 2458 struct mlx5_eswitch_rep *rep; 2459 unsigned long i; 2460 2461 __unload_reps_sf_vport(esw, rep_type); 2462 2463 mlx5_esw_for_each_vf_rep(esw, i, rep) 2464 __esw_offloads_unload_rep(esw, rep, rep_type); 2465 2466 if (mlx5_ecpf_vport_exists(esw->dev)) { 2467 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF); 2468 __esw_offloads_unload_rep(esw, rep, rep_type); 2469 } 2470 2471 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 2472 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF); 2473 __esw_offloads_unload_rep(esw, rep, rep_type); 2474 } 2475 2476 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 2477 __esw_offloads_unload_rep(esw, rep, rep_type); 2478 } 2479 2480 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2481 { 2482 struct mlx5_eswitch_rep *rep; 2483 int rep_type; 2484 int err; 2485 2486 rep = mlx5_eswitch_get_rep(esw, vport_num); 2487 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2488 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2489 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 2490 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2491 if (err) 2492 goto err_reps; 2493 } 2494 2495 return 0; 2496 2497 err_reps: 2498 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2499 for (--rep_type; rep_type >= 0; rep_type--) 2500 __esw_offloads_unload_rep(esw, rep, rep_type); 2501 return err; 2502 } 2503 2504 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2505 { 2506 struct mlx5_eswitch_rep *rep; 2507 int rep_type; 2508 2509 rep = mlx5_eswitch_get_rep(esw, vport_num); 2510 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2511 __esw_offloads_unload_rep(esw, rep, rep_type); 2512 } 2513 2514 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) 2515 { 2516 int err; 2517 2518 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2519 return 0; 2520 2521 if (vport_num != MLX5_VPORT_UPLINK) { 2522 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num); 2523 if (err) 2524 return err; 2525 } 2526 2527 err = mlx5_esw_offloads_rep_load(esw, vport_num); 2528 if (err) 2529 goto load_err; 2530 return err; 2531 2532 load_err: 2533 if (vport_num != MLX5_VPORT_UPLINK) 2534 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2535 return err; 2536 } 2537 2538 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) 2539 { 2540 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2541 return; 2542 2543 mlx5_esw_offloads_rep_unload(esw, vport_num); 2544 2545 if (vport_num != MLX5_VPORT_UPLINK) 2546 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2547 } 2548 2549 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2550 struct mlx5_core_dev *slave) 2551 { 2552 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2553 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2554 struct mlx5_flow_root_namespace *root; 2555 struct mlx5_flow_namespace *ns; 2556 int err; 2557 2558 MLX5_SET(set_flow_table_root_in, in, opcode, 2559 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2560 MLX5_SET(set_flow_table_root_in, in, table_type, 2561 FS_FT_FDB); 2562 2563 if (master) { 2564 ns = mlx5_get_flow_namespace(master, 2565 MLX5_FLOW_NAMESPACE_FDB); 2566 root = find_root(&ns->node); 2567 mutex_lock(&root->chain_lock); 2568 MLX5_SET(set_flow_table_root_in, in, 2569 table_eswitch_owner_vhca_id_valid, 1); 2570 MLX5_SET(set_flow_table_root_in, in, 2571 table_eswitch_owner_vhca_id, 2572 MLX5_CAP_GEN(master, vhca_id)); 2573 MLX5_SET(set_flow_table_root_in, in, table_id, 2574 root->root_ft->id); 2575 } else { 2576 ns = mlx5_get_flow_namespace(slave, 2577 MLX5_FLOW_NAMESPACE_FDB); 2578 root = find_root(&ns->node); 2579 mutex_lock(&root->chain_lock); 2580 MLX5_SET(set_flow_table_root_in, in, table_id, 2581 root->root_ft->id); 2582 } 2583 2584 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2585 mutex_unlock(&root->chain_lock); 2586 2587 return err; 2588 } 2589 2590 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2591 struct mlx5_core_dev *slave, 2592 struct mlx5_vport *vport, 2593 struct mlx5_flow_table *acl) 2594 { 2595 struct mlx5_flow_handle *flow_rule = NULL; 2596 struct mlx5_flow_destination dest = {}; 2597 struct mlx5_flow_act flow_act = {}; 2598 struct mlx5_flow_spec *spec; 2599 int err = 0; 2600 void *misc; 2601 2602 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2603 if (!spec) 2604 return -ENOMEM; 2605 2606 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2607 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2608 misc_parameters); 2609 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2610 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 2611 MLX5_CAP_GEN(slave, vhca_id)); 2612 2613 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2614 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2615 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2616 source_eswitch_owner_vhca_id); 2617 2618 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2619 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2620 dest.vport.num = slave->priv.eswitch->manager_vport; 2621 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2622 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2623 2624 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2625 &dest, 1); 2626 if (IS_ERR(flow_rule)) 2627 err = PTR_ERR(flow_rule); 2628 else 2629 vport->egress.offloads.bounce_rule = flow_rule; 2630 2631 kvfree(spec); 2632 return err; 2633 } 2634 2635 static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 2636 struct mlx5_core_dev *slave) 2637 { 2638 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2639 struct mlx5_eswitch *esw = master->priv.eswitch; 2640 struct mlx5_flow_table_attr ft_attr = { 2641 .max_fte = 1, .prio = 0, .level = 0, 2642 .flags = MLX5_FLOW_TABLE_OTHER_VPORT, 2643 }; 2644 struct mlx5_flow_namespace *egress_ns; 2645 struct mlx5_flow_table *acl; 2646 struct mlx5_flow_group *g; 2647 struct mlx5_vport *vport; 2648 void *match_criteria; 2649 u32 *flow_group_in; 2650 int err; 2651 2652 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 2653 if (IS_ERR(vport)) 2654 return PTR_ERR(vport); 2655 2656 egress_ns = mlx5_get_flow_vport_acl_namespace(master, 2657 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 2658 vport->index); 2659 if (!egress_ns) 2660 return -EINVAL; 2661 2662 if (vport->egress.acl) 2663 return -EINVAL; 2664 2665 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2666 if (!flow_group_in) 2667 return -ENOMEM; 2668 2669 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2670 if (IS_ERR(acl)) { 2671 err = PTR_ERR(acl); 2672 goto out; 2673 } 2674 2675 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2676 match_criteria); 2677 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2678 misc_parameters.source_port); 2679 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2680 misc_parameters.source_eswitch_owner_vhca_id); 2681 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2682 MLX5_MATCH_MISC_PARAMETERS); 2683 2684 MLX5_SET(create_flow_group_in, flow_group_in, 2685 source_eswitch_owner_vhca_id_valid, 1); 2686 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2687 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0); 2688 2689 g = mlx5_create_flow_group(acl, flow_group_in); 2690 if (IS_ERR(g)) { 2691 err = PTR_ERR(g); 2692 goto err_group; 2693 } 2694 2695 err = __esw_set_master_egress_rule(master, slave, vport, acl); 2696 if (err) 2697 goto err_rule; 2698 2699 vport->egress.acl = acl; 2700 vport->egress.offloads.bounce_grp = g; 2701 2702 kvfree(flow_group_in); 2703 2704 return 0; 2705 2706 err_rule: 2707 mlx5_destroy_flow_group(g); 2708 err_group: 2709 mlx5_destroy_flow_table(acl); 2710 out: 2711 kvfree(flow_group_in); 2712 return err; 2713 } 2714 2715 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev) 2716 { 2717 struct mlx5_vport *vport; 2718 2719 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 2720 dev->priv.eswitch->manager_vport); 2721 2722 esw_acl_egress_ofld_cleanup(vport); 2723 } 2724 2725 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw, 2726 struct mlx5_eswitch *slave_esw) 2727 { 2728 int err; 2729 2730 err = esw_set_slave_root_fdb(master_esw->dev, 2731 slave_esw->dev); 2732 if (err) 2733 return err; 2734 2735 err = esw_set_master_egress_rule(master_esw->dev, 2736 slave_esw->dev); 2737 if (err) 2738 goto err_acl; 2739 2740 return err; 2741 2742 err_acl: 2743 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2744 2745 return err; 2746 } 2747 2748 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw, 2749 struct mlx5_eswitch *slave_esw) 2750 { 2751 esw_unset_master_egress_rule(master_esw->dev); 2752 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2753 } 2754 2755 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 2756 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2757 2758 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw) 2759 { 2760 const struct mlx5_eswitch_rep_ops *ops; 2761 struct mlx5_eswitch_rep *rep; 2762 unsigned long i; 2763 u8 rep_type; 2764 2765 mlx5_esw_for_each_rep(esw, i, rep) { 2766 rep_type = NUM_REP_TYPES; 2767 while (rep_type--) { 2768 ops = esw->offloads.rep_ops[rep_type]; 2769 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2770 ops->event) 2771 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL); 2772 } 2773 } 2774 } 2775 2776 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) 2777 { 2778 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2779 mlx5e_tc_clean_fdb_peer_flows(esw); 2780 #endif 2781 mlx5_esw_offloads_rep_event_unpair(esw); 2782 esw_del_fdb_peer_miss_rules(esw); 2783 } 2784 2785 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2786 struct mlx5_eswitch *peer_esw) 2787 { 2788 const struct mlx5_eswitch_rep_ops *ops; 2789 struct mlx5_eswitch_rep *rep; 2790 unsigned long i; 2791 u8 rep_type; 2792 int err; 2793 2794 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2795 if (err) 2796 return err; 2797 2798 mlx5_esw_for_each_rep(esw, i, rep) { 2799 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2800 ops = esw->offloads.rep_ops[rep_type]; 2801 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2802 ops->event) { 2803 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 2804 if (err) 2805 goto err_out; 2806 } 2807 } 2808 } 2809 2810 return 0; 2811 2812 err_out: 2813 mlx5_esw_offloads_unpair(esw); 2814 return err; 2815 } 2816 2817 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2818 struct mlx5_eswitch *peer_esw, 2819 bool pair) 2820 { 2821 struct mlx5_flow_root_namespace *peer_ns; 2822 struct mlx5_flow_root_namespace *ns; 2823 int err; 2824 2825 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2826 ns = esw->dev->priv.steering->fdb_root_ns; 2827 2828 if (pair) { 2829 err = mlx5_flow_namespace_set_peer(ns, peer_ns); 2830 if (err) 2831 return err; 2832 2833 err = mlx5_flow_namespace_set_peer(peer_ns, ns); 2834 if (err) { 2835 mlx5_flow_namespace_set_peer(ns, NULL); 2836 return err; 2837 } 2838 } else { 2839 mlx5_flow_namespace_set_peer(ns, NULL); 2840 mlx5_flow_namespace_set_peer(peer_ns, NULL); 2841 } 2842 2843 return 0; 2844 } 2845 2846 static int mlx5_esw_offloads_devcom_event(int event, 2847 void *my_data, 2848 void *event_data) 2849 { 2850 struct mlx5_eswitch *esw = my_data; 2851 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2852 struct mlx5_eswitch *peer_esw = event_data; 2853 int err; 2854 2855 switch (event) { 2856 case ESW_OFFLOADS_DEVCOM_PAIR: 2857 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2858 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2859 break; 2860 2861 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2862 if (err) 2863 goto err_out; 2864 err = mlx5_esw_offloads_pair(esw, peer_esw); 2865 if (err) 2866 goto err_peer; 2867 2868 err = mlx5_esw_offloads_pair(peer_esw, esw); 2869 if (err) 2870 goto err_pair; 2871 2872 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); 2873 break; 2874 2875 case ESW_OFFLOADS_DEVCOM_UNPAIR: 2876 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) 2877 break; 2878 2879 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); 2880 mlx5_esw_offloads_unpair(peer_esw); 2881 mlx5_esw_offloads_unpair(esw); 2882 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2883 break; 2884 } 2885 2886 return 0; 2887 2888 err_pair: 2889 mlx5_esw_offloads_unpair(esw); 2890 err_peer: 2891 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2892 err_out: 2893 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 2894 event, err); 2895 return err; 2896 } 2897 2898 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw) 2899 { 2900 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2901 2902 INIT_LIST_HEAD(&esw->offloads.peer_flows); 2903 mutex_init(&esw->offloads.peer_mutex); 2904 2905 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2906 return; 2907 2908 if (!mlx5_is_lag_supported(esw->dev)) 2909 return; 2910 2911 mlx5_devcom_register_component(devcom, 2912 MLX5_DEVCOM_ESW_OFFLOADS, 2913 mlx5_esw_offloads_devcom_event, 2914 esw); 2915 2916 mlx5_devcom_send_event(devcom, 2917 MLX5_DEVCOM_ESW_OFFLOADS, 2918 ESW_OFFLOADS_DEVCOM_PAIR, esw); 2919 } 2920 2921 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 2922 { 2923 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2924 2925 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2926 return; 2927 2928 if (!mlx5_is_lag_supported(esw->dev)) 2929 return; 2930 2931 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, 2932 ESW_OFFLOADS_DEVCOM_UNPAIR, esw); 2933 2934 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); 2935 } 2936 2937 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 2938 { 2939 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 2940 return false; 2941 2942 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 2943 MLX5_FDB_TO_VPORT_REG_C_0)) 2944 return false; 2945 2946 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 2947 return false; 2948 2949 return true; 2950 } 2951 2952 #define MLX5_ESW_METADATA_RSVD_UPLINK 1 2953 2954 /* Share the same metadata for uplink's. This is fine because: 2955 * (a) In shared FDB mode (LAG) both uplink's are treated the 2956 * same and tagged with the same metadata. 2957 * (b) In non shared FDB mode, packets from physical port0 2958 * cannot hit eswitch of PF1 and vice versa. 2959 */ 2960 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 2961 { 2962 return MLX5_ESW_METADATA_RSVD_UPLINK; 2963 } 2964 2965 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 2966 { 2967 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 2968 /* Reserve 0xf for internal port offload */ 2969 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 2970 u32 pf_num; 2971 int id; 2972 2973 /* Only 4 bits of pf_num */ 2974 pf_num = mlx5_get_dev_index(esw->dev); 2975 if (pf_num > max_pf_num) 2976 return 0; 2977 2978 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 2979 /* Use only non-zero vport_id (2-4095) for all PF's */ 2980 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 2981 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 2982 vport_end_ida, GFP_KERNEL); 2983 if (id < 0) 2984 return 0; 2985 id = (pf_num << ESW_VPORT_BITS) | id; 2986 return id; 2987 } 2988 2989 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 2990 { 2991 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 2992 2993 /* Metadata contains only 12 bits of actual ida id */ 2994 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 2995 } 2996 2997 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 2998 struct mlx5_vport *vport) 2999 { 3000 if (vport->vport == MLX5_VPORT_UPLINK) 3001 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 3002 else 3003 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 3004 3005 vport->metadata = vport->default_metadata; 3006 return vport->metadata ? 0 : -ENOSPC; 3007 } 3008 3009 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 3010 struct mlx5_vport *vport) 3011 { 3012 if (!vport->default_metadata) 3013 return; 3014 3015 if (vport->vport == MLX5_VPORT_UPLINK) 3016 return; 3017 3018 WARN_ON(vport->metadata != vport->default_metadata); 3019 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 3020 } 3021 3022 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 3023 { 3024 struct mlx5_vport *vport; 3025 unsigned long i; 3026 3027 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3028 return; 3029 3030 mlx5_esw_for_each_vport(esw, i, vport) 3031 esw_offloads_vport_metadata_cleanup(esw, vport); 3032 } 3033 3034 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 3035 { 3036 struct mlx5_vport *vport; 3037 unsigned long i; 3038 int err; 3039 3040 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3041 return 0; 3042 3043 mlx5_esw_for_each_vport(esw, i, vport) { 3044 err = esw_offloads_vport_metadata_setup(esw, vport); 3045 if (err) 3046 goto metadata_err; 3047 } 3048 3049 return 0; 3050 3051 metadata_err: 3052 esw_offloads_metadata_uninit(esw); 3053 return err; 3054 } 3055 3056 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable) 3057 { 3058 int err = 0; 3059 3060 down_write(&esw->mode_lock); 3061 if (mlx5_esw_is_fdb_created(esw)) { 3062 err = -EBUSY; 3063 goto done; 3064 } 3065 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 3066 err = -EOPNOTSUPP; 3067 goto done; 3068 } 3069 if (enable) 3070 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 3071 else 3072 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 3073 done: 3074 up_write(&esw->mode_lock); 3075 return err; 3076 } 3077 3078 int 3079 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 3080 struct mlx5_vport *vport) 3081 { 3082 int err; 3083 3084 err = esw_acl_ingress_ofld_setup(esw, vport); 3085 if (err) 3086 return err; 3087 3088 err = esw_acl_egress_ofld_setup(esw, vport); 3089 if (err) 3090 goto egress_err; 3091 3092 return 0; 3093 3094 egress_err: 3095 esw_acl_ingress_ofld_cleanup(esw, vport); 3096 return err; 3097 } 3098 3099 void 3100 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 3101 struct mlx5_vport *vport) 3102 { 3103 esw_acl_egress_ofld_cleanup(vport); 3104 esw_acl_ingress_ofld_cleanup(esw, vport); 3105 } 3106 3107 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 3108 { 3109 struct mlx5_vport *vport; 3110 3111 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3112 if (IS_ERR(vport)) 3113 return PTR_ERR(vport); 3114 3115 return esw_vport_create_offloads_acl_tables(esw, vport); 3116 } 3117 3118 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 3119 { 3120 struct mlx5_vport *vport; 3121 3122 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3123 if (IS_ERR(vport)) 3124 return; 3125 3126 esw_vport_destroy_offloads_acl_tables(esw, vport); 3127 } 3128 3129 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) 3130 { 3131 struct mlx5_eswitch_rep *rep; 3132 unsigned long i; 3133 int ret; 3134 3135 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3136 return 0; 3137 3138 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3139 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3140 return 0; 3141 3142 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3143 if (ret) 3144 return ret; 3145 3146 mlx5_esw_for_each_rep(esw, i, rep) { 3147 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3148 mlx5_esw_offloads_rep_load(esw, rep->vport); 3149 } 3150 3151 return 0; 3152 } 3153 3154 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3155 { 3156 struct mlx5_esw_indir_table *indir; 3157 int err; 3158 3159 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3160 mutex_init(&esw->fdb_table.offloads.vports.lock); 3161 hash_init(esw->fdb_table.offloads.vports.table); 3162 atomic64_set(&esw->user_count, 0); 3163 3164 indir = mlx5_esw_indir_table_init(); 3165 if (IS_ERR(indir)) { 3166 err = PTR_ERR(indir); 3167 goto create_indir_err; 3168 } 3169 esw->fdb_table.offloads.indir = indir; 3170 3171 err = esw_create_uplink_offloads_acl_tables(esw); 3172 if (err) 3173 goto create_acl_err; 3174 3175 err = esw_create_offloads_table(esw); 3176 if (err) 3177 goto create_offloads_err; 3178 3179 err = esw_create_restore_table(esw); 3180 if (err) 3181 goto create_restore_err; 3182 3183 err = esw_create_offloads_fdb_tables(esw); 3184 if (err) 3185 goto create_fdb_err; 3186 3187 err = esw_create_vport_rx_group(esw); 3188 if (err) 3189 goto create_fg_err; 3190 3191 err = esw_create_vport_rx_drop_group(esw); 3192 if (err) 3193 goto create_rx_drop_fg_err; 3194 3195 err = esw_create_vport_rx_drop_rule(esw); 3196 if (err) 3197 goto create_rx_drop_rule_err; 3198 3199 return 0; 3200 3201 create_rx_drop_rule_err: 3202 esw_destroy_vport_rx_drop_group(esw); 3203 create_rx_drop_fg_err: 3204 esw_destroy_vport_rx_group(esw); 3205 create_fg_err: 3206 esw_destroy_offloads_fdb_tables(esw); 3207 create_fdb_err: 3208 esw_destroy_restore_table(esw); 3209 create_restore_err: 3210 esw_destroy_offloads_table(esw); 3211 create_offloads_err: 3212 esw_destroy_uplink_offloads_acl_tables(esw); 3213 create_acl_err: 3214 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3215 create_indir_err: 3216 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3217 return err; 3218 } 3219 3220 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3221 { 3222 esw_destroy_vport_rx_drop_rule(esw); 3223 esw_destroy_vport_rx_drop_group(esw); 3224 esw_destroy_vport_rx_group(esw); 3225 esw_destroy_offloads_fdb_tables(esw); 3226 esw_destroy_restore_table(esw); 3227 esw_destroy_offloads_table(esw); 3228 esw_destroy_uplink_offloads_acl_tables(esw); 3229 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3230 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3231 } 3232 3233 static void 3234 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 3235 { 3236 struct devlink *devlink; 3237 bool host_pf_disabled; 3238 u16 new_num_vfs; 3239 3240 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3241 host_params_context.host_num_of_vfs); 3242 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3243 host_params_context.host_pf_disabled); 3244 3245 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3246 return; 3247 3248 devlink = priv_to_devlink(esw->dev); 3249 devl_lock(devlink); 3250 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3251 if (esw->esw_funcs.num_vfs > 0) { 3252 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3253 } else { 3254 int err; 3255 3256 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3257 MLX5_VPORT_UC_ADDR_CHANGE); 3258 if (err) { 3259 devl_unlock(devlink); 3260 return; 3261 } 3262 } 3263 esw->esw_funcs.num_vfs = new_num_vfs; 3264 devl_unlock(devlink); 3265 } 3266 3267 static void esw_functions_changed_event_handler(struct work_struct *work) 3268 { 3269 struct mlx5_host_work *host_work; 3270 struct mlx5_eswitch *esw; 3271 const u32 *out; 3272 3273 host_work = container_of(work, struct mlx5_host_work, work); 3274 esw = host_work->esw; 3275 3276 out = mlx5_esw_query_functions(esw->dev); 3277 if (IS_ERR(out)) 3278 goto out; 3279 3280 esw_vfs_changed_event_handler(esw, out); 3281 kvfree(out); 3282 out: 3283 kfree(host_work); 3284 } 3285 3286 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3287 { 3288 struct mlx5_esw_functions *esw_funcs; 3289 struct mlx5_host_work *host_work; 3290 struct mlx5_eswitch *esw; 3291 3292 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 3293 if (!host_work) 3294 return NOTIFY_DONE; 3295 3296 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3297 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3298 3299 host_work->esw = esw; 3300 3301 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3302 queue_work(esw->work_queue, &host_work->work); 3303 3304 return NOTIFY_OK; 3305 } 3306 3307 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3308 { 3309 const u32 *query_host_out; 3310 3311 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3312 return 0; 3313 3314 query_host_out = mlx5_esw_query_functions(esw->dev); 3315 if (IS_ERR(query_host_out)) 3316 return PTR_ERR(query_host_out); 3317 3318 /* Mark non local controller with non zero controller number. */ 3319 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3320 host_params_context.host_number); 3321 kvfree(query_host_out); 3322 return 0; 3323 } 3324 3325 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3326 { 3327 /* Local controller is always valid */ 3328 if (controller == 0) 3329 return true; 3330 3331 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3332 return false; 3333 3334 /* External host number starts with zero in device */ 3335 return (controller == esw->offloads.host_number + 1); 3336 } 3337 3338 int esw_offloads_enable(struct mlx5_eswitch *esw) 3339 { 3340 struct mapping_ctx *reg_c0_obj_pool; 3341 struct mlx5_vport *vport; 3342 unsigned long i; 3343 u64 mapping_id; 3344 int err; 3345 3346 mutex_init(&esw->offloads.termtbl_mutex); 3347 mlx5_rdma_enable_roce(esw->dev); 3348 3349 err = mlx5_esw_host_number_init(esw); 3350 if (err) 3351 goto err_metadata; 3352 3353 err = esw_offloads_metadata_init(esw); 3354 if (err) 3355 goto err_metadata; 3356 3357 err = esw_set_passing_vport_metadata(esw, true); 3358 if (err) 3359 goto err_vport_metadata; 3360 3361 mapping_id = mlx5_query_nic_system_image_guid(esw->dev); 3362 3363 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN, 3364 sizeof(struct mlx5_mapped_obj), 3365 ESW_REG_C0_USER_DATA_METADATA_MASK, 3366 true); 3367 3368 if (IS_ERR(reg_c0_obj_pool)) { 3369 err = PTR_ERR(reg_c0_obj_pool); 3370 goto err_pool; 3371 } 3372 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3373 3374 err = esw_offloads_steering_init(esw); 3375 if (err) 3376 goto err_steering_init; 3377 3378 /* Representor will control the vport link state */ 3379 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3380 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3381 3382 /* Uplink vport rep must load first. */ 3383 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); 3384 if (err) 3385 goto err_uplink; 3386 3387 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3388 if (err) 3389 goto err_vports; 3390 3391 esw_offloads_devcom_init(esw); 3392 3393 return 0; 3394 3395 err_vports: 3396 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 3397 err_uplink: 3398 esw_offloads_steering_cleanup(esw); 3399 err_steering_init: 3400 mapping_destroy(reg_c0_obj_pool); 3401 err_pool: 3402 esw_set_passing_vport_metadata(esw, false); 3403 err_vport_metadata: 3404 esw_offloads_metadata_uninit(esw); 3405 err_metadata: 3406 mlx5_rdma_disable_roce(esw->dev); 3407 mutex_destroy(&esw->offloads.termtbl_mutex); 3408 return err; 3409 } 3410 3411 static int esw_offloads_stop(struct mlx5_eswitch *esw, 3412 struct netlink_ext_ack *extack) 3413 { 3414 int err; 3415 3416 esw->mode = MLX5_ESWITCH_LEGACY; 3417 3418 /* If changing from switchdev to legacy mode without sriov enabled, 3419 * no need to create legacy fdb. 3420 */ 3421 if (!mlx5_sriov_is_enabled(esw->dev)) 3422 return 0; 3423 3424 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3425 if (err) 3426 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3427 3428 return err; 3429 } 3430 3431 void esw_offloads_disable(struct mlx5_eswitch *esw) 3432 { 3433 esw_offloads_devcom_cleanup(esw); 3434 mlx5_eswitch_disable_pf_vf_vports(esw); 3435 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 3436 esw_set_passing_vport_metadata(esw, false); 3437 esw_offloads_steering_cleanup(esw); 3438 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3439 esw_offloads_metadata_uninit(esw); 3440 mlx5_rdma_disable_roce(esw->dev); 3441 mutex_destroy(&esw->offloads.termtbl_mutex); 3442 } 3443 3444 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3445 { 3446 switch (mode) { 3447 case DEVLINK_ESWITCH_MODE_LEGACY: 3448 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3449 break; 3450 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3451 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3452 break; 3453 default: 3454 return -EINVAL; 3455 } 3456 3457 return 0; 3458 } 3459 3460 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 3461 { 3462 switch (mlx5_mode) { 3463 case MLX5_ESWITCH_LEGACY: 3464 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3465 break; 3466 case MLX5_ESWITCH_OFFLOADS: 3467 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3468 break; 3469 default: 3470 return -EINVAL; 3471 } 3472 3473 return 0; 3474 } 3475 3476 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3477 { 3478 switch (mode) { 3479 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3480 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3481 break; 3482 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3483 *mlx5_mode = MLX5_INLINE_MODE_L2; 3484 break; 3485 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3486 *mlx5_mode = MLX5_INLINE_MODE_IP; 3487 break; 3488 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3489 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3490 break; 3491 default: 3492 return -EINVAL; 3493 } 3494 3495 return 0; 3496 } 3497 3498 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3499 { 3500 switch (mlx5_mode) { 3501 case MLX5_INLINE_MODE_NONE: 3502 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3503 break; 3504 case MLX5_INLINE_MODE_L2: 3505 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3506 break; 3507 case MLX5_INLINE_MODE_IP: 3508 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3509 break; 3510 case MLX5_INLINE_MODE_TCP_UDP: 3511 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3512 break; 3513 default: 3514 return -EINVAL; 3515 } 3516 3517 return 0; 3518 } 3519 3520 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 3521 struct netlink_ext_ack *extack) 3522 { 3523 u16 cur_mlx5_mode, mlx5_mode = 0; 3524 struct mlx5_eswitch *esw; 3525 int err = 0; 3526 3527 esw = mlx5_devlink_eswitch_get(devlink); 3528 if (IS_ERR(esw)) 3529 return PTR_ERR(esw); 3530 3531 if (esw_mode_from_devlink(mode, &mlx5_mode)) 3532 return -EINVAL; 3533 3534 mlx5_lag_disable_change(esw->dev); 3535 err = mlx5_esw_try_lock(esw); 3536 if (err < 0) { 3537 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 3538 goto enable_lag; 3539 } 3540 cur_mlx5_mode = err; 3541 err = 0; 3542 3543 if (cur_mlx5_mode == mlx5_mode) 3544 goto unlock; 3545 3546 mlx5_eswitch_disable_locked(esw); 3547 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { 3548 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 3549 NL_SET_ERR_MSG_MOD(extack, 3550 "Can't change mode while devlink traps are active"); 3551 err = -EOPNOTSUPP; 3552 goto unlock; 3553 } 3554 err = esw_offloads_start(esw, extack); 3555 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { 3556 err = esw_offloads_stop(esw, extack); 3557 mlx5_rescan_drivers(esw->dev); 3558 } else { 3559 err = -EINVAL; 3560 } 3561 3562 unlock: 3563 mlx5_esw_unlock(esw); 3564 enable_lag: 3565 mlx5_lag_enable_change(esw->dev); 3566 return err; 3567 } 3568 3569 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 3570 { 3571 struct mlx5_eswitch *esw; 3572 int err; 3573 3574 esw = mlx5_devlink_eswitch_get(devlink); 3575 if (IS_ERR(esw)) 3576 return PTR_ERR(esw); 3577 3578 down_write(&esw->mode_lock); 3579 err = esw_mode_to_devlink(esw->mode, mode); 3580 up_write(&esw->mode_lock); 3581 return err; 3582 } 3583 3584 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 3585 struct netlink_ext_ack *extack) 3586 { 3587 struct mlx5_core_dev *dev = esw->dev; 3588 struct mlx5_vport *vport; 3589 u16 err_vport_num = 0; 3590 unsigned long i; 3591 int err = 0; 3592 3593 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3594 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3595 if (err) { 3596 err_vport_num = vport->vport; 3597 NL_SET_ERR_MSG_MOD(extack, 3598 "Failed to set min inline on vport"); 3599 goto revert_inline_mode; 3600 } 3601 } 3602 return 0; 3603 3604 revert_inline_mode: 3605 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3606 if (vport->vport == err_vport_num) 3607 break; 3608 mlx5_modify_nic_vport_min_inline(dev, 3609 vport->vport, 3610 esw->offloads.inline_mode); 3611 } 3612 return err; 3613 } 3614 3615 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 3616 struct netlink_ext_ack *extack) 3617 { 3618 struct mlx5_core_dev *dev = devlink_priv(devlink); 3619 struct mlx5_eswitch *esw; 3620 u8 mlx5_mode; 3621 int err; 3622 3623 esw = mlx5_devlink_eswitch_get(devlink); 3624 if (IS_ERR(esw)) 3625 return PTR_ERR(esw); 3626 3627 down_write(&esw->mode_lock); 3628 3629 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 3630 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 3631 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 3632 err = 0; 3633 goto out; 3634 } 3635 3636 fallthrough; 3637 case MLX5_CAP_INLINE_MODE_L2: 3638 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 3639 err = -EOPNOTSUPP; 3640 goto out; 3641 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 3642 break; 3643 } 3644 3645 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3646 NL_SET_ERR_MSG_MOD(extack, 3647 "Can't set inline mode when flows are configured"); 3648 err = -EOPNOTSUPP; 3649 goto out; 3650 } 3651 3652 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 3653 if (err) 3654 goto out; 3655 3656 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 3657 if (err) 3658 goto out; 3659 3660 esw->offloads.inline_mode = mlx5_mode; 3661 up_write(&esw->mode_lock); 3662 return 0; 3663 3664 out: 3665 up_write(&esw->mode_lock); 3666 return err; 3667 } 3668 3669 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3670 { 3671 struct mlx5_eswitch *esw; 3672 int err; 3673 3674 esw = mlx5_devlink_eswitch_get(devlink); 3675 if (IS_ERR(esw)) 3676 return PTR_ERR(esw); 3677 3678 down_write(&esw->mode_lock); 3679 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3680 up_write(&esw->mode_lock); 3681 return err; 3682 } 3683 3684 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3685 enum devlink_eswitch_encap_mode encap, 3686 struct netlink_ext_ack *extack) 3687 { 3688 struct mlx5_core_dev *dev = devlink_priv(devlink); 3689 struct mlx5_eswitch *esw; 3690 int err = 0; 3691 3692 esw = mlx5_devlink_eswitch_get(devlink); 3693 if (IS_ERR(esw)) 3694 return PTR_ERR(esw); 3695 3696 down_write(&esw->mode_lock); 3697 3698 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3699 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3700 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3701 err = -EOPNOTSUPP; 3702 goto unlock; 3703 } 3704 3705 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3706 err = -EOPNOTSUPP; 3707 goto unlock; 3708 } 3709 3710 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3711 esw->offloads.encap = encap; 3712 goto unlock; 3713 } 3714 3715 if (esw->offloads.encap == encap) 3716 goto unlock; 3717 3718 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3719 NL_SET_ERR_MSG_MOD(extack, 3720 "Can't set encapsulation when flows are configured"); 3721 err = -EOPNOTSUPP; 3722 goto unlock; 3723 } 3724 3725 esw_destroy_offloads_fdb_tables(esw); 3726 3727 esw->offloads.encap = encap; 3728 3729 err = esw_create_offloads_fdb_tables(esw); 3730 3731 if (err) { 3732 NL_SET_ERR_MSG_MOD(extack, 3733 "Failed re-creating fast FDB table"); 3734 esw->offloads.encap = !encap; 3735 (void)esw_create_offloads_fdb_tables(esw); 3736 } 3737 3738 unlock: 3739 up_write(&esw->mode_lock); 3740 return err; 3741 } 3742 3743 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 3744 enum devlink_eswitch_encap_mode *encap) 3745 { 3746 struct mlx5_eswitch *esw; 3747 3748 esw = mlx5_devlink_eswitch_get(devlink); 3749 if (IS_ERR(esw)) 3750 return PTR_ERR(esw); 3751 3752 down_write(&esw->mode_lock); 3753 *encap = esw->offloads.encap; 3754 up_write(&esw->mode_lock); 3755 return 0; 3756 } 3757 3758 static bool 3759 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 3760 { 3761 /* Currently, only ECPF based device has representor for host PF. */ 3762 if (vport_num == MLX5_VPORT_PF && 3763 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 3764 return false; 3765 3766 if (vport_num == MLX5_VPORT_ECPF && 3767 !mlx5_ecpf_vport_exists(esw->dev)) 3768 return false; 3769 3770 return true; 3771 } 3772 3773 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 3774 const struct mlx5_eswitch_rep_ops *ops, 3775 u8 rep_type) 3776 { 3777 struct mlx5_eswitch_rep_data *rep_data; 3778 struct mlx5_eswitch_rep *rep; 3779 unsigned long i; 3780 3781 esw->offloads.rep_ops[rep_type] = ops; 3782 mlx5_esw_for_each_rep(esw, i, rep) { 3783 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 3784 rep->esw = esw; 3785 rep_data = &rep->rep_data[rep_type]; 3786 atomic_set(&rep_data->state, REP_REGISTERED); 3787 } 3788 } 3789 } 3790 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 3791 3792 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 3793 { 3794 struct mlx5_eswitch_rep *rep; 3795 unsigned long i; 3796 3797 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 3798 __unload_reps_all_vport(esw, rep_type); 3799 3800 mlx5_esw_for_each_rep(esw, i, rep) 3801 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 3802 } 3803 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 3804 3805 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 3806 { 3807 struct mlx5_eswitch_rep *rep; 3808 3809 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3810 return rep->rep_data[rep_type].priv; 3811 } 3812 3813 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 3814 u16 vport, 3815 u8 rep_type) 3816 { 3817 struct mlx5_eswitch_rep *rep; 3818 3819 rep = mlx5_eswitch_get_rep(esw, vport); 3820 3821 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3822 esw->offloads.rep_ops[rep_type]->get_proto_dev) 3823 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 3824 return NULL; 3825 } 3826 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 3827 3828 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 3829 { 3830 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 3831 } 3832 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 3833 3834 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 3835 u16 vport) 3836 { 3837 return mlx5_eswitch_get_rep(esw, vport); 3838 } 3839 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 3840 3841 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 3842 { 3843 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 3844 } 3845 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 3846 3847 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 3848 { 3849 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 3850 } 3851 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 3852 3853 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 3854 u16 vport_num) 3855 { 3856 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3857 3858 if (WARN_ON_ONCE(IS_ERR(vport))) 3859 return 0; 3860 3861 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 3862 } 3863 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 3864 3865 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port, 3866 u16 vport_num, u32 controller, u32 sfnum) 3867 { 3868 int err; 3869 3870 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE); 3871 if (err) 3872 return err; 3873 3874 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum); 3875 if (err) 3876 goto devlink_err; 3877 3878 mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum); 3879 err = mlx5_esw_offloads_rep_load(esw, vport_num); 3880 if (err) 3881 goto rep_err; 3882 return 0; 3883 3884 rep_err: 3885 mlx5_esw_vport_debugfs_destroy(esw, vport_num); 3886 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3887 devlink_err: 3888 mlx5_esw_vport_disable(esw, vport_num); 3889 return err; 3890 } 3891 3892 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num) 3893 { 3894 mlx5_esw_offloads_rep_unload(esw, vport_num); 3895 mlx5_esw_vport_debugfs_destroy(esw, vport_num); 3896 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3897 mlx5_esw_vport_disable(esw, vport_num); 3898 } 3899 3900 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 3901 { 3902 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 3903 void *query_ctx; 3904 void *hca_caps; 3905 int err; 3906 3907 *vhca_id = 0; 3908 if (mlx5_esw_is_manager_vport(esw, vport_num) || 3909 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) 3910 return -EPERM; 3911 3912 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 3913 if (!query_ctx) 3914 return -ENOMEM; 3915 3916 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx); 3917 if (err) 3918 goto out_free; 3919 3920 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 3921 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 3922 3923 out_free: 3924 kfree(query_ctx); 3925 return err; 3926 } 3927 3928 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 3929 { 3930 u16 *old_entry, *vhca_map_entry, vhca_id; 3931 int err; 3932 3933 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3934 if (err) { 3935 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 3936 vport_num, err); 3937 return err; 3938 } 3939 3940 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 3941 if (!vhca_map_entry) 3942 return -ENOMEM; 3943 3944 *vhca_map_entry = vport_num; 3945 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 3946 if (xa_is_err(old_entry)) { 3947 kfree(vhca_map_entry); 3948 return xa_err(old_entry); 3949 } 3950 kfree(old_entry); 3951 return 0; 3952 } 3953 3954 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 3955 { 3956 u16 *vhca_map_entry, vhca_id; 3957 int err; 3958 3959 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3960 if (err) 3961 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 3962 vport_num, err); 3963 3964 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 3965 kfree(vhca_map_entry); 3966 } 3967 3968 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 3969 { 3970 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 3971 3972 if (!res) 3973 return -ENOENT; 3974 3975 *vport_num = *res; 3976 return 0; 3977 } 3978 3979 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 3980 u16 vport_num) 3981 { 3982 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3983 3984 if (WARN_ON_ONCE(IS_ERR(vport))) 3985 return 0; 3986 3987 return vport->metadata; 3988 } 3989 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 3990 3991 static bool 3992 is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num) 3993 { 3994 return vport_num == MLX5_VPORT_PF || 3995 mlx5_eswitch_is_vf_vport(esw, vport_num) || 3996 mlx5_esw_is_sf_vport(esw, vport_num); 3997 } 3998 3999 int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port, 4000 u8 *hw_addr, int *hw_addr_len, 4001 struct netlink_ext_ack *extack) 4002 { 4003 struct mlx5_eswitch *esw; 4004 struct mlx5_vport *vport; 4005 u16 vport_num; 4006 4007 esw = mlx5_devlink_eswitch_get(port->devlink); 4008 if (IS_ERR(esw)) 4009 return PTR_ERR(esw); 4010 4011 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4012 if (!is_port_function_supported(esw, vport_num)) 4013 return -EOPNOTSUPP; 4014 4015 vport = mlx5_eswitch_get_vport(esw, vport_num); 4016 if (IS_ERR(vport)) { 4017 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4018 return PTR_ERR(vport); 4019 } 4020 4021 mutex_lock(&esw->state_lock); 4022 ether_addr_copy(hw_addr, vport->info.mac); 4023 *hw_addr_len = ETH_ALEN; 4024 mutex_unlock(&esw->state_lock); 4025 return 0; 4026 } 4027 4028 int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, 4029 const u8 *hw_addr, int hw_addr_len, 4030 struct netlink_ext_ack *extack) 4031 { 4032 struct mlx5_eswitch *esw; 4033 u16 vport_num; 4034 4035 esw = mlx5_devlink_eswitch_get(port->devlink); 4036 if (IS_ERR(esw)) { 4037 NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr"); 4038 return PTR_ERR(esw); 4039 } 4040 4041 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4042 if (!is_port_function_supported(esw, vport_num)) { 4043 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr"); 4044 return -EINVAL; 4045 } 4046 4047 return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr); 4048 } 4049 4050 static struct mlx5_vport * 4051 mlx5_devlink_port_fn_get_vport(struct devlink_port *port, struct mlx5_eswitch *esw) 4052 { 4053 u16 vport_num; 4054 4055 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) 4056 return ERR_PTR(-EOPNOTSUPP); 4057 4058 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4059 if (!is_port_function_supported(esw, vport_num)) 4060 return ERR_PTR(-EOPNOTSUPP); 4061 4062 return mlx5_eswitch_get_vport(esw, vport_num); 4063 } 4064 4065 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 4066 struct netlink_ext_ack *extack) 4067 { 4068 struct mlx5_eswitch *esw; 4069 struct mlx5_vport *vport; 4070 int err = -EOPNOTSUPP; 4071 4072 esw = mlx5_devlink_eswitch_get(port->devlink); 4073 if (IS_ERR(esw)) 4074 return PTR_ERR(esw); 4075 4076 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4077 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4078 return err; 4079 } 4080 4081 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4082 if (IS_ERR(vport)) { 4083 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4084 return PTR_ERR(vport); 4085 } 4086 4087 mutex_lock(&esw->state_lock); 4088 if (vport->enabled) { 4089 *is_enabled = vport->info.mig_enabled; 4090 err = 0; 4091 } 4092 mutex_unlock(&esw->state_lock); 4093 return err; 4094 } 4095 4096 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 4097 struct netlink_ext_ack *extack) 4098 { 4099 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4100 struct mlx5_eswitch *esw; 4101 struct mlx5_vport *vport; 4102 void *query_ctx; 4103 void *hca_caps; 4104 int err = -EOPNOTSUPP; 4105 4106 esw = mlx5_devlink_eswitch_get(port->devlink); 4107 if (IS_ERR(esw)) 4108 return PTR_ERR(esw); 4109 4110 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4111 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4112 return err; 4113 } 4114 4115 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4116 if (IS_ERR(vport)) { 4117 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4118 return PTR_ERR(vport); 4119 } 4120 4121 mutex_lock(&esw->state_lock); 4122 if (!vport->enabled) { 4123 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4124 goto out; 4125 } 4126 4127 if (vport->info.mig_enabled == enable) { 4128 err = 0; 4129 goto out; 4130 } 4131 4132 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4133 if (!query_ctx) { 4134 err = -ENOMEM; 4135 goto out; 4136 } 4137 4138 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 4139 MLX5_CAP_GENERAL_2); 4140 if (err) { 4141 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4142 goto out_free; 4143 } 4144 4145 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4146 memcpy(hca_caps, MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability), 4147 MLX5_UN_SZ_BYTES(hca_cap_union)); 4148 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, 1); 4149 4150 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 4151 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 4152 if (err) { 4153 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 4154 goto out_free; 4155 } 4156 4157 vport->info.mig_enabled = enable; 4158 4159 out_free: 4160 kfree(query_ctx); 4161 out: 4162 mutex_unlock(&esw->state_lock); 4163 return err; 4164 } 4165 4166 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 4167 struct netlink_ext_ack *extack) 4168 { 4169 struct mlx5_eswitch *esw; 4170 struct mlx5_vport *vport; 4171 int err = -EOPNOTSUPP; 4172 4173 esw = mlx5_devlink_eswitch_get(port->devlink); 4174 if (IS_ERR(esw)) 4175 return PTR_ERR(esw); 4176 4177 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4178 if (IS_ERR(vport)) { 4179 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4180 return PTR_ERR(vport); 4181 } 4182 4183 mutex_lock(&esw->state_lock); 4184 if (vport->enabled) { 4185 *is_enabled = vport->info.roce_enabled; 4186 err = 0; 4187 } 4188 mutex_unlock(&esw->state_lock); 4189 return err; 4190 } 4191 4192 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 4193 struct netlink_ext_ack *extack) 4194 { 4195 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4196 struct mlx5_eswitch *esw; 4197 struct mlx5_vport *vport; 4198 int err = -EOPNOTSUPP; 4199 void *query_ctx; 4200 void *hca_caps; 4201 u16 vport_num; 4202 4203 esw = mlx5_devlink_eswitch_get(port->devlink); 4204 if (IS_ERR(esw)) 4205 return PTR_ERR(esw); 4206 4207 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4208 if (IS_ERR(vport)) { 4209 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4210 return PTR_ERR(vport); 4211 } 4212 vport_num = vport->vport; 4213 4214 mutex_lock(&esw->state_lock); 4215 if (!vport->enabled) { 4216 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4217 goto out; 4218 } 4219 4220 if (vport->info.roce_enabled == enable) { 4221 err = 0; 4222 goto out; 4223 } 4224 4225 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4226 if (!query_ctx) { 4227 err = -ENOMEM; 4228 goto out; 4229 } 4230 4231 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4232 MLX5_CAP_GENERAL); 4233 if (err) { 4234 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4235 goto out_free; 4236 } 4237 4238 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4239 memcpy(hca_caps, MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability), 4240 MLX5_UN_SZ_BYTES(hca_cap_union)); 4241 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 4242 4243 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4244 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4245 if (err) { 4246 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 4247 goto out_free; 4248 } 4249 4250 vport->info.roce_enabled = enable; 4251 4252 out_free: 4253 kfree(query_ctx); 4254 out: 4255 mutex_unlock(&esw->state_lock); 4256 return err; 4257 } 4258