1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/indir_table.h" 42 #include "esw/acl/ofld.h" 43 #include "rdma.h" 44 #include "en.h" 45 #include "fs_core.h" 46 #include "lib/devcom.h" 47 #include "lib/eq.h" 48 #include "lib/fs_chains.h" 49 #include "en_tc.h" 50 #include "en/mapping.h" 51 #include "devlink.h" 52 #include "lag/lag.h" 53 #include "en/tc/post_meter.h" 54 55 #define mlx5_esw_for_each_rep(esw, i, rep) \ 56 xa_for_each(&((esw)->offloads.vport_reps), i, rep) 57 58 #define mlx5_esw_for_each_sf_rep(esw, i, rep) \ 59 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF) 60 61 #define mlx5_esw_for_each_vf_rep(esw, index, rep) \ 62 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \ 63 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF) 64 65 /* There are two match-all miss flows, one for unicast dst mac and 66 * one for multicast. 67 */ 68 #define MLX5_ESW_MISS_FLOWS (2) 69 #define UPLINK_REP_INDEX 0 70 71 #define MLX5_ESW_VPORT_TBL_SIZE 128 72 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 73 74 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 75 76 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 77 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 78 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 79 .flags = 0, 80 }; 81 82 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 83 u16 vport_num) 84 { 85 return xa_load(&esw->offloads.vport_reps, vport_num); 86 } 87 88 static void 89 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 90 struct mlx5_flow_spec *spec, 91 struct mlx5_esw_flow_attr *attr) 92 { 93 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 94 return; 95 96 if (attr->int_port) { 97 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 98 99 return; 100 } 101 102 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 103 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 104 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 105 } 106 107 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 108 * are not needed as well in the following process. So clear them all for simplicity. 109 */ 110 void 111 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 112 { 113 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 114 void *misc2; 115 116 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 117 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 118 119 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 120 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 121 122 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 123 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 124 } 125 } 126 127 static void 128 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 129 struct mlx5_flow_spec *spec, 130 struct mlx5_flow_attr *attr, 131 struct mlx5_eswitch *src_esw, 132 u16 vport) 133 { 134 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 135 u32 metadata; 136 void *misc2; 137 void *misc; 138 139 /* Use metadata matching because vport is not represented by single 140 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 141 */ 142 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 143 if (mlx5_esw_indir_table_decap_vport(attr)) 144 vport = mlx5_esw_indir_table_decap_vport(attr); 145 146 if (!attr->chain && esw_attr && esw_attr->int_port) 147 metadata = 148 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 149 else 150 metadata = 151 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 152 153 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 154 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 155 156 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 157 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 158 mlx5_eswitch_get_vport_metadata_mask()); 159 160 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 161 } else { 162 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 163 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 164 165 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 166 MLX5_SET(fte_match_set_misc, misc, 167 source_eswitch_owner_vhca_id, 168 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 169 170 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 171 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 172 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 173 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 174 source_eswitch_owner_vhca_id); 175 176 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 177 } 178 } 179 180 static int 181 esw_setup_decap_indir(struct mlx5_eswitch *esw, 182 struct mlx5_flow_attr *attr) 183 { 184 struct mlx5_flow_table *ft; 185 186 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 187 return -EOPNOTSUPP; 188 189 ft = mlx5_esw_indir_table_get(esw, attr, 190 mlx5_esw_indir_table_decap_vport(attr), true); 191 return PTR_ERR_OR_ZERO(ft); 192 } 193 194 static void 195 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 196 struct mlx5_flow_attr *attr) 197 { 198 if (mlx5_esw_indir_table_decap_vport(attr)) 199 mlx5_esw_indir_table_put(esw, 200 mlx5_esw_indir_table_decap_vport(attr), 201 true); 202 } 203 204 static int 205 esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 206 struct mlx5e_meter_attr *meter, 207 int i) 208 { 209 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 210 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 211 dest[i].range.min = 0; 212 dest[i].range.max = meter->params.mtu; 213 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 214 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 215 216 return 0; 217 } 218 219 static int 220 esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 221 struct mlx5_flow_act *flow_act, 222 u32 sampler_id, 223 int i) 224 { 225 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 226 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 227 dest[i].sampler_id = sampler_id; 228 229 return 0; 230 } 231 232 static int 233 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 234 struct mlx5_flow_act *flow_act, 235 struct mlx5_eswitch *esw, 236 struct mlx5_flow_attr *attr, 237 int i) 238 { 239 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 240 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 241 dest[i].ft = attr->dest_ft; 242 243 if (mlx5_esw_indir_table_decap_vport(attr)) 244 return esw_setup_decap_indir(esw, attr); 245 return 0; 246 } 247 248 static void 249 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 250 struct mlx5_fs_chains *chains, int i) 251 { 252 if (mlx5_chains_ignore_flow_level_supported(chains)) 253 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 254 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 255 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 256 } 257 258 static void 259 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 260 struct mlx5_eswitch *esw, int i) 261 { 262 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 263 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 264 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 265 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 266 } 267 268 static int 269 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 270 struct mlx5_flow_act *flow_act, 271 struct mlx5_fs_chains *chains, 272 u32 chain, u32 prio, u32 level, 273 int i) 274 { 275 struct mlx5_flow_table *ft; 276 277 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 278 ft = mlx5_chains_get_table(chains, chain, prio, level); 279 if (IS_ERR(ft)) 280 return PTR_ERR(ft); 281 282 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 283 dest[i].ft = ft; 284 return 0; 285 } 286 287 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 288 int from, int to) 289 { 290 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 291 struct mlx5_fs_chains *chains = esw_chains(esw); 292 int i; 293 294 for (i = from; i < to; i++) 295 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 296 mlx5_chains_put_table(chains, 0, 1, 0); 297 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 298 esw_attr->dests[i].mdev)) 299 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].rep->vport, 300 false); 301 } 302 303 static bool 304 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 305 { 306 int i; 307 308 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 309 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 310 return true; 311 return false; 312 } 313 314 static int 315 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 316 struct mlx5_flow_act *flow_act, 317 struct mlx5_eswitch *esw, 318 struct mlx5_fs_chains *chains, 319 struct mlx5_flow_attr *attr, 320 int *i) 321 { 322 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 323 int err; 324 325 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 326 return -EOPNOTSUPP; 327 328 /* flow steering cannot handle more than one dest with the same ft 329 * in a single flow 330 */ 331 if (esw_attr->out_count - esw_attr->split_count > 1) 332 return -EOPNOTSUPP; 333 334 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 335 if (err) 336 return err; 337 338 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 339 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 340 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 341 } 342 (*i)++; 343 344 return 0; 345 } 346 347 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 348 struct mlx5_flow_attr *attr) 349 { 350 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 351 352 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 353 } 354 355 static bool 356 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 357 { 358 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 359 bool result = false; 360 int i; 361 362 /* Indirect table is supported only for flows with in_port uplink 363 * and the destination is vport on the same eswitch as the uplink, 364 * return false in case at least one of destinations doesn't meet 365 * this criteria. 366 */ 367 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 368 if (esw_attr->dests[i].rep && 369 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport, 370 esw_attr->dests[i].mdev)) { 371 result = true; 372 } else { 373 result = false; 374 break; 375 } 376 } 377 return result; 378 } 379 380 static int 381 esw_setup_indir_table(struct mlx5_flow_destination *dest, 382 struct mlx5_flow_act *flow_act, 383 struct mlx5_eswitch *esw, 384 struct mlx5_flow_attr *attr, 385 bool ignore_flow_lvl, 386 int *i) 387 { 388 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 389 int j, err; 390 391 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 392 return -EOPNOTSUPP; 393 394 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 395 if (ignore_flow_lvl) 396 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 397 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 398 399 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, 400 esw_attr->dests[j].rep->vport, false); 401 if (IS_ERR(dest[*i].ft)) { 402 err = PTR_ERR(dest[*i].ft); 403 goto err_indir_tbl_get; 404 } 405 } 406 407 if (mlx5_esw_indir_table_decap_vport(attr)) { 408 err = esw_setup_decap_indir(esw, attr); 409 if (err) 410 goto err_indir_tbl_get; 411 } 412 413 return 0; 414 415 err_indir_tbl_get: 416 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 417 return err; 418 } 419 420 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 421 { 422 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 423 424 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 425 esw_cleanup_decap_indir(esw, attr); 426 } 427 428 static void 429 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 430 { 431 mlx5_chains_put_table(chains, chain, prio, level); 432 } 433 434 static void 435 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 436 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 437 int attr_idx, int dest_idx, bool pkt_reformat) 438 { 439 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 440 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport; 441 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 442 dest[dest_idx].vport.vhca_id = 443 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 444 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 445 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 446 mlx5_lag_is_mpesw(esw->dev)) 447 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 448 } 449 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 450 if (pkt_reformat) { 451 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 452 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 453 } 454 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 455 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 456 } 457 } 458 459 static int 460 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 461 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 462 int i) 463 { 464 int j; 465 466 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 467 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 468 return i; 469 } 470 471 static bool 472 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 473 { 474 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 475 mlx5_eswitch_vport_match_metadata_enabled(esw) && 476 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 477 } 478 479 static int 480 esw_setup_dests(struct mlx5_flow_destination *dest, 481 struct mlx5_flow_act *flow_act, 482 struct mlx5_eswitch *esw, 483 struct mlx5_flow_attr *attr, 484 struct mlx5_flow_spec *spec, 485 int *i) 486 { 487 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 488 struct mlx5_fs_chains *chains = esw_chains(esw); 489 int err = 0; 490 491 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 492 esw_src_port_rewrite_supported(esw)) 493 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 494 495 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 496 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 497 (*i)++; 498 goto out; 499 } 500 501 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 502 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 503 (*i)++; 504 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 505 esw_setup_accept_dest(dest, flow_act, chains, *i); 506 (*i)++; 507 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 508 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 509 (*i)++; 510 } else if (esw_is_indir_table(esw, attr)) { 511 err = esw_setup_indir_table(dest, flow_act, esw, attr, true, i); 512 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 513 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 514 } else { 515 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 516 517 if (attr->dest_ft) { 518 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i); 519 (*i)++; 520 } else if (attr->dest_chain) { 521 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 522 1, 0, *i); 523 (*i)++; 524 } 525 } 526 527 out: 528 return err; 529 } 530 531 static void 532 esw_cleanup_dests(struct mlx5_eswitch *esw, 533 struct mlx5_flow_attr *attr) 534 { 535 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 536 struct mlx5_fs_chains *chains = esw_chains(esw); 537 538 if (attr->dest_ft) { 539 esw_cleanup_decap_indir(esw, attr); 540 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 541 if (attr->dest_chain) 542 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 543 else if (esw_is_indir_table(esw, attr)) 544 esw_cleanup_indir_table(esw, attr); 545 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 546 esw_cleanup_chain_src_port_rewrite(esw, attr); 547 } 548 } 549 550 static void 551 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 552 { 553 struct mlx5e_flow_meter_handle *meter; 554 555 meter = attr->meter_attr.meter; 556 flow_act->exe_aso.type = attr->exe_aso_type; 557 flow_act->exe_aso.object_id = meter->obj_id; 558 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 559 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 560 /* use metadata reg 5 for packet color */ 561 flow_act->exe_aso.return_reg_id = 5; 562 } 563 564 struct mlx5_flow_handle * 565 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 566 struct mlx5_flow_spec *spec, 567 struct mlx5_flow_attr *attr) 568 { 569 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 570 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 571 struct mlx5_fs_chains *chains = esw_chains(esw); 572 bool split = !!(esw_attr->split_count); 573 struct mlx5_vport_tbl_attr fwd_attr; 574 struct mlx5_flow_destination *dest; 575 struct mlx5_flow_handle *rule; 576 struct mlx5_flow_table *fdb; 577 int i = 0; 578 579 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 580 return ERR_PTR(-EOPNOTSUPP); 581 582 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 583 return ERR_PTR(-EOPNOTSUPP); 584 585 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 586 if (!dest) 587 return ERR_PTR(-ENOMEM); 588 589 flow_act.action = attr->action; 590 591 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 592 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 593 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 594 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 595 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 596 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 597 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 598 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 599 } 600 } 601 602 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 603 604 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 605 int err; 606 607 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 608 if (err) { 609 rule = ERR_PTR(err); 610 goto err_create_goto_table; 611 } 612 } 613 614 if (esw_attr->decap_pkt_reformat) 615 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 616 617 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 618 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 619 dest[i].counter_id = mlx5_fc_id(attr->counter); 620 i++; 621 } 622 623 if (attr->outer_match_level != MLX5_MATCH_NONE) 624 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 625 if (attr->inner_match_level != MLX5_MATCH_NONE) 626 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 627 628 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 629 flow_act.modify_hdr = attr->modify_hdr; 630 631 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 632 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 633 esw_setup_meter(attr, &flow_act); 634 635 if (split) { 636 fwd_attr.chain = attr->chain; 637 fwd_attr.prio = attr->prio; 638 fwd_attr.vport = esw_attr->in_rep->vport; 639 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 640 641 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 642 } else { 643 if (attr->chain || attr->prio) 644 fdb = mlx5_chains_get_table(chains, attr->chain, 645 attr->prio, 0); 646 else 647 fdb = attr->ft; 648 649 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 650 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 651 esw_attr->in_mdev->priv.eswitch, 652 esw_attr->in_rep->vport); 653 } 654 if (IS_ERR(fdb)) { 655 rule = ERR_CAST(fdb); 656 goto err_esw_get; 657 } 658 659 if (!i) { 660 kfree(dest); 661 dest = NULL; 662 } 663 664 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 665 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 666 &flow_act, dest, i); 667 else 668 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 669 if (IS_ERR(rule)) 670 goto err_add_rule; 671 else 672 atomic64_inc(&esw->offloads.num_flows); 673 674 kfree(dest); 675 return rule; 676 677 err_add_rule: 678 if (split) 679 mlx5_esw_vporttbl_put(esw, &fwd_attr); 680 else if (attr->chain || attr->prio) 681 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 682 err_esw_get: 683 esw_cleanup_dests(esw, attr); 684 err_create_goto_table: 685 kfree(dest); 686 return rule; 687 } 688 689 struct mlx5_flow_handle * 690 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 691 struct mlx5_flow_spec *spec, 692 struct mlx5_flow_attr *attr) 693 { 694 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 695 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 696 struct mlx5_fs_chains *chains = esw_chains(esw); 697 struct mlx5_vport_tbl_attr fwd_attr; 698 struct mlx5_flow_destination *dest; 699 struct mlx5_flow_table *fast_fdb; 700 struct mlx5_flow_table *fwd_fdb; 701 struct mlx5_flow_handle *rule; 702 int i, err = 0; 703 704 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 705 if (!dest) 706 return ERR_PTR(-ENOMEM); 707 708 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 709 if (IS_ERR(fast_fdb)) { 710 rule = ERR_CAST(fast_fdb); 711 goto err_get_fast; 712 } 713 714 fwd_attr.chain = attr->chain; 715 fwd_attr.prio = attr->prio; 716 fwd_attr.vport = esw_attr->in_rep->vport; 717 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 718 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 719 if (IS_ERR(fwd_fdb)) { 720 rule = ERR_CAST(fwd_fdb); 721 goto err_get_fwd; 722 } 723 724 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 725 for (i = 0; i < esw_attr->split_count; i++) { 726 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 727 /* Source port rewrite (forward to ovs internal port or statck device) isn't 728 * supported in the rule of split action. 729 */ 730 err = -EOPNOTSUPP; 731 else 732 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 733 734 if (err) { 735 rule = ERR_PTR(err); 736 goto err_chain_src_rewrite; 737 } 738 } 739 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 740 dest[i].ft = fwd_fdb; 741 i++; 742 743 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 744 esw_attr->in_mdev->priv.eswitch, 745 esw_attr->in_rep->vport); 746 747 if (attr->outer_match_level != MLX5_MATCH_NONE) 748 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 749 750 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 751 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 752 753 if (IS_ERR(rule)) { 754 i = esw_attr->split_count; 755 goto err_chain_src_rewrite; 756 } 757 758 atomic64_inc(&esw->offloads.num_flows); 759 760 kfree(dest); 761 return rule; 762 err_chain_src_rewrite: 763 mlx5_esw_vporttbl_put(esw, &fwd_attr); 764 err_get_fwd: 765 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 766 err_get_fast: 767 kfree(dest); 768 return rule; 769 } 770 771 static void 772 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 773 struct mlx5_flow_handle *rule, 774 struct mlx5_flow_attr *attr, 775 bool fwd_rule) 776 { 777 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 778 struct mlx5_fs_chains *chains = esw_chains(esw); 779 bool split = (esw_attr->split_count > 0); 780 struct mlx5_vport_tbl_attr fwd_attr; 781 int i; 782 783 mlx5_del_flow_rules(rule); 784 785 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 786 /* unref the term table */ 787 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 788 if (esw_attr->dests[i].termtbl) 789 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 790 } 791 } 792 793 atomic64_dec(&esw->offloads.num_flows); 794 795 if (fwd_rule || split) { 796 fwd_attr.chain = attr->chain; 797 fwd_attr.prio = attr->prio; 798 fwd_attr.vport = esw_attr->in_rep->vport; 799 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 800 } 801 802 if (fwd_rule) { 803 mlx5_esw_vporttbl_put(esw, &fwd_attr); 804 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 805 } else { 806 if (split) 807 mlx5_esw_vporttbl_put(esw, &fwd_attr); 808 else if (attr->chain || attr->prio) 809 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 810 esw_cleanup_dests(esw, attr); 811 } 812 } 813 814 void 815 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 816 struct mlx5_flow_handle *rule, 817 struct mlx5_flow_attr *attr) 818 { 819 __mlx5_eswitch_del_rule(esw, rule, attr, false); 820 } 821 822 void 823 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 824 struct mlx5_flow_handle *rule, 825 struct mlx5_flow_attr *attr) 826 { 827 __mlx5_eswitch_del_rule(esw, rule, attr, true); 828 } 829 830 struct mlx5_flow_handle * 831 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 832 struct mlx5_eswitch *from_esw, 833 struct mlx5_eswitch_rep *rep, 834 u32 sqn) 835 { 836 struct mlx5_flow_act flow_act = {0}; 837 struct mlx5_flow_destination dest = {}; 838 struct mlx5_flow_handle *flow_rule; 839 struct mlx5_flow_spec *spec; 840 void *misc; 841 842 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 843 if (!spec) { 844 flow_rule = ERR_PTR(-ENOMEM); 845 goto out; 846 } 847 848 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 849 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 850 /* source vport is the esw manager */ 851 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport); 852 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 853 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 854 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 855 856 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 857 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 858 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 859 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 860 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 861 source_eswitch_owner_vhca_id); 862 863 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 864 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 865 dest.vport.num = rep->vport; 866 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 867 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 868 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 869 870 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && 871 rep->vport == MLX5_VPORT_UPLINK) 872 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 873 874 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 875 spec, &flow_act, &dest, 1); 876 if (IS_ERR(flow_rule)) 877 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", 878 PTR_ERR(flow_rule)); 879 out: 880 kvfree(spec); 881 return flow_rule; 882 } 883 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 884 885 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 886 { 887 mlx5_del_flow_rules(rule); 888 } 889 890 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 891 { 892 if (rule) 893 mlx5_del_flow_rules(rule); 894 } 895 896 struct mlx5_flow_handle * 897 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 898 { 899 struct mlx5_flow_destination dest = {}; 900 struct mlx5_flow_act flow_act = {0}; 901 struct mlx5_flow_handle *flow_rule; 902 struct mlx5_flow_spec *spec; 903 904 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 905 if (!spec) 906 return ERR_PTR(-ENOMEM); 907 908 MLX5_SET(fte_match_param, spec->match_criteria, 909 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 910 MLX5_SET(fte_match_param, spec->match_criteria, 911 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 912 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 913 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 914 915 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 916 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 917 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 918 919 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 920 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 921 dest.vport.num = vport_num; 922 923 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 924 spec, &flow_act, &dest, 1); 925 if (IS_ERR(flow_rule)) 926 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", 927 vport_num, PTR_ERR(flow_rule)); 928 929 kvfree(spec); 930 return flow_rule; 931 } 932 933 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 934 { 935 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 936 MLX5_FDB_TO_VPORT_REG_C_1; 937 } 938 939 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 940 { 941 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 942 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 943 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 944 u8 curr, wanted; 945 int err; 946 947 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 948 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 949 return 0; 950 951 MLX5_SET(query_esw_vport_context_in, in, opcode, 952 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 953 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 954 if (err) 955 return err; 956 957 curr = MLX5_GET(query_esw_vport_context_out, out, 958 esw_vport_context.fdb_to_vport_reg_c_id); 959 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 960 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 961 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 962 963 if (enable) 964 curr |= wanted; 965 else 966 curr &= ~wanted; 967 968 MLX5_SET(modify_esw_vport_context_in, min, 969 esw_vport_context.fdb_to_vport_reg_c_id, curr); 970 MLX5_SET(modify_esw_vport_context_in, min, 971 field_select.fdb_to_vport_reg_c_id, 1); 972 973 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 974 if (!err) { 975 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 976 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 977 else 978 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 979 } 980 981 return err; 982 } 983 984 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 985 struct mlx5_core_dev *peer_dev, 986 struct mlx5_flow_spec *spec, 987 struct mlx5_flow_destination *dest) 988 { 989 void *misc; 990 991 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 992 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 993 misc_parameters_2); 994 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 995 mlx5_eswitch_get_vport_metadata_mask()); 996 997 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 998 } else { 999 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1000 misc_parameters); 1001 1002 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1003 MLX5_CAP_GEN(peer_dev, vhca_id)); 1004 1005 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1006 1007 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1008 misc_parameters); 1009 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1010 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1011 source_eswitch_owner_vhca_id); 1012 } 1013 1014 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1015 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1016 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1017 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1018 } 1019 1020 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1021 struct mlx5_eswitch *peer_esw, 1022 struct mlx5_flow_spec *spec, 1023 u16 vport) 1024 { 1025 void *misc; 1026 1027 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1028 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1029 misc_parameters_2); 1030 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1031 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1032 vport)); 1033 } else { 1034 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1035 misc_parameters); 1036 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1037 } 1038 } 1039 1040 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1041 struct mlx5_core_dev *peer_dev) 1042 { 1043 struct mlx5_flow_destination dest = {}; 1044 struct mlx5_flow_act flow_act = {0}; 1045 struct mlx5_flow_handle **flows; 1046 /* total vports is the same for both e-switches */ 1047 int nvports = esw->total_vports; 1048 struct mlx5_flow_handle *flow; 1049 struct mlx5_flow_spec *spec; 1050 struct mlx5_vport *vport; 1051 unsigned long i; 1052 void *misc; 1053 int err; 1054 1055 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1056 if (!spec) 1057 return -ENOMEM; 1058 1059 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1060 1061 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); 1062 if (!flows) { 1063 err = -ENOMEM; 1064 goto alloc_flows_err; 1065 } 1066 1067 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1068 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1069 misc_parameters); 1070 1071 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1072 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1073 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1074 spec, MLX5_VPORT_PF); 1075 1076 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1077 spec, &flow_act, &dest, 1); 1078 if (IS_ERR(flow)) { 1079 err = PTR_ERR(flow); 1080 goto add_pf_flow_err; 1081 } 1082 flows[vport->index] = flow; 1083 } 1084 1085 if (mlx5_ecpf_vport_exists(esw->dev)) { 1086 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1087 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1088 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1089 spec, &flow_act, &dest, 1); 1090 if (IS_ERR(flow)) { 1091 err = PTR_ERR(flow); 1092 goto add_ecpf_flow_err; 1093 } 1094 flows[vport->index] = flow; 1095 } 1096 1097 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1098 esw_set_peer_miss_rule_source_port(esw, 1099 peer_dev->priv.eswitch, 1100 spec, vport->vport); 1101 1102 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1103 spec, &flow_act, &dest, 1); 1104 if (IS_ERR(flow)) { 1105 err = PTR_ERR(flow); 1106 goto add_vf_flow_err; 1107 } 1108 flows[vport->index] = flow; 1109 } 1110 1111 esw->fdb_table.offloads.peer_miss_rules = flows; 1112 1113 kvfree(spec); 1114 return 0; 1115 1116 add_vf_flow_err: 1117 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1118 if (!flows[vport->index]) 1119 continue; 1120 mlx5_del_flow_rules(flows[vport->index]); 1121 } 1122 if (mlx5_ecpf_vport_exists(esw->dev)) { 1123 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1124 mlx5_del_flow_rules(flows[vport->index]); 1125 } 1126 add_ecpf_flow_err: 1127 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1128 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1129 mlx5_del_flow_rules(flows[vport->index]); 1130 } 1131 add_pf_flow_err: 1132 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1133 kvfree(flows); 1134 alloc_flows_err: 1135 kvfree(spec); 1136 return err; 1137 } 1138 1139 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) 1140 { 1141 struct mlx5_flow_handle **flows; 1142 struct mlx5_vport *vport; 1143 unsigned long i; 1144 1145 flows = esw->fdb_table.offloads.peer_miss_rules; 1146 1147 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) 1148 mlx5_del_flow_rules(flows[vport->index]); 1149 1150 if (mlx5_ecpf_vport_exists(esw->dev)) { 1151 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1152 mlx5_del_flow_rules(flows[vport->index]); 1153 } 1154 1155 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1156 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1157 mlx5_del_flow_rules(flows[vport->index]); 1158 } 1159 kvfree(flows); 1160 } 1161 1162 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1163 { 1164 struct mlx5_flow_act flow_act = {0}; 1165 struct mlx5_flow_destination dest = {}; 1166 struct mlx5_flow_handle *flow_rule = NULL; 1167 struct mlx5_flow_spec *spec; 1168 void *headers_c; 1169 void *headers_v; 1170 int err = 0; 1171 u8 *dmac_c; 1172 u8 *dmac_v; 1173 1174 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1175 if (!spec) { 1176 err = -ENOMEM; 1177 goto out; 1178 } 1179 1180 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1181 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1182 outer_headers); 1183 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1184 outer_headers.dmac_47_16); 1185 dmac_c[0] = 0x01; 1186 1187 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1188 dest.vport.num = esw->manager_vport; 1189 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1190 1191 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1192 spec, &flow_act, &dest, 1); 1193 if (IS_ERR(flow_rule)) { 1194 err = PTR_ERR(flow_rule); 1195 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1196 goto out; 1197 } 1198 1199 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1200 1201 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1202 outer_headers); 1203 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1204 outer_headers.dmac_47_16); 1205 dmac_v[0] = 0x01; 1206 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1207 spec, &flow_act, &dest, 1); 1208 if (IS_ERR(flow_rule)) { 1209 err = PTR_ERR(flow_rule); 1210 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1211 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1212 goto out; 1213 } 1214 1215 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1216 1217 out: 1218 kvfree(spec); 1219 return err; 1220 } 1221 1222 struct mlx5_flow_handle * 1223 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1224 { 1225 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1226 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1227 struct mlx5_flow_context *flow_context; 1228 struct mlx5_flow_handle *flow_rule; 1229 struct mlx5_flow_destination dest; 1230 struct mlx5_flow_spec *spec; 1231 void *misc; 1232 1233 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1234 return ERR_PTR(-EOPNOTSUPP); 1235 1236 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1237 if (!spec) 1238 return ERR_PTR(-ENOMEM); 1239 1240 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1241 misc_parameters_2); 1242 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1243 ESW_REG_C0_USER_DATA_METADATA_MASK); 1244 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1245 misc_parameters_2); 1246 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1247 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1248 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1249 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1250 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1251 1252 flow_context = &spec->flow_context; 1253 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1254 flow_context->flow_tag = tag; 1255 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1256 dest.ft = esw->offloads.ft_offloads; 1257 1258 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1259 kvfree(spec); 1260 1261 if (IS_ERR(flow_rule)) 1262 esw_warn(esw->dev, 1263 "Failed to create restore rule for tag: %d, err(%d)\n", 1264 tag, (int)PTR_ERR(flow_rule)); 1265 1266 return flow_rule; 1267 } 1268 1269 #define MAX_PF_SQ 256 1270 #define MAX_SQ_NVPORTS 32 1271 1272 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1273 u32 *flow_group_in) 1274 { 1275 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1276 flow_group_in, 1277 match_criteria); 1278 1279 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1280 MLX5_SET(create_flow_group_in, flow_group_in, 1281 match_criteria_enable, 1282 MLX5_MATCH_MISC_PARAMETERS_2); 1283 1284 MLX5_SET(fte_match_param, match_criteria, 1285 misc_parameters_2.metadata_reg_c_0, 1286 mlx5_eswitch_get_vport_metadata_mask()); 1287 } else { 1288 MLX5_SET(create_flow_group_in, flow_group_in, 1289 match_criteria_enable, 1290 MLX5_MATCH_MISC_PARAMETERS); 1291 1292 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1293 misc_parameters.source_port); 1294 } 1295 } 1296 1297 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1298 static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1299 { 1300 struct mlx5_vport_tbl_attr attr; 1301 struct mlx5_vport *vport; 1302 unsigned long i; 1303 1304 attr.chain = 0; 1305 attr.prio = 1; 1306 mlx5_esw_for_each_vport(esw, i, vport) { 1307 attr.vport = vport->vport; 1308 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1309 mlx5_esw_vporttbl_put(esw, &attr); 1310 } 1311 } 1312 1313 static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1314 { 1315 struct mlx5_vport_tbl_attr attr; 1316 struct mlx5_flow_table *fdb; 1317 struct mlx5_vport *vport; 1318 unsigned long i; 1319 1320 attr.chain = 0; 1321 attr.prio = 1; 1322 mlx5_esw_for_each_vport(esw, i, vport) { 1323 attr.vport = vport->vport; 1324 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1325 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1326 if (IS_ERR(fdb)) 1327 goto out; 1328 } 1329 return 0; 1330 1331 out: 1332 esw_vport_tbl_put(esw); 1333 return PTR_ERR(fdb); 1334 } 1335 1336 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1337 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1338 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1339 { 1340 struct mlx5_core_dev *dev = esw->dev; 1341 1342 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1343 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1344 1345 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1346 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1347 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1348 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1349 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1350 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1351 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1352 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1353 /* Disabled when ttl workaround is needed, e.g 1354 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1355 */ 1356 esw_warn(dev, 1357 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1358 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1359 } else { 1360 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1361 esw_info(dev, "Supported tc chains and prios offload\n"); 1362 } 1363 1364 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1365 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1366 } 1367 1368 static int 1369 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1370 { 1371 struct mlx5_core_dev *dev = esw->dev; 1372 struct mlx5_flow_table *nf_ft, *ft; 1373 struct mlx5_chains_attr attr = {}; 1374 struct mlx5_fs_chains *chains; 1375 int err; 1376 1377 esw_init_chains_offload_flags(esw, &attr.flags); 1378 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1379 attr.fs_base_prio = FDB_TC_OFFLOAD; 1380 attr.max_grp_num = esw->params.large_group_num; 1381 attr.default_ft = miss_fdb; 1382 attr.mapping = esw->offloads.reg_c0_obj_pool; 1383 1384 chains = mlx5_chains_create(dev, &attr); 1385 if (IS_ERR(chains)) { 1386 err = PTR_ERR(chains); 1387 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1388 return err; 1389 } 1390 mlx5_chains_print_info(chains); 1391 1392 esw->fdb_table.offloads.esw_chains_priv = chains; 1393 1394 /* Create tc_end_ft which is the always created ft chain */ 1395 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1396 1, 0); 1397 if (IS_ERR(nf_ft)) { 1398 err = PTR_ERR(nf_ft); 1399 goto nf_ft_err; 1400 } 1401 1402 /* Always open the root for fast path */ 1403 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1404 if (IS_ERR(ft)) { 1405 err = PTR_ERR(ft); 1406 goto level_0_err; 1407 } 1408 1409 /* Open level 1 for split fdb rules now if prios isn't supported */ 1410 if (!mlx5_chains_prios_supported(chains)) { 1411 err = esw_vport_tbl_get(esw); 1412 if (err) 1413 goto level_1_err; 1414 } 1415 1416 mlx5_chains_set_end_ft(chains, nf_ft); 1417 1418 return 0; 1419 1420 level_1_err: 1421 mlx5_chains_put_table(chains, 0, 1, 0); 1422 level_0_err: 1423 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1424 nf_ft_err: 1425 mlx5_chains_destroy(chains); 1426 esw->fdb_table.offloads.esw_chains_priv = NULL; 1427 1428 return err; 1429 } 1430 1431 static void 1432 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1433 { 1434 if (!mlx5_chains_prios_supported(chains)) 1435 esw_vport_tbl_put(esw); 1436 mlx5_chains_put_table(chains, 0, 1, 0); 1437 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1438 mlx5_chains_destroy(chains); 1439 } 1440 1441 #else /* CONFIG_MLX5_CLS_ACT */ 1442 1443 static int 1444 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1445 { return 0; } 1446 1447 static void 1448 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1449 {} 1450 1451 #endif 1452 1453 static int 1454 esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1455 struct mlx5_flow_table *fdb, 1456 u32 *flow_group_in, 1457 int *ix) 1458 { 1459 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1460 struct mlx5_flow_group *g; 1461 void *match_criteria; 1462 int count, err = 0; 1463 1464 memset(flow_group_in, 0, inlen); 1465 1466 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1467 MLX5_MATCH_MISC_PARAMETERS); 1468 1469 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1470 1471 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1472 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 1473 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1474 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1475 misc_parameters.source_eswitch_owner_vhca_id); 1476 MLX5_SET(create_flow_group_in, flow_group_in, 1477 source_eswitch_owner_vhca_id_valid, 1); 1478 } 1479 1480 /* See comment at table_size calculation */ 1481 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1482 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1483 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1484 *ix += count; 1485 1486 g = mlx5_create_flow_group(fdb, flow_group_in); 1487 if (IS_ERR(g)) { 1488 err = PTR_ERR(g); 1489 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1490 goto out; 1491 } 1492 esw->fdb_table.offloads.send_to_vport_grp = g; 1493 1494 out: 1495 return err; 1496 } 1497 1498 static int 1499 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1500 struct mlx5_flow_table *fdb, 1501 u32 *flow_group_in, 1502 int *ix) 1503 { 1504 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1505 struct mlx5_flow_group *g; 1506 void *match_criteria; 1507 int err = 0; 1508 1509 if (!esw_src_port_rewrite_supported(esw)) 1510 return 0; 1511 1512 memset(flow_group_in, 0, inlen); 1513 1514 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1515 MLX5_MATCH_MISC_PARAMETERS_2); 1516 1517 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1518 1519 MLX5_SET(fte_match_param, match_criteria, 1520 misc_parameters_2.metadata_reg_c_0, 1521 mlx5_eswitch_get_vport_metadata_mask()); 1522 MLX5_SET(fte_match_param, match_criteria, 1523 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1524 1525 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1526 MLX5_SET(create_flow_group_in, flow_group_in, 1527 end_flow_index, *ix + esw->total_vports - 1); 1528 *ix += esw->total_vports; 1529 1530 g = mlx5_create_flow_group(fdb, flow_group_in); 1531 if (IS_ERR(g)) { 1532 err = PTR_ERR(g); 1533 esw_warn(esw->dev, 1534 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1535 goto send_vport_meta_err; 1536 } 1537 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1538 1539 return 0; 1540 1541 send_vport_meta_err: 1542 return err; 1543 } 1544 1545 static int 1546 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1547 struct mlx5_flow_table *fdb, 1548 u32 *flow_group_in, 1549 int *ix) 1550 { 1551 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1552 struct mlx5_flow_group *g; 1553 void *match_criteria; 1554 int err = 0; 1555 1556 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1557 return 0; 1558 1559 memset(flow_group_in, 0, inlen); 1560 1561 esw_set_flow_group_source_port(esw, flow_group_in); 1562 1563 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1564 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1565 flow_group_in, 1566 match_criteria); 1567 1568 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1569 misc_parameters.source_eswitch_owner_vhca_id); 1570 1571 MLX5_SET(create_flow_group_in, flow_group_in, 1572 source_eswitch_owner_vhca_id_valid, 1); 1573 } 1574 1575 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1576 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1577 *ix + esw->total_vports - 1); 1578 *ix += esw->total_vports; 1579 1580 g = mlx5_create_flow_group(fdb, flow_group_in); 1581 if (IS_ERR(g)) { 1582 err = PTR_ERR(g); 1583 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1584 goto out; 1585 } 1586 esw->fdb_table.offloads.peer_miss_grp = g; 1587 1588 out: 1589 return err; 1590 } 1591 1592 static int 1593 esw_create_miss_group(struct mlx5_eswitch *esw, 1594 struct mlx5_flow_table *fdb, 1595 u32 *flow_group_in, 1596 int *ix) 1597 { 1598 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1599 struct mlx5_flow_group *g; 1600 void *match_criteria; 1601 int err = 0; 1602 u8 *dmac; 1603 1604 memset(flow_group_in, 0, inlen); 1605 1606 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1607 MLX5_MATCH_OUTER_HEADERS); 1608 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1609 match_criteria); 1610 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1611 outer_headers.dmac_47_16); 1612 dmac[0] = 0x01; 1613 1614 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1615 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1616 *ix + MLX5_ESW_MISS_FLOWS); 1617 1618 g = mlx5_create_flow_group(fdb, flow_group_in); 1619 if (IS_ERR(g)) { 1620 err = PTR_ERR(g); 1621 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1622 goto miss_err; 1623 } 1624 esw->fdb_table.offloads.miss_grp = g; 1625 1626 err = esw_add_fdb_miss_rule(esw); 1627 if (err) 1628 goto miss_rule_err; 1629 1630 return 0; 1631 1632 miss_rule_err: 1633 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1634 miss_err: 1635 return err; 1636 } 1637 1638 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1639 { 1640 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1641 struct mlx5_flow_table_attr ft_attr = {}; 1642 struct mlx5_core_dev *dev = esw->dev; 1643 struct mlx5_flow_namespace *root_ns; 1644 struct mlx5_flow_table *fdb = NULL; 1645 int table_size, ix = 0, err = 0; 1646 u32 flags = 0, *flow_group_in; 1647 1648 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1649 1650 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1651 if (!flow_group_in) 1652 return -ENOMEM; 1653 1654 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1655 if (!root_ns) { 1656 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1657 err = -EOPNOTSUPP; 1658 goto ns_err; 1659 } 1660 esw->fdb_table.offloads.ns = root_ns; 1661 err = mlx5_flow_namespace_set_mode(root_ns, 1662 esw->dev->priv.steering->mode); 1663 if (err) { 1664 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1665 goto ns_err; 1666 } 1667 1668 /* To be strictly correct: 1669 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1670 * should be: 1671 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1672 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1673 * but as the peer device might not be in switchdev mode it's not 1674 * possible. We use the fact that by default FW sets max vfs and max sfs 1675 * to the same value on both devices. If it needs to be changed in the future note 1676 * the peer miss group should also be created based on the number of 1677 * total vports of the peer (currently is also uses esw->total_vports). 1678 */ 1679 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1680 esw->total_vports * 2 + MLX5_ESW_MISS_FLOWS; 1681 1682 /* create the slow path fdb with encap set, so further table instances 1683 * can be created at run time while VFs are probed if the FW allows that. 1684 */ 1685 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1686 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1687 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1688 1689 ft_attr.flags = flags; 1690 ft_attr.max_fte = table_size; 1691 ft_attr.prio = FDB_SLOW_PATH; 1692 1693 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1694 if (IS_ERR(fdb)) { 1695 err = PTR_ERR(fdb); 1696 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1697 goto slow_fdb_err; 1698 } 1699 esw->fdb_table.offloads.slow_fdb = fdb; 1700 1701 /* Create empty TC-miss managed table. This allows plugging in following 1702 * priorities without directly exposing their level 0 table to 1703 * eswitch_offloads and passing it as miss_fdb to following call to 1704 * esw_chains_create(). 1705 */ 1706 memset(&ft_attr, 0, sizeof(ft_attr)); 1707 ft_attr.prio = FDB_TC_MISS; 1708 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1709 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1710 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1711 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1712 goto tc_miss_table_err; 1713 } 1714 1715 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1716 if (err) { 1717 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1718 goto fdb_chains_err; 1719 } 1720 1721 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1722 if (err) 1723 goto send_vport_err; 1724 1725 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1726 if (err) 1727 goto send_vport_meta_err; 1728 1729 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 1730 if (err) 1731 goto peer_miss_err; 1732 1733 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 1734 if (err) 1735 goto miss_err; 1736 1737 kvfree(flow_group_in); 1738 return 0; 1739 1740 miss_err: 1741 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1742 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1743 peer_miss_err: 1744 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1745 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1746 send_vport_meta_err: 1747 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1748 send_vport_err: 1749 esw_chains_destroy(esw, esw_chains(esw)); 1750 fdb_chains_err: 1751 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1752 tc_miss_table_err: 1753 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1754 slow_fdb_err: 1755 /* Holds true only as long as DMFS is the default */ 1756 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1757 ns_err: 1758 kvfree(flow_group_in); 1759 return err; 1760 } 1761 1762 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1763 { 1764 if (!mlx5_eswitch_get_slow_fdb(esw)) 1765 return; 1766 1767 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1768 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1769 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1770 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1771 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1772 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1773 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1774 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1775 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1776 1777 esw_chains_destroy(esw, esw_chains(esw)); 1778 1779 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1780 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1781 /* Holds true only as long as DMFS is the default */ 1782 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1783 MLX5_FLOW_STEERING_MODE_DMFS); 1784 atomic64_set(&esw->user_count, 0); 1785 } 1786 1787 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 1788 { 1789 int nvports; 1790 1791 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1792 if (mlx5e_tc_int_port_supported(esw)) 1793 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 1794 1795 return nvports; 1796 } 1797 1798 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1799 { 1800 struct mlx5_flow_table_attr ft_attr = {}; 1801 struct mlx5_core_dev *dev = esw->dev; 1802 struct mlx5_flow_table *ft_offloads; 1803 struct mlx5_flow_namespace *ns; 1804 int err = 0; 1805 1806 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1807 if (!ns) { 1808 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1809 return -EOPNOTSUPP; 1810 } 1811 1812 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 1813 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 1814 ft_attr.prio = 1; 1815 1816 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 1817 if (IS_ERR(ft_offloads)) { 1818 err = PTR_ERR(ft_offloads); 1819 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 1820 return err; 1821 } 1822 1823 esw->offloads.ft_offloads = ft_offloads; 1824 return 0; 1825 } 1826 1827 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 1828 { 1829 struct mlx5_esw_offload *offloads = &esw->offloads; 1830 1831 mlx5_destroy_flow_table(offloads->ft_offloads); 1832 } 1833 1834 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 1835 { 1836 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1837 struct mlx5_flow_group *g; 1838 u32 *flow_group_in; 1839 int nvports; 1840 int err = 0; 1841 1842 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 1843 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1844 if (!flow_group_in) 1845 return -ENOMEM; 1846 1847 /* create vport rx group */ 1848 esw_set_flow_group_source_port(esw, flow_group_in); 1849 1850 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1851 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 1852 1853 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 1854 1855 if (IS_ERR(g)) { 1856 err = PTR_ERR(g); 1857 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 1858 goto out; 1859 } 1860 1861 esw->offloads.vport_rx_group = g; 1862 out: 1863 kvfree(flow_group_in); 1864 return err; 1865 } 1866 1867 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 1868 { 1869 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 1870 } 1871 1872 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 1873 { 1874 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 1875 * for the drop rule, which is placed at the end of the table. 1876 * So return the total of vport and int_port as rule index. 1877 */ 1878 return esw_get_nr_ft_offloads_steering_src_ports(esw); 1879 } 1880 1881 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 1882 { 1883 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1884 struct mlx5_flow_group *g; 1885 u32 *flow_group_in; 1886 int flow_index; 1887 int err = 0; 1888 1889 flow_index = esw_create_vport_rx_drop_rule_index(esw); 1890 1891 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1892 if (!flow_group_in) 1893 return -ENOMEM; 1894 1895 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 1896 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 1897 1898 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 1899 1900 if (IS_ERR(g)) { 1901 err = PTR_ERR(g); 1902 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); 1903 goto out; 1904 } 1905 1906 esw->offloads.vport_rx_drop_group = g; 1907 out: 1908 kvfree(flow_group_in); 1909 return err; 1910 } 1911 1912 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 1913 { 1914 if (esw->offloads.vport_rx_drop_group) 1915 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 1916 } 1917 1918 struct mlx5_flow_handle * 1919 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 1920 struct mlx5_flow_destination *dest) 1921 { 1922 struct mlx5_flow_act flow_act = {0}; 1923 struct mlx5_flow_handle *flow_rule; 1924 struct mlx5_flow_spec *spec; 1925 void *misc; 1926 1927 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1928 if (!spec) { 1929 flow_rule = ERR_PTR(-ENOMEM); 1930 goto out; 1931 } 1932 1933 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1934 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 1935 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1936 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 1937 1938 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 1939 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1940 mlx5_eswitch_get_vport_metadata_mask()); 1941 1942 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1943 } else { 1944 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1945 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1946 1947 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1948 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1949 1950 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1951 } 1952 1953 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1954 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 1955 &flow_act, dest, 1); 1956 if (IS_ERR(flow_rule)) { 1957 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 1958 goto out; 1959 } 1960 1961 out: 1962 kvfree(spec); 1963 return flow_rule; 1964 } 1965 1966 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 1967 { 1968 struct mlx5_flow_act flow_act = {}; 1969 struct mlx5_flow_handle *flow_rule; 1970 1971 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 1972 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 1973 &flow_act, NULL, 0); 1974 if (IS_ERR(flow_rule)) { 1975 esw_warn(esw->dev, 1976 "fs offloads: Failed to add vport rx drop rule err %ld\n", 1977 PTR_ERR(flow_rule)); 1978 return PTR_ERR(flow_rule); 1979 } 1980 1981 esw->offloads.vport_rx_drop_rule = flow_rule; 1982 1983 return 0; 1984 } 1985 1986 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 1987 { 1988 if (esw->offloads.vport_rx_drop_rule) 1989 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 1990 } 1991 1992 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 1993 { 1994 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 1995 struct mlx5_core_dev *dev = esw->dev; 1996 struct mlx5_vport *vport; 1997 unsigned long i; 1998 1999 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2000 return -EOPNOTSUPP; 2001 2002 if (!mlx5_esw_is_fdb_created(esw)) 2003 return -EOPNOTSUPP; 2004 2005 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2006 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2007 mlx5_mode = MLX5_INLINE_MODE_NONE; 2008 goto out; 2009 case MLX5_CAP_INLINE_MODE_L2: 2010 mlx5_mode = MLX5_INLINE_MODE_L2; 2011 goto out; 2012 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2013 goto query_vports; 2014 } 2015 2016 query_vports: 2017 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2018 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2019 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2020 if (prev_mlx5_mode != mlx5_mode) 2021 return -EINVAL; 2022 prev_mlx5_mode = mlx5_mode; 2023 } 2024 2025 out: 2026 *mode = mlx5_mode; 2027 return 0; 2028 } 2029 2030 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2031 { 2032 struct mlx5_esw_offload *offloads = &esw->offloads; 2033 2034 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2035 return; 2036 2037 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2038 mlx5_destroy_flow_group(offloads->restore_group); 2039 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2040 } 2041 2042 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2043 { 2044 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2045 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2046 struct mlx5_flow_table_attr ft_attr = {}; 2047 struct mlx5_core_dev *dev = esw->dev; 2048 struct mlx5_flow_namespace *ns; 2049 struct mlx5_modify_hdr *mod_hdr; 2050 void *match_criteria, *misc; 2051 struct mlx5_flow_table *ft; 2052 struct mlx5_flow_group *g; 2053 u32 *flow_group_in; 2054 int err = 0; 2055 2056 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2057 return 0; 2058 2059 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2060 if (!ns) { 2061 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2062 return -EOPNOTSUPP; 2063 } 2064 2065 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2066 if (!flow_group_in) { 2067 err = -ENOMEM; 2068 goto out_free; 2069 } 2070 2071 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2072 ft = mlx5_create_flow_table(ns, &ft_attr); 2073 if (IS_ERR(ft)) { 2074 err = PTR_ERR(ft); 2075 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2076 err); 2077 goto out_free; 2078 } 2079 2080 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2081 match_criteria); 2082 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2083 misc_parameters_2); 2084 2085 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2086 ESW_REG_C0_USER_DATA_METADATA_MASK); 2087 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2088 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2089 ft_attr.max_fte - 1); 2090 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2091 MLX5_MATCH_MISC_PARAMETERS_2); 2092 g = mlx5_create_flow_group(ft, flow_group_in); 2093 if (IS_ERR(g)) { 2094 err = PTR_ERR(g); 2095 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2096 err); 2097 goto err_group; 2098 } 2099 2100 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2101 MLX5_SET(copy_action_in, modact, src_field, 2102 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2103 MLX5_SET(copy_action_in, modact, dst_field, 2104 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2105 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2106 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2107 modact); 2108 if (IS_ERR(mod_hdr)) { 2109 err = PTR_ERR(mod_hdr); 2110 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2111 err); 2112 goto err_mod_hdr; 2113 } 2114 2115 esw->offloads.ft_offloads_restore = ft; 2116 esw->offloads.restore_group = g; 2117 esw->offloads.restore_copy_hdr_id = mod_hdr; 2118 2119 kvfree(flow_group_in); 2120 2121 return 0; 2122 2123 err_mod_hdr: 2124 mlx5_destroy_flow_group(g); 2125 err_group: 2126 mlx5_destroy_flow_table(ft); 2127 out_free: 2128 kvfree(flow_group_in); 2129 2130 return err; 2131 } 2132 2133 static int esw_offloads_start(struct mlx5_eswitch *esw, 2134 struct netlink_ext_ack *extack) 2135 { 2136 int err; 2137 2138 esw->mode = MLX5_ESWITCH_OFFLOADS; 2139 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2140 if (err) { 2141 NL_SET_ERR_MSG_MOD(extack, 2142 "Failed setting eswitch to offloads"); 2143 esw->mode = MLX5_ESWITCH_LEGACY; 2144 mlx5_rescan_drivers(esw->dev); 2145 } 2146 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2147 if (mlx5_eswitch_inline_mode_get(esw, 2148 &esw->offloads.inline_mode)) { 2149 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2150 NL_SET_ERR_MSG_MOD(extack, 2151 "Inline mode is different between vports"); 2152 } 2153 } 2154 return err; 2155 } 2156 2157 static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw, 2158 struct mlx5_eswitch_rep *rep, 2159 xa_mark_t mark) 2160 { 2161 bool mark_set; 2162 2163 /* Copy the mark from vport to its rep */ 2164 mark_set = xa_get_mark(&esw->vports, rep->vport, mark); 2165 if (mark_set) 2166 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark); 2167 } 2168 2169 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) 2170 { 2171 struct mlx5_eswitch_rep *rep; 2172 int rep_type; 2173 int err; 2174 2175 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 2176 if (!rep) 2177 return -ENOMEM; 2178 2179 rep->vport = vport->vport; 2180 rep->vport_index = vport->index; 2181 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2182 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2183 2184 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2185 if (err) 2186 goto insert_err; 2187 2188 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN); 2189 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF); 2190 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF); 2191 return 0; 2192 2193 insert_err: 2194 kfree(rep); 2195 return err; 2196 } 2197 2198 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2199 struct mlx5_eswitch_rep *rep) 2200 { 2201 xa_erase(&esw->offloads.vport_reps, rep->vport); 2202 kfree(rep); 2203 } 2204 2205 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2206 { 2207 struct mlx5_eswitch_rep *rep; 2208 unsigned long i; 2209 2210 mlx5_esw_for_each_rep(esw, i, rep) 2211 mlx5_esw_offloads_rep_cleanup(esw, rep); 2212 xa_destroy(&esw->offloads.vport_reps); 2213 } 2214 2215 static int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2216 { 2217 struct mlx5_vport *vport; 2218 unsigned long i; 2219 int err; 2220 2221 xa_init(&esw->offloads.vport_reps); 2222 2223 mlx5_esw_for_each_vport(esw, i, vport) { 2224 err = mlx5_esw_offloads_rep_init(esw, vport); 2225 if (err) 2226 goto err; 2227 } 2228 return 0; 2229 2230 err: 2231 esw_offloads_cleanup_reps(esw); 2232 return err; 2233 } 2234 2235 static int esw_port_metadata_set(struct devlink *devlink, u32 id, 2236 struct devlink_param_gset_ctx *ctx) 2237 { 2238 struct mlx5_core_dev *dev = devlink_priv(devlink); 2239 struct mlx5_eswitch *esw = dev->priv.eswitch; 2240 int err = 0; 2241 2242 down_write(&esw->mode_lock); 2243 if (mlx5_esw_is_fdb_created(esw)) { 2244 err = -EBUSY; 2245 goto done; 2246 } 2247 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2248 err = -EOPNOTSUPP; 2249 goto done; 2250 } 2251 if (ctx->val.vbool) 2252 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2253 else 2254 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2255 done: 2256 up_write(&esw->mode_lock); 2257 return err; 2258 } 2259 2260 static int esw_port_metadata_get(struct devlink *devlink, u32 id, 2261 struct devlink_param_gset_ctx *ctx) 2262 { 2263 struct mlx5_core_dev *dev = devlink_priv(devlink); 2264 2265 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch); 2266 return 0; 2267 } 2268 2269 static int esw_port_metadata_validate(struct devlink *devlink, u32 id, 2270 union devlink_param_value val, 2271 struct netlink_ext_ack *extack) 2272 { 2273 struct mlx5_core_dev *dev = devlink_priv(devlink); 2274 u8 esw_mode; 2275 2276 esw_mode = mlx5_eswitch_mode(dev); 2277 if (esw_mode == MLX5_ESWITCH_OFFLOADS) { 2278 NL_SET_ERR_MSG_MOD(extack, 2279 "E-Switch must either disabled or non switchdev mode"); 2280 return -EBUSY; 2281 } 2282 return 0; 2283 } 2284 2285 static const struct devlink_param esw_devlink_params[] = { 2286 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA, 2287 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL, 2288 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 2289 esw_port_metadata_get, 2290 esw_port_metadata_set, 2291 esw_port_metadata_validate), 2292 }; 2293 2294 int esw_offloads_init(struct mlx5_eswitch *esw) 2295 { 2296 int err; 2297 2298 err = esw_offloads_init_reps(esw); 2299 if (err) 2300 return err; 2301 2302 err = devl_params_register(priv_to_devlink(esw->dev), 2303 esw_devlink_params, 2304 ARRAY_SIZE(esw_devlink_params)); 2305 if (err) 2306 goto err_params; 2307 2308 return 0; 2309 2310 err_params: 2311 esw_offloads_cleanup_reps(esw); 2312 return err; 2313 } 2314 2315 void esw_offloads_cleanup(struct mlx5_eswitch *esw) 2316 { 2317 devl_params_unregister(priv_to_devlink(esw->dev), 2318 esw_devlink_params, 2319 ARRAY_SIZE(esw_devlink_params)); 2320 esw_offloads_cleanup_reps(esw); 2321 } 2322 2323 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2324 struct mlx5_eswitch_rep *rep, u8 rep_type) 2325 { 2326 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2327 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2328 esw->offloads.rep_ops[rep_type]->unload(rep); 2329 } 2330 2331 static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type) 2332 { 2333 struct mlx5_eswitch_rep *rep; 2334 unsigned long i; 2335 2336 mlx5_esw_for_each_sf_rep(esw, i, rep) 2337 __esw_offloads_unload_rep(esw, rep, rep_type); 2338 } 2339 2340 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2341 { 2342 struct mlx5_eswitch_rep *rep; 2343 unsigned long i; 2344 2345 __unload_reps_sf_vport(esw, rep_type); 2346 2347 mlx5_esw_for_each_vf_rep(esw, i, rep) 2348 __esw_offloads_unload_rep(esw, rep, rep_type); 2349 2350 if (mlx5_ecpf_vport_exists(esw->dev)) { 2351 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF); 2352 __esw_offloads_unload_rep(esw, rep, rep_type); 2353 } 2354 2355 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 2356 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF); 2357 __esw_offloads_unload_rep(esw, rep, rep_type); 2358 } 2359 2360 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 2361 __esw_offloads_unload_rep(esw, rep, rep_type); 2362 } 2363 2364 int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2365 { 2366 struct mlx5_eswitch_rep *rep; 2367 int rep_type; 2368 int err; 2369 2370 rep = mlx5_eswitch_get_rep(esw, vport_num); 2371 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2372 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2373 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 2374 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2375 if (err) 2376 goto err_reps; 2377 } 2378 2379 return 0; 2380 2381 err_reps: 2382 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2383 for (--rep_type; rep_type >= 0; rep_type--) 2384 __esw_offloads_unload_rep(esw, rep, rep_type); 2385 return err; 2386 } 2387 2388 void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2389 { 2390 struct mlx5_eswitch_rep *rep; 2391 int rep_type; 2392 2393 rep = mlx5_eswitch_get_rep(esw, vport_num); 2394 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2395 __esw_offloads_unload_rep(esw, rep, rep_type); 2396 } 2397 2398 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) 2399 { 2400 int err; 2401 2402 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2403 return 0; 2404 2405 if (vport_num != MLX5_VPORT_UPLINK) { 2406 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num); 2407 if (err) 2408 return err; 2409 } 2410 2411 err = mlx5_esw_offloads_rep_load(esw, vport_num); 2412 if (err) 2413 goto load_err; 2414 return err; 2415 2416 load_err: 2417 if (vport_num != MLX5_VPORT_UPLINK) 2418 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2419 return err; 2420 } 2421 2422 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) 2423 { 2424 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2425 return; 2426 2427 mlx5_esw_offloads_rep_unload(esw, vport_num); 2428 2429 if (vport_num != MLX5_VPORT_UPLINK) 2430 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 2431 } 2432 2433 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2434 struct mlx5_core_dev *slave) 2435 { 2436 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2437 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2438 struct mlx5_flow_root_namespace *root; 2439 struct mlx5_flow_namespace *ns; 2440 int err; 2441 2442 MLX5_SET(set_flow_table_root_in, in, opcode, 2443 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2444 MLX5_SET(set_flow_table_root_in, in, table_type, 2445 FS_FT_FDB); 2446 2447 if (master) { 2448 ns = mlx5_get_flow_namespace(master, 2449 MLX5_FLOW_NAMESPACE_FDB); 2450 root = find_root(&ns->node); 2451 mutex_lock(&root->chain_lock); 2452 MLX5_SET(set_flow_table_root_in, in, 2453 table_eswitch_owner_vhca_id_valid, 1); 2454 MLX5_SET(set_flow_table_root_in, in, 2455 table_eswitch_owner_vhca_id, 2456 MLX5_CAP_GEN(master, vhca_id)); 2457 MLX5_SET(set_flow_table_root_in, in, table_id, 2458 root->root_ft->id); 2459 } else { 2460 ns = mlx5_get_flow_namespace(slave, 2461 MLX5_FLOW_NAMESPACE_FDB); 2462 root = find_root(&ns->node); 2463 mutex_lock(&root->chain_lock); 2464 MLX5_SET(set_flow_table_root_in, in, table_id, 2465 root->root_ft->id); 2466 } 2467 2468 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2469 mutex_unlock(&root->chain_lock); 2470 2471 return err; 2472 } 2473 2474 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2475 struct mlx5_core_dev *slave, 2476 struct mlx5_vport *vport, 2477 struct mlx5_flow_table *acl) 2478 { 2479 struct mlx5_flow_handle *flow_rule = NULL; 2480 struct mlx5_flow_destination dest = {}; 2481 struct mlx5_flow_act flow_act = {}; 2482 struct mlx5_flow_spec *spec; 2483 int err = 0; 2484 void *misc; 2485 2486 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2487 if (!spec) 2488 return -ENOMEM; 2489 2490 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2491 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2492 misc_parameters); 2493 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2494 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 2495 MLX5_CAP_GEN(slave, vhca_id)); 2496 2497 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2498 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2499 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2500 source_eswitch_owner_vhca_id); 2501 2502 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2503 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2504 dest.vport.num = slave->priv.eswitch->manager_vport; 2505 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2506 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2507 2508 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2509 &dest, 1); 2510 if (IS_ERR(flow_rule)) 2511 err = PTR_ERR(flow_rule); 2512 else 2513 vport->egress.offloads.bounce_rule = flow_rule; 2514 2515 kvfree(spec); 2516 return err; 2517 } 2518 2519 static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 2520 struct mlx5_core_dev *slave) 2521 { 2522 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2523 struct mlx5_eswitch *esw = master->priv.eswitch; 2524 struct mlx5_flow_table_attr ft_attr = { 2525 .max_fte = 1, .prio = 0, .level = 0, 2526 .flags = MLX5_FLOW_TABLE_OTHER_VPORT, 2527 }; 2528 struct mlx5_flow_namespace *egress_ns; 2529 struct mlx5_flow_table *acl; 2530 struct mlx5_flow_group *g; 2531 struct mlx5_vport *vport; 2532 void *match_criteria; 2533 u32 *flow_group_in; 2534 int err; 2535 2536 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 2537 if (IS_ERR(vport)) 2538 return PTR_ERR(vport); 2539 2540 egress_ns = mlx5_get_flow_vport_acl_namespace(master, 2541 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 2542 vport->index); 2543 if (!egress_ns) 2544 return -EINVAL; 2545 2546 if (vport->egress.acl) 2547 return -EINVAL; 2548 2549 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2550 if (!flow_group_in) 2551 return -ENOMEM; 2552 2553 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2554 if (IS_ERR(acl)) { 2555 err = PTR_ERR(acl); 2556 goto out; 2557 } 2558 2559 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2560 match_criteria); 2561 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2562 misc_parameters.source_port); 2563 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2564 misc_parameters.source_eswitch_owner_vhca_id); 2565 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2566 MLX5_MATCH_MISC_PARAMETERS); 2567 2568 MLX5_SET(create_flow_group_in, flow_group_in, 2569 source_eswitch_owner_vhca_id_valid, 1); 2570 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2571 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0); 2572 2573 g = mlx5_create_flow_group(acl, flow_group_in); 2574 if (IS_ERR(g)) { 2575 err = PTR_ERR(g); 2576 goto err_group; 2577 } 2578 2579 err = __esw_set_master_egress_rule(master, slave, vport, acl); 2580 if (err) 2581 goto err_rule; 2582 2583 vport->egress.acl = acl; 2584 vport->egress.offloads.bounce_grp = g; 2585 2586 kvfree(flow_group_in); 2587 2588 return 0; 2589 2590 err_rule: 2591 mlx5_destroy_flow_group(g); 2592 err_group: 2593 mlx5_destroy_flow_table(acl); 2594 out: 2595 kvfree(flow_group_in); 2596 return err; 2597 } 2598 2599 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev) 2600 { 2601 struct mlx5_vport *vport; 2602 2603 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 2604 dev->priv.eswitch->manager_vport); 2605 2606 esw_acl_egress_ofld_cleanup(vport); 2607 } 2608 2609 int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw, 2610 struct mlx5_eswitch *slave_esw) 2611 { 2612 int err; 2613 2614 err = esw_set_slave_root_fdb(master_esw->dev, 2615 slave_esw->dev); 2616 if (err) 2617 return err; 2618 2619 err = esw_set_master_egress_rule(master_esw->dev, 2620 slave_esw->dev); 2621 if (err) 2622 goto err_acl; 2623 2624 return err; 2625 2626 err_acl: 2627 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2628 2629 return err; 2630 } 2631 2632 void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw, 2633 struct mlx5_eswitch *slave_esw) 2634 { 2635 esw_unset_master_egress_rule(master_esw->dev); 2636 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2637 } 2638 2639 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 2640 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2641 2642 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw) 2643 { 2644 const struct mlx5_eswitch_rep_ops *ops; 2645 struct mlx5_eswitch_rep *rep; 2646 unsigned long i; 2647 u8 rep_type; 2648 2649 mlx5_esw_for_each_rep(esw, i, rep) { 2650 rep_type = NUM_REP_TYPES; 2651 while (rep_type--) { 2652 ops = esw->offloads.rep_ops[rep_type]; 2653 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2654 ops->event) 2655 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL); 2656 } 2657 } 2658 } 2659 2660 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) 2661 { 2662 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2663 mlx5e_tc_clean_fdb_peer_flows(esw); 2664 #endif 2665 mlx5_esw_offloads_rep_event_unpair(esw); 2666 esw_del_fdb_peer_miss_rules(esw); 2667 } 2668 2669 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2670 struct mlx5_eswitch *peer_esw) 2671 { 2672 const struct mlx5_eswitch_rep_ops *ops; 2673 struct mlx5_eswitch_rep *rep; 2674 unsigned long i; 2675 u8 rep_type; 2676 int err; 2677 2678 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2679 if (err) 2680 return err; 2681 2682 mlx5_esw_for_each_rep(esw, i, rep) { 2683 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2684 ops = esw->offloads.rep_ops[rep_type]; 2685 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2686 ops->event) { 2687 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 2688 if (err) 2689 goto err_out; 2690 } 2691 } 2692 } 2693 2694 return 0; 2695 2696 err_out: 2697 mlx5_esw_offloads_unpair(esw); 2698 return err; 2699 } 2700 2701 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2702 struct mlx5_eswitch *peer_esw, 2703 bool pair) 2704 { 2705 struct mlx5_flow_root_namespace *peer_ns; 2706 struct mlx5_flow_root_namespace *ns; 2707 int err; 2708 2709 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2710 ns = esw->dev->priv.steering->fdb_root_ns; 2711 2712 if (pair) { 2713 err = mlx5_flow_namespace_set_peer(ns, peer_ns); 2714 if (err) 2715 return err; 2716 2717 err = mlx5_flow_namespace_set_peer(peer_ns, ns); 2718 if (err) { 2719 mlx5_flow_namespace_set_peer(ns, NULL); 2720 return err; 2721 } 2722 } else { 2723 mlx5_flow_namespace_set_peer(ns, NULL); 2724 mlx5_flow_namespace_set_peer(peer_ns, NULL); 2725 } 2726 2727 return 0; 2728 } 2729 2730 static int mlx5_esw_offloads_devcom_event(int event, 2731 void *my_data, 2732 void *event_data) 2733 { 2734 struct mlx5_eswitch *esw = my_data; 2735 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2736 struct mlx5_eswitch *peer_esw = event_data; 2737 int err; 2738 2739 switch (event) { 2740 case ESW_OFFLOADS_DEVCOM_PAIR: 2741 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2742 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2743 break; 2744 2745 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2746 if (err) 2747 goto err_out; 2748 err = mlx5_esw_offloads_pair(esw, peer_esw); 2749 if (err) 2750 goto err_peer; 2751 2752 err = mlx5_esw_offloads_pair(peer_esw, esw); 2753 if (err) 2754 goto err_pair; 2755 2756 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); 2757 break; 2758 2759 case ESW_OFFLOADS_DEVCOM_UNPAIR: 2760 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) 2761 break; 2762 2763 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); 2764 mlx5_esw_offloads_unpair(peer_esw); 2765 mlx5_esw_offloads_unpair(esw); 2766 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2767 break; 2768 } 2769 2770 return 0; 2771 2772 err_pair: 2773 mlx5_esw_offloads_unpair(esw); 2774 err_peer: 2775 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 2776 err_out: 2777 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 2778 event, err); 2779 return err; 2780 } 2781 2782 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw) 2783 { 2784 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2785 2786 INIT_LIST_HEAD(&esw->offloads.peer_flows); 2787 mutex_init(&esw->offloads.peer_mutex); 2788 2789 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2790 return; 2791 2792 if (!mlx5_is_lag_supported(esw->dev)) 2793 return; 2794 2795 mlx5_devcom_register_component(devcom, 2796 MLX5_DEVCOM_ESW_OFFLOADS, 2797 mlx5_esw_offloads_devcom_event, 2798 esw); 2799 2800 mlx5_devcom_send_event(devcom, 2801 MLX5_DEVCOM_ESW_OFFLOADS, 2802 ESW_OFFLOADS_DEVCOM_PAIR, esw); 2803 } 2804 2805 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 2806 { 2807 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2808 2809 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2810 return; 2811 2812 if (!mlx5_is_lag_supported(esw->dev)) 2813 return; 2814 2815 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, 2816 ESW_OFFLOADS_DEVCOM_UNPAIR, esw); 2817 2818 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); 2819 } 2820 2821 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 2822 { 2823 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 2824 return false; 2825 2826 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 2827 MLX5_FDB_TO_VPORT_REG_C_0)) 2828 return false; 2829 2830 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 2831 return false; 2832 2833 return true; 2834 } 2835 2836 #define MLX5_ESW_METADATA_RSVD_UPLINK 1 2837 2838 /* Share the same metadata for uplink's. This is fine because: 2839 * (a) In shared FDB mode (LAG) both uplink's are treated the 2840 * same and tagged with the same metadata. 2841 * (b) In non shared FDB mode, packets from physical port0 2842 * cannot hit eswitch of PF1 and vice versa. 2843 */ 2844 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 2845 { 2846 return MLX5_ESW_METADATA_RSVD_UPLINK; 2847 } 2848 2849 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 2850 { 2851 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 2852 /* Reserve 0xf for internal port offload */ 2853 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 2854 u32 pf_num; 2855 int id; 2856 2857 /* Only 4 bits of pf_num */ 2858 pf_num = mlx5_get_dev_index(esw->dev); 2859 if (pf_num > max_pf_num) 2860 return 0; 2861 2862 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 2863 /* Use only non-zero vport_id (2-4095) for all PF's */ 2864 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 2865 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 2866 vport_end_ida, GFP_KERNEL); 2867 if (id < 0) 2868 return 0; 2869 id = (pf_num << ESW_VPORT_BITS) | id; 2870 return id; 2871 } 2872 2873 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 2874 { 2875 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 2876 2877 /* Metadata contains only 12 bits of actual ida id */ 2878 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 2879 } 2880 2881 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 2882 struct mlx5_vport *vport) 2883 { 2884 if (vport->vport == MLX5_VPORT_UPLINK) 2885 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 2886 else 2887 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 2888 2889 vport->metadata = vport->default_metadata; 2890 return vport->metadata ? 0 : -ENOSPC; 2891 } 2892 2893 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 2894 struct mlx5_vport *vport) 2895 { 2896 if (!vport->default_metadata) 2897 return; 2898 2899 if (vport->vport == MLX5_VPORT_UPLINK) 2900 return; 2901 2902 WARN_ON(vport->metadata != vport->default_metadata); 2903 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 2904 } 2905 2906 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 2907 { 2908 struct mlx5_vport *vport; 2909 unsigned long i; 2910 2911 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2912 return; 2913 2914 mlx5_esw_for_each_vport(esw, i, vport) 2915 esw_offloads_vport_metadata_cleanup(esw, vport); 2916 } 2917 2918 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 2919 { 2920 struct mlx5_vport *vport; 2921 unsigned long i; 2922 int err; 2923 2924 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2925 return 0; 2926 2927 mlx5_esw_for_each_vport(esw, i, vport) { 2928 err = esw_offloads_vport_metadata_setup(esw, vport); 2929 if (err) 2930 goto metadata_err; 2931 } 2932 2933 return 0; 2934 2935 metadata_err: 2936 esw_offloads_metadata_uninit(esw); 2937 return err; 2938 } 2939 2940 int 2941 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 2942 struct mlx5_vport *vport) 2943 { 2944 int err; 2945 2946 err = esw_acl_ingress_ofld_setup(esw, vport); 2947 if (err) 2948 return err; 2949 2950 err = esw_acl_egress_ofld_setup(esw, vport); 2951 if (err) 2952 goto egress_err; 2953 2954 return 0; 2955 2956 egress_err: 2957 esw_acl_ingress_ofld_cleanup(esw, vport); 2958 return err; 2959 } 2960 2961 void 2962 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 2963 struct mlx5_vport *vport) 2964 { 2965 esw_acl_egress_ofld_cleanup(vport); 2966 esw_acl_ingress_ofld_cleanup(esw, vport); 2967 } 2968 2969 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2970 { 2971 struct mlx5_vport *vport; 2972 2973 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2974 if (IS_ERR(vport)) 2975 return PTR_ERR(vport); 2976 2977 return esw_vport_create_offloads_acl_tables(esw, vport); 2978 } 2979 2980 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2981 { 2982 struct mlx5_vport *vport; 2983 2984 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2985 if (IS_ERR(vport)) 2986 return; 2987 2988 esw_vport_destroy_offloads_acl_tables(esw, vport); 2989 } 2990 2991 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) 2992 { 2993 struct mlx5_eswitch_rep *rep; 2994 unsigned long i; 2995 int ret; 2996 2997 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 2998 return 0; 2999 3000 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3001 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3002 return 0; 3003 3004 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3005 if (ret) 3006 return ret; 3007 3008 mlx5_esw_for_each_rep(esw, i, rep) { 3009 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3010 mlx5_esw_offloads_rep_load(esw, rep->vport); 3011 } 3012 3013 return 0; 3014 } 3015 3016 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3017 { 3018 struct mlx5_esw_indir_table *indir; 3019 int err; 3020 3021 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3022 mutex_init(&esw->fdb_table.offloads.vports.lock); 3023 hash_init(esw->fdb_table.offloads.vports.table); 3024 atomic64_set(&esw->user_count, 0); 3025 3026 indir = mlx5_esw_indir_table_init(); 3027 if (IS_ERR(indir)) { 3028 err = PTR_ERR(indir); 3029 goto create_indir_err; 3030 } 3031 esw->fdb_table.offloads.indir = indir; 3032 3033 err = esw_create_uplink_offloads_acl_tables(esw); 3034 if (err) 3035 goto create_acl_err; 3036 3037 err = esw_create_offloads_table(esw); 3038 if (err) 3039 goto create_offloads_err; 3040 3041 err = esw_create_restore_table(esw); 3042 if (err) 3043 goto create_restore_err; 3044 3045 err = esw_create_offloads_fdb_tables(esw); 3046 if (err) 3047 goto create_fdb_err; 3048 3049 err = esw_create_vport_rx_group(esw); 3050 if (err) 3051 goto create_fg_err; 3052 3053 err = esw_create_vport_rx_drop_group(esw); 3054 if (err) 3055 goto create_rx_drop_fg_err; 3056 3057 err = esw_create_vport_rx_drop_rule(esw); 3058 if (err) 3059 goto create_rx_drop_rule_err; 3060 3061 return 0; 3062 3063 create_rx_drop_rule_err: 3064 esw_destroy_vport_rx_drop_group(esw); 3065 create_rx_drop_fg_err: 3066 esw_destroy_vport_rx_group(esw); 3067 create_fg_err: 3068 esw_destroy_offloads_fdb_tables(esw); 3069 create_fdb_err: 3070 esw_destroy_restore_table(esw); 3071 create_restore_err: 3072 esw_destroy_offloads_table(esw); 3073 create_offloads_err: 3074 esw_destroy_uplink_offloads_acl_tables(esw); 3075 create_acl_err: 3076 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3077 create_indir_err: 3078 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3079 return err; 3080 } 3081 3082 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3083 { 3084 esw_destroy_vport_rx_drop_rule(esw); 3085 esw_destroy_vport_rx_drop_group(esw); 3086 esw_destroy_vport_rx_group(esw); 3087 esw_destroy_offloads_fdb_tables(esw); 3088 esw_destroy_restore_table(esw); 3089 esw_destroy_offloads_table(esw); 3090 esw_destroy_uplink_offloads_acl_tables(esw); 3091 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3092 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3093 } 3094 3095 static void 3096 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 3097 { 3098 struct devlink *devlink; 3099 bool host_pf_disabled; 3100 u16 new_num_vfs; 3101 3102 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3103 host_params_context.host_num_of_vfs); 3104 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3105 host_params_context.host_pf_disabled); 3106 3107 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3108 return; 3109 3110 devlink = priv_to_devlink(esw->dev); 3111 devl_lock(devlink); 3112 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3113 if (esw->esw_funcs.num_vfs > 0) { 3114 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3115 } else { 3116 int err; 3117 3118 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3119 MLX5_VPORT_UC_ADDR_CHANGE); 3120 if (err) { 3121 devl_unlock(devlink); 3122 return; 3123 } 3124 } 3125 esw->esw_funcs.num_vfs = new_num_vfs; 3126 devl_unlock(devlink); 3127 } 3128 3129 static void esw_functions_changed_event_handler(struct work_struct *work) 3130 { 3131 struct mlx5_host_work *host_work; 3132 struct mlx5_eswitch *esw; 3133 const u32 *out; 3134 3135 host_work = container_of(work, struct mlx5_host_work, work); 3136 esw = host_work->esw; 3137 3138 out = mlx5_esw_query_functions(esw->dev); 3139 if (IS_ERR(out)) 3140 goto out; 3141 3142 esw_vfs_changed_event_handler(esw, out); 3143 kvfree(out); 3144 out: 3145 kfree(host_work); 3146 } 3147 3148 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3149 { 3150 struct mlx5_esw_functions *esw_funcs; 3151 struct mlx5_host_work *host_work; 3152 struct mlx5_eswitch *esw; 3153 3154 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 3155 if (!host_work) 3156 return NOTIFY_DONE; 3157 3158 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3159 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3160 3161 host_work->esw = esw; 3162 3163 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3164 queue_work(esw->work_queue, &host_work->work); 3165 3166 return NOTIFY_OK; 3167 } 3168 3169 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3170 { 3171 const u32 *query_host_out; 3172 3173 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3174 return 0; 3175 3176 query_host_out = mlx5_esw_query_functions(esw->dev); 3177 if (IS_ERR(query_host_out)) 3178 return PTR_ERR(query_host_out); 3179 3180 /* Mark non local controller with non zero controller number. */ 3181 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3182 host_params_context.host_number); 3183 kvfree(query_host_out); 3184 return 0; 3185 } 3186 3187 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3188 { 3189 /* Local controller is always valid */ 3190 if (controller == 0) 3191 return true; 3192 3193 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3194 return false; 3195 3196 /* External host number starts with zero in device */ 3197 return (controller == esw->offloads.host_number + 1); 3198 } 3199 3200 int esw_offloads_enable(struct mlx5_eswitch *esw) 3201 { 3202 struct mapping_ctx *reg_c0_obj_pool; 3203 struct mlx5_vport *vport; 3204 unsigned long i; 3205 u64 mapping_id; 3206 int err; 3207 3208 mutex_init(&esw->offloads.termtbl_mutex); 3209 mlx5_rdma_enable_roce(esw->dev); 3210 3211 err = mlx5_esw_host_number_init(esw); 3212 if (err) 3213 goto err_metadata; 3214 3215 err = esw_offloads_metadata_init(esw); 3216 if (err) 3217 goto err_metadata; 3218 3219 err = esw_set_passing_vport_metadata(esw, true); 3220 if (err) 3221 goto err_vport_metadata; 3222 3223 mapping_id = mlx5_query_nic_system_image_guid(esw->dev); 3224 3225 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN, 3226 sizeof(struct mlx5_mapped_obj), 3227 ESW_REG_C0_USER_DATA_METADATA_MASK, 3228 true); 3229 3230 if (IS_ERR(reg_c0_obj_pool)) { 3231 err = PTR_ERR(reg_c0_obj_pool); 3232 goto err_pool; 3233 } 3234 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3235 3236 err = esw_offloads_steering_init(esw); 3237 if (err) 3238 goto err_steering_init; 3239 3240 /* Representor will control the vport link state */ 3241 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3242 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3243 3244 /* Uplink vport rep must load first. */ 3245 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); 3246 if (err) 3247 goto err_uplink; 3248 3249 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3250 if (err) 3251 goto err_vports; 3252 3253 esw_offloads_devcom_init(esw); 3254 3255 return 0; 3256 3257 err_vports: 3258 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 3259 err_uplink: 3260 esw_offloads_steering_cleanup(esw); 3261 err_steering_init: 3262 mapping_destroy(reg_c0_obj_pool); 3263 err_pool: 3264 esw_set_passing_vport_metadata(esw, false); 3265 err_vport_metadata: 3266 esw_offloads_metadata_uninit(esw); 3267 err_metadata: 3268 mlx5_rdma_disable_roce(esw->dev); 3269 mutex_destroy(&esw->offloads.termtbl_mutex); 3270 return err; 3271 } 3272 3273 static int esw_offloads_stop(struct mlx5_eswitch *esw, 3274 struct netlink_ext_ack *extack) 3275 { 3276 int err; 3277 3278 esw->mode = MLX5_ESWITCH_LEGACY; 3279 3280 /* If changing from switchdev to legacy mode without sriov enabled, 3281 * no need to create legacy fdb. 3282 */ 3283 if (!mlx5_sriov_is_enabled(esw->dev)) 3284 return 0; 3285 3286 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3287 if (err) 3288 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3289 3290 return err; 3291 } 3292 3293 void esw_offloads_disable(struct mlx5_eswitch *esw) 3294 { 3295 esw_offloads_devcom_cleanup(esw); 3296 mlx5_eswitch_disable_pf_vf_vports(esw); 3297 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 3298 esw_set_passing_vport_metadata(esw, false); 3299 esw_offloads_steering_cleanup(esw); 3300 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3301 esw_offloads_metadata_uninit(esw); 3302 mlx5_rdma_disable_roce(esw->dev); 3303 mutex_destroy(&esw->offloads.termtbl_mutex); 3304 } 3305 3306 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3307 { 3308 switch (mode) { 3309 case DEVLINK_ESWITCH_MODE_LEGACY: 3310 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3311 break; 3312 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3313 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3314 break; 3315 default: 3316 return -EINVAL; 3317 } 3318 3319 return 0; 3320 } 3321 3322 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 3323 { 3324 switch (mlx5_mode) { 3325 case MLX5_ESWITCH_LEGACY: 3326 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3327 break; 3328 case MLX5_ESWITCH_OFFLOADS: 3329 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3330 break; 3331 default: 3332 return -EINVAL; 3333 } 3334 3335 return 0; 3336 } 3337 3338 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3339 { 3340 switch (mode) { 3341 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3342 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3343 break; 3344 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3345 *mlx5_mode = MLX5_INLINE_MODE_L2; 3346 break; 3347 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3348 *mlx5_mode = MLX5_INLINE_MODE_IP; 3349 break; 3350 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3351 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3352 break; 3353 default: 3354 return -EINVAL; 3355 } 3356 3357 return 0; 3358 } 3359 3360 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3361 { 3362 switch (mlx5_mode) { 3363 case MLX5_INLINE_MODE_NONE: 3364 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3365 break; 3366 case MLX5_INLINE_MODE_L2: 3367 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3368 break; 3369 case MLX5_INLINE_MODE_IP: 3370 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3371 break; 3372 case MLX5_INLINE_MODE_TCP_UDP: 3373 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3374 break; 3375 default: 3376 return -EINVAL; 3377 } 3378 3379 return 0; 3380 } 3381 3382 static bool esw_offloads_devlink_ns_eq_netdev_ns(struct devlink *devlink) 3383 { 3384 struct net *devl_net, *netdev_net; 3385 struct mlx5_eswitch *esw; 3386 3387 esw = mlx5_devlink_eswitch_get(devlink); 3388 netdev_net = dev_net(esw->dev->mlx5e_res.uplink_netdev); 3389 devl_net = devlink_net(devlink); 3390 3391 return net_eq(devl_net, netdev_net); 3392 } 3393 3394 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 3395 struct netlink_ext_ack *extack) 3396 { 3397 u16 cur_mlx5_mode, mlx5_mode = 0; 3398 struct mlx5_eswitch *esw; 3399 int err = 0; 3400 3401 esw = mlx5_devlink_eswitch_get(devlink); 3402 if (IS_ERR(esw)) 3403 return PTR_ERR(esw); 3404 3405 if (esw_mode_from_devlink(mode, &mlx5_mode)) 3406 return -EINVAL; 3407 3408 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && 3409 !esw_offloads_devlink_ns_eq_netdev_ns(devlink)) { 3410 NL_SET_ERR_MSG_MOD(extack, 3411 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); 3412 return -EPERM; 3413 } 3414 3415 mlx5_lag_disable_change(esw->dev); 3416 err = mlx5_esw_try_lock(esw); 3417 if (err < 0) { 3418 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 3419 goto enable_lag; 3420 } 3421 cur_mlx5_mode = err; 3422 err = 0; 3423 3424 if (cur_mlx5_mode == mlx5_mode) 3425 goto unlock; 3426 3427 mlx5_eswitch_disable_locked(esw); 3428 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { 3429 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 3430 NL_SET_ERR_MSG_MOD(extack, 3431 "Can't change mode while devlink traps are active"); 3432 err = -EOPNOTSUPP; 3433 goto unlock; 3434 } 3435 err = esw_offloads_start(esw, extack); 3436 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { 3437 err = esw_offloads_stop(esw, extack); 3438 mlx5_rescan_drivers(esw->dev); 3439 } else { 3440 err = -EINVAL; 3441 } 3442 3443 unlock: 3444 mlx5_esw_unlock(esw); 3445 enable_lag: 3446 mlx5_lag_enable_change(esw->dev); 3447 return err; 3448 } 3449 3450 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 3451 { 3452 struct mlx5_eswitch *esw; 3453 int err; 3454 3455 esw = mlx5_devlink_eswitch_get(devlink); 3456 if (IS_ERR(esw)) 3457 return PTR_ERR(esw); 3458 3459 down_read(&esw->mode_lock); 3460 err = esw_mode_to_devlink(esw->mode, mode); 3461 up_read(&esw->mode_lock); 3462 return err; 3463 } 3464 3465 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 3466 struct netlink_ext_ack *extack) 3467 { 3468 struct mlx5_core_dev *dev = esw->dev; 3469 struct mlx5_vport *vport; 3470 u16 err_vport_num = 0; 3471 unsigned long i; 3472 int err = 0; 3473 3474 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3475 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3476 if (err) { 3477 err_vport_num = vport->vport; 3478 NL_SET_ERR_MSG_MOD(extack, 3479 "Failed to set min inline on vport"); 3480 goto revert_inline_mode; 3481 } 3482 } 3483 return 0; 3484 3485 revert_inline_mode: 3486 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3487 if (vport->vport == err_vport_num) 3488 break; 3489 mlx5_modify_nic_vport_min_inline(dev, 3490 vport->vport, 3491 esw->offloads.inline_mode); 3492 } 3493 return err; 3494 } 3495 3496 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 3497 struct netlink_ext_ack *extack) 3498 { 3499 struct mlx5_core_dev *dev = devlink_priv(devlink); 3500 struct mlx5_eswitch *esw; 3501 u8 mlx5_mode; 3502 int err; 3503 3504 esw = mlx5_devlink_eswitch_get(devlink); 3505 if (IS_ERR(esw)) 3506 return PTR_ERR(esw); 3507 3508 down_write(&esw->mode_lock); 3509 3510 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 3511 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 3512 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 3513 err = 0; 3514 goto out; 3515 } 3516 3517 fallthrough; 3518 case MLX5_CAP_INLINE_MODE_L2: 3519 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 3520 err = -EOPNOTSUPP; 3521 goto out; 3522 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 3523 break; 3524 } 3525 3526 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3527 NL_SET_ERR_MSG_MOD(extack, 3528 "Can't set inline mode when flows are configured"); 3529 err = -EOPNOTSUPP; 3530 goto out; 3531 } 3532 3533 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 3534 if (err) 3535 goto out; 3536 3537 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 3538 if (err) 3539 goto out; 3540 3541 esw->offloads.inline_mode = mlx5_mode; 3542 up_write(&esw->mode_lock); 3543 return 0; 3544 3545 out: 3546 up_write(&esw->mode_lock); 3547 return err; 3548 } 3549 3550 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3551 { 3552 struct mlx5_eswitch *esw; 3553 int err; 3554 3555 esw = mlx5_devlink_eswitch_get(devlink); 3556 if (IS_ERR(esw)) 3557 return PTR_ERR(esw); 3558 3559 down_read(&esw->mode_lock); 3560 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3561 up_read(&esw->mode_lock); 3562 return err; 3563 } 3564 3565 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev) 3566 { 3567 struct devlink *devlink = priv_to_devlink(dev); 3568 struct mlx5_eswitch *esw; 3569 3570 devl_lock(devlink); 3571 esw = mlx5_devlink_eswitch_get(devlink); 3572 if (IS_ERR(esw)) { 3573 devl_unlock(devlink); 3574 /* Failure means no eswitch => not possible to change encap */ 3575 return true; 3576 } 3577 3578 down_write(&esw->mode_lock); 3579 if (esw->mode != MLX5_ESWITCH_LEGACY && 3580 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 3581 up_write(&esw->mode_lock); 3582 devl_unlock(devlink); 3583 return false; 3584 } 3585 3586 esw->offloads.num_block_encap++; 3587 up_write(&esw->mode_lock); 3588 devl_unlock(devlink); 3589 return true; 3590 } 3591 3592 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) 3593 { 3594 struct devlink *devlink = priv_to_devlink(dev); 3595 struct mlx5_eswitch *esw; 3596 3597 esw = mlx5_devlink_eswitch_get(devlink); 3598 if (IS_ERR(esw)) 3599 return; 3600 3601 down_write(&esw->mode_lock); 3602 esw->offloads.num_block_encap--; 3603 up_write(&esw->mode_lock); 3604 } 3605 3606 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3607 enum devlink_eswitch_encap_mode encap, 3608 struct netlink_ext_ack *extack) 3609 { 3610 struct mlx5_core_dev *dev = devlink_priv(devlink); 3611 struct mlx5_eswitch *esw; 3612 int err = 0; 3613 3614 esw = mlx5_devlink_eswitch_get(devlink); 3615 if (IS_ERR(esw)) 3616 return PTR_ERR(esw); 3617 3618 down_write(&esw->mode_lock); 3619 3620 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3621 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3622 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3623 err = -EOPNOTSUPP; 3624 goto unlock; 3625 } 3626 3627 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3628 err = -EOPNOTSUPP; 3629 goto unlock; 3630 } 3631 3632 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3633 esw->offloads.encap = encap; 3634 goto unlock; 3635 } 3636 3637 if (esw->offloads.encap == encap) 3638 goto unlock; 3639 3640 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3641 NL_SET_ERR_MSG_MOD(extack, 3642 "Can't set encapsulation when flows are configured"); 3643 err = -EOPNOTSUPP; 3644 goto unlock; 3645 } 3646 3647 if (esw->offloads.num_block_encap) { 3648 NL_SET_ERR_MSG_MOD(extack, 3649 "Can't set encapsulation when IPsec SA and/or policies are configured"); 3650 err = -EOPNOTSUPP; 3651 goto unlock; 3652 } 3653 3654 esw_destroy_offloads_fdb_tables(esw); 3655 3656 esw->offloads.encap = encap; 3657 3658 err = esw_create_offloads_fdb_tables(esw); 3659 3660 if (err) { 3661 NL_SET_ERR_MSG_MOD(extack, 3662 "Failed re-creating fast FDB table"); 3663 esw->offloads.encap = !encap; 3664 (void)esw_create_offloads_fdb_tables(esw); 3665 } 3666 3667 unlock: 3668 up_write(&esw->mode_lock); 3669 return err; 3670 } 3671 3672 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 3673 enum devlink_eswitch_encap_mode *encap) 3674 { 3675 struct mlx5_eswitch *esw; 3676 3677 esw = mlx5_devlink_eswitch_get(devlink); 3678 if (IS_ERR(esw)) 3679 return PTR_ERR(esw); 3680 3681 down_read(&esw->mode_lock); 3682 *encap = esw->offloads.encap; 3683 up_read(&esw->mode_lock); 3684 return 0; 3685 } 3686 3687 static bool 3688 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 3689 { 3690 /* Currently, only ECPF based device has representor for host PF. */ 3691 if (vport_num == MLX5_VPORT_PF && 3692 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 3693 return false; 3694 3695 if (vport_num == MLX5_VPORT_ECPF && 3696 !mlx5_ecpf_vport_exists(esw->dev)) 3697 return false; 3698 3699 return true; 3700 } 3701 3702 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 3703 const struct mlx5_eswitch_rep_ops *ops, 3704 u8 rep_type) 3705 { 3706 struct mlx5_eswitch_rep_data *rep_data; 3707 struct mlx5_eswitch_rep *rep; 3708 unsigned long i; 3709 3710 esw->offloads.rep_ops[rep_type] = ops; 3711 mlx5_esw_for_each_rep(esw, i, rep) { 3712 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 3713 rep->esw = esw; 3714 rep_data = &rep->rep_data[rep_type]; 3715 atomic_set(&rep_data->state, REP_REGISTERED); 3716 } 3717 } 3718 } 3719 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 3720 3721 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 3722 { 3723 struct mlx5_eswitch_rep *rep; 3724 unsigned long i; 3725 3726 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 3727 __unload_reps_all_vport(esw, rep_type); 3728 3729 mlx5_esw_for_each_rep(esw, i, rep) 3730 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 3731 } 3732 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 3733 3734 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 3735 { 3736 struct mlx5_eswitch_rep *rep; 3737 3738 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3739 return rep->rep_data[rep_type].priv; 3740 } 3741 3742 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 3743 u16 vport, 3744 u8 rep_type) 3745 { 3746 struct mlx5_eswitch_rep *rep; 3747 3748 rep = mlx5_eswitch_get_rep(esw, vport); 3749 3750 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 3751 esw->offloads.rep_ops[rep_type]->get_proto_dev) 3752 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 3753 return NULL; 3754 } 3755 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 3756 3757 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 3758 { 3759 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 3760 } 3761 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 3762 3763 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 3764 u16 vport) 3765 { 3766 return mlx5_eswitch_get_rep(esw, vport); 3767 } 3768 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 3769 3770 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 3771 { 3772 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 3773 } 3774 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 3775 3776 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 3777 { 3778 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 3779 } 3780 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 3781 3782 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 3783 u16 vport_num) 3784 { 3785 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3786 3787 if (WARN_ON_ONCE(IS_ERR(vport))) 3788 return 0; 3789 3790 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 3791 } 3792 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 3793 3794 int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port, 3795 u16 vport_num, u32 controller, u32 sfnum) 3796 { 3797 int err; 3798 3799 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE); 3800 if (err) 3801 return err; 3802 3803 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum); 3804 if (err) 3805 goto devlink_err; 3806 3807 err = mlx5_esw_offloads_rep_load(esw, vport_num); 3808 if (err) 3809 goto rep_err; 3810 return 0; 3811 3812 rep_err: 3813 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3814 devlink_err: 3815 mlx5_esw_vport_disable(esw, vport_num); 3816 return err; 3817 } 3818 3819 void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num) 3820 { 3821 mlx5_esw_offloads_rep_unload(esw, vport_num); 3822 mlx5_esw_devlink_sf_port_unregister(esw, vport_num); 3823 mlx5_esw_vport_disable(esw, vport_num); 3824 } 3825 3826 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 3827 { 3828 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 3829 void *query_ctx; 3830 void *hca_caps; 3831 int err; 3832 3833 *vhca_id = 0; 3834 if (mlx5_esw_is_manager_vport(esw, vport_num) || 3835 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) 3836 return -EPERM; 3837 3838 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 3839 if (!query_ctx) 3840 return -ENOMEM; 3841 3842 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx); 3843 if (err) 3844 goto out_free; 3845 3846 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 3847 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 3848 3849 out_free: 3850 kfree(query_ctx); 3851 return err; 3852 } 3853 3854 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 3855 { 3856 u16 *old_entry, *vhca_map_entry, vhca_id; 3857 int err; 3858 3859 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3860 if (err) { 3861 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 3862 vport_num, err); 3863 return err; 3864 } 3865 3866 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 3867 if (!vhca_map_entry) 3868 return -ENOMEM; 3869 3870 *vhca_map_entry = vport_num; 3871 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 3872 if (xa_is_err(old_entry)) { 3873 kfree(vhca_map_entry); 3874 return xa_err(old_entry); 3875 } 3876 kfree(old_entry); 3877 return 0; 3878 } 3879 3880 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 3881 { 3882 u16 *vhca_map_entry, vhca_id; 3883 int err; 3884 3885 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 3886 if (err) 3887 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 3888 vport_num, err); 3889 3890 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 3891 kfree(vhca_map_entry); 3892 } 3893 3894 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 3895 { 3896 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 3897 3898 if (!res) 3899 return -ENOENT; 3900 3901 *vport_num = *res; 3902 return 0; 3903 } 3904 3905 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 3906 u16 vport_num) 3907 { 3908 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 3909 3910 if (WARN_ON_ONCE(IS_ERR(vport))) 3911 return 0; 3912 3913 return vport->metadata; 3914 } 3915 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 3916 3917 static bool 3918 is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num) 3919 { 3920 return vport_num == MLX5_VPORT_PF || 3921 mlx5_eswitch_is_vf_vport(esw, vport_num) || 3922 mlx5_esw_is_sf_vport(esw, vport_num); 3923 } 3924 3925 int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port, 3926 u8 *hw_addr, int *hw_addr_len, 3927 struct netlink_ext_ack *extack) 3928 { 3929 struct mlx5_eswitch *esw; 3930 struct mlx5_vport *vport; 3931 u16 vport_num; 3932 3933 esw = mlx5_devlink_eswitch_get(port->devlink); 3934 if (IS_ERR(esw)) 3935 return PTR_ERR(esw); 3936 3937 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 3938 if (!is_port_function_supported(esw, vport_num)) 3939 return -EOPNOTSUPP; 3940 3941 vport = mlx5_eswitch_get_vport(esw, vport_num); 3942 if (IS_ERR(vport)) { 3943 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 3944 return PTR_ERR(vport); 3945 } 3946 3947 mutex_lock(&esw->state_lock); 3948 ether_addr_copy(hw_addr, vport->info.mac); 3949 *hw_addr_len = ETH_ALEN; 3950 mutex_unlock(&esw->state_lock); 3951 return 0; 3952 } 3953 3954 int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, 3955 const u8 *hw_addr, int hw_addr_len, 3956 struct netlink_ext_ack *extack) 3957 { 3958 struct mlx5_eswitch *esw; 3959 u16 vport_num; 3960 3961 esw = mlx5_devlink_eswitch_get(port->devlink); 3962 if (IS_ERR(esw)) { 3963 NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr"); 3964 return PTR_ERR(esw); 3965 } 3966 3967 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 3968 if (!is_port_function_supported(esw, vport_num)) { 3969 NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr"); 3970 return -EINVAL; 3971 } 3972 3973 return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr); 3974 } 3975 3976 static struct mlx5_vport * 3977 mlx5_devlink_port_fn_get_vport(struct devlink_port *port, struct mlx5_eswitch *esw) 3978 { 3979 u16 vport_num; 3980 3981 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) 3982 return ERR_PTR(-EOPNOTSUPP); 3983 3984 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 3985 if (!is_port_function_supported(esw, vport_num)) 3986 return ERR_PTR(-EOPNOTSUPP); 3987 3988 return mlx5_eswitch_get_vport(esw, vport_num); 3989 } 3990 3991 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 3992 struct netlink_ext_ack *extack) 3993 { 3994 struct mlx5_eswitch *esw; 3995 struct mlx5_vport *vport; 3996 int err = -EOPNOTSUPP; 3997 3998 esw = mlx5_devlink_eswitch_get(port->devlink); 3999 if (IS_ERR(esw)) 4000 return PTR_ERR(esw); 4001 4002 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4003 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4004 return err; 4005 } 4006 4007 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4008 if (IS_ERR(vport)) { 4009 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4010 return PTR_ERR(vport); 4011 } 4012 4013 mutex_lock(&esw->state_lock); 4014 if (vport->enabled) { 4015 *is_enabled = vport->info.mig_enabled; 4016 err = 0; 4017 } 4018 mutex_unlock(&esw->state_lock); 4019 return err; 4020 } 4021 4022 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 4023 struct netlink_ext_ack *extack) 4024 { 4025 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4026 struct mlx5_eswitch *esw; 4027 struct mlx5_vport *vport; 4028 void *query_ctx; 4029 void *hca_caps; 4030 int err = -EOPNOTSUPP; 4031 4032 esw = mlx5_devlink_eswitch_get(port->devlink); 4033 if (IS_ERR(esw)) 4034 return PTR_ERR(esw); 4035 4036 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4037 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4038 return err; 4039 } 4040 4041 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4042 if (IS_ERR(vport)) { 4043 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4044 return PTR_ERR(vport); 4045 } 4046 4047 mutex_lock(&esw->state_lock); 4048 if (!vport->enabled) { 4049 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4050 goto out; 4051 } 4052 4053 if (vport->info.mig_enabled == enable) { 4054 err = 0; 4055 goto out; 4056 } 4057 4058 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4059 if (!query_ctx) { 4060 err = -ENOMEM; 4061 goto out; 4062 } 4063 4064 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 4065 MLX5_CAP_GENERAL_2); 4066 if (err) { 4067 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4068 goto out_free; 4069 } 4070 4071 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4072 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, 1); 4073 4074 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 4075 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 4076 if (err) { 4077 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 4078 goto out_free; 4079 } 4080 4081 vport->info.mig_enabled = enable; 4082 4083 out_free: 4084 kfree(query_ctx); 4085 out: 4086 mutex_unlock(&esw->state_lock); 4087 return err; 4088 } 4089 4090 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 4091 struct netlink_ext_ack *extack) 4092 { 4093 struct mlx5_eswitch *esw; 4094 struct mlx5_vport *vport; 4095 int err = -EOPNOTSUPP; 4096 4097 esw = mlx5_devlink_eswitch_get(port->devlink); 4098 if (IS_ERR(esw)) 4099 return PTR_ERR(esw); 4100 4101 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4102 if (IS_ERR(vport)) { 4103 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4104 return PTR_ERR(vport); 4105 } 4106 4107 mutex_lock(&esw->state_lock); 4108 if (vport->enabled) { 4109 *is_enabled = vport->info.roce_enabled; 4110 err = 0; 4111 } 4112 mutex_unlock(&esw->state_lock); 4113 return err; 4114 } 4115 4116 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 4117 struct netlink_ext_ack *extack) 4118 { 4119 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4120 struct mlx5_eswitch *esw; 4121 struct mlx5_vport *vport; 4122 int err = -EOPNOTSUPP; 4123 void *query_ctx; 4124 void *hca_caps; 4125 u16 vport_num; 4126 4127 esw = mlx5_devlink_eswitch_get(port->devlink); 4128 if (IS_ERR(esw)) 4129 return PTR_ERR(esw); 4130 4131 vport = mlx5_devlink_port_fn_get_vport(port, esw); 4132 if (IS_ERR(vport)) { 4133 NL_SET_ERR_MSG_MOD(extack, "Invalid port"); 4134 return PTR_ERR(vport); 4135 } 4136 vport_num = vport->vport; 4137 4138 mutex_lock(&esw->state_lock); 4139 if (!vport->enabled) { 4140 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4141 goto out; 4142 } 4143 4144 if (vport->info.roce_enabled == enable) { 4145 err = 0; 4146 goto out; 4147 } 4148 4149 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4150 if (!query_ctx) { 4151 err = -ENOMEM; 4152 goto out; 4153 } 4154 4155 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4156 MLX5_CAP_GENERAL); 4157 if (err) { 4158 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4159 goto out_free; 4160 } 4161 4162 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4163 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 4164 4165 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4166 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4167 if (err) { 4168 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 4169 goto out_free; 4170 } 4171 4172 vport->info.roce_enabled = enable; 4173 4174 out_free: 4175 kfree(query_ctx); 4176 out: 4177 mutex_unlock(&esw->state_lock); 4178 return err; 4179 } 4180