1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/acl/ofld.h" 42 #include "esw/chains.h" 43 #include "rdma.h" 44 #include "en.h" 45 #include "fs_core.h" 46 #include "lib/devcom.h" 47 #include "lib/eq.h" 48 49 /* There are two match-all miss flows, one for unicast dst mac and 50 * one for multicast. 51 */ 52 #define MLX5_ESW_MISS_FLOWS (2) 53 #define UPLINK_REP_INDEX 0 54 55 /* Per vport tables */ 56 57 #define MLX5_ESW_VPORT_TABLE_SIZE 128 58 59 /* This struct is used as a key to the hash table and we need it to be packed 60 * so hash result is consistent 61 */ 62 struct mlx5_vport_key { 63 u32 chain; 64 u16 prio; 65 u16 vport; 66 u16 vhca_id; 67 } __packed; 68 69 struct mlx5_vport_table { 70 struct hlist_node hlist; 71 struct mlx5_flow_table *fdb; 72 u32 num_rules; 73 struct mlx5_vport_key key; 74 }; 75 76 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 77 78 static struct mlx5_flow_table * 79 esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns) 80 { 81 struct mlx5_flow_table_attr ft_attr = {}; 82 struct mlx5_flow_table *fdb; 83 84 ft_attr.autogroup.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS; 85 ft_attr.max_fte = MLX5_ESW_VPORT_TABLE_SIZE; 86 ft_attr.prio = FDB_PER_VPORT; 87 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 88 if (IS_ERR(fdb)) { 89 esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n", 90 PTR_ERR(fdb)); 91 } 92 93 return fdb; 94 } 95 96 static u32 flow_attr_to_vport_key(struct mlx5_eswitch *esw, 97 struct mlx5_esw_flow_attr *attr, 98 struct mlx5_vport_key *key) 99 { 100 key->vport = attr->in_rep->vport; 101 key->chain = attr->chain; 102 key->prio = attr->prio; 103 key->vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 104 return jhash(key, sizeof(*key), 0); 105 } 106 107 /* caller must hold vports.lock */ 108 static struct mlx5_vport_table * 109 esw_vport_tbl_lookup(struct mlx5_eswitch *esw, struct mlx5_vport_key *skey, u32 key) 110 { 111 struct mlx5_vport_table *e; 112 113 hash_for_each_possible(esw->fdb_table.offloads.vports.table, e, hlist, key) 114 if (!memcmp(&e->key, skey, sizeof(*skey))) 115 return e; 116 117 return NULL; 118 } 119 120 static void 121 esw_vport_tbl_put(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *attr) 122 { 123 struct mlx5_vport_table *e; 124 struct mlx5_vport_key key; 125 u32 hkey; 126 127 mutex_lock(&esw->fdb_table.offloads.vports.lock); 128 hkey = flow_attr_to_vport_key(esw, attr, &key); 129 e = esw_vport_tbl_lookup(esw, &key, hkey); 130 if (!e || --e->num_rules) 131 goto out; 132 133 hash_del(&e->hlist); 134 mlx5_destroy_flow_table(e->fdb); 135 kfree(e); 136 out: 137 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 138 } 139 140 static struct mlx5_flow_table * 141 esw_vport_tbl_get(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *attr) 142 { 143 struct mlx5_core_dev *dev = esw->dev; 144 struct mlx5_flow_namespace *ns; 145 struct mlx5_flow_table *fdb; 146 struct mlx5_vport_table *e; 147 struct mlx5_vport_key skey; 148 u32 hkey; 149 150 mutex_lock(&esw->fdb_table.offloads.vports.lock); 151 hkey = flow_attr_to_vport_key(esw, attr, &skey); 152 e = esw_vport_tbl_lookup(esw, &skey, hkey); 153 if (e) { 154 e->num_rules++; 155 goto out; 156 } 157 158 e = kzalloc(sizeof(*e), GFP_KERNEL); 159 if (!e) { 160 fdb = ERR_PTR(-ENOMEM); 161 goto err_alloc; 162 } 163 164 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 165 if (!ns) { 166 esw_warn(dev, "Failed to get FDB namespace\n"); 167 fdb = ERR_PTR(-ENOENT); 168 goto err_ns; 169 } 170 171 fdb = esw_vport_tbl_create(esw, ns); 172 if (IS_ERR(fdb)) 173 goto err_ns; 174 175 e->fdb = fdb; 176 e->num_rules = 1; 177 e->key = skey; 178 hash_add(esw->fdb_table.offloads.vports.table, &e->hlist, hkey); 179 out: 180 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 181 return e->fdb; 182 183 err_ns: 184 kfree(e); 185 err_alloc: 186 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 187 return fdb; 188 } 189 190 int mlx5_esw_vport_tbl_get(struct mlx5_eswitch *esw) 191 { 192 struct mlx5_esw_flow_attr attr = {}; 193 struct mlx5_eswitch_rep rep = {}; 194 struct mlx5_flow_table *fdb; 195 struct mlx5_vport *vport; 196 int i; 197 198 attr.prio = 1; 199 attr.in_rep = &rep; 200 mlx5_esw_for_all_vports(esw, i, vport) { 201 attr.in_rep->vport = vport->vport; 202 fdb = esw_vport_tbl_get(esw, &attr); 203 if (IS_ERR(fdb)) 204 goto out; 205 } 206 return 0; 207 208 out: 209 mlx5_esw_vport_tbl_put(esw); 210 return PTR_ERR(fdb); 211 } 212 213 void mlx5_esw_vport_tbl_put(struct mlx5_eswitch *esw) 214 { 215 struct mlx5_esw_flow_attr attr = {}; 216 struct mlx5_eswitch_rep rep = {}; 217 struct mlx5_vport *vport; 218 int i; 219 220 attr.prio = 1; 221 attr.in_rep = &rep; 222 mlx5_esw_for_all_vports(esw, i, vport) { 223 attr.in_rep->vport = vport->vport; 224 esw_vport_tbl_put(esw, &attr); 225 } 226 } 227 228 /* End: Per vport tables */ 229 230 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 231 u16 vport_num) 232 { 233 int idx = mlx5_eswitch_vport_num_to_index(esw, vport_num); 234 235 WARN_ON(idx > esw->total_vports - 1); 236 return &esw->offloads.vport_reps[idx]; 237 } 238 239 240 static void 241 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 242 struct mlx5_flow_spec *spec, 243 struct mlx5_esw_flow_attr *attr) 244 { 245 void *misc2; 246 void *misc; 247 248 /* Use metadata matching because vport is not represented by single 249 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 250 */ 251 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 252 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 253 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 254 mlx5_eswitch_get_vport_metadata_for_match(attr->in_mdev->priv.eswitch, 255 attr->in_rep->vport)); 256 257 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 258 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 259 mlx5_eswitch_get_vport_metadata_mask()); 260 261 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 262 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 263 if (memchr_inv(misc, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc))) 264 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 265 } else { 266 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 267 MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport); 268 269 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 270 MLX5_SET(fte_match_set_misc, misc, 271 source_eswitch_owner_vhca_id, 272 MLX5_CAP_GEN(attr->in_mdev, vhca_id)); 273 274 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 275 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 276 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 277 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 278 source_eswitch_owner_vhca_id); 279 280 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 281 } 282 283 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) && 284 attr->in_rep->vport == MLX5_VPORT_UPLINK) 285 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK; 286 } 287 288 struct mlx5_flow_handle * 289 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 290 struct mlx5_flow_spec *spec, 291 struct mlx5_esw_flow_attr *attr) 292 { 293 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {}; 294 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 295 bool split = !!(attr->split_count); 296 struct mlx5_flow_handle *rule; 297 struct mlx5_flow_table *fdb; 298 int j, i = 0; 299 300 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 301 return ERR_PTR(-EOPNOTSUPP); 302 303 flow_act.action = attr->action; 304 /* if per flow vlan pop/push is emulated, don't set that into the firmware */ 305 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 306 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | 307 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 308 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 309 flow_act.vlan[0].ethtype = ntohs(attr->vlan_proto[0]); 310 flow_act.vlan[0].vid = attr->vlan_vid[0]; 311 flow_act.vlan[0].prio = attr->vlan_prio[0]; 312 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 313 flow_act.vlan[1].ethtype = ntohs(attr->vlan_proto[1]); 314 flow_act.vlan[1].vid = attr->vlan_vid[1]; 315 flow_act.vlan[1].prio = attr->vlan_prio[1]; 316 } 317 } 318 319 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 320 struct mlx5_flow_table *ft; 321 322 if (attr->dest_ft) { 323 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 324 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 325 dest[i].ft = attr->dest_ft; 326 i++; 327 } else if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) { 328 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 329 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 330 dest[i].ft = mlx5_esw_chains_get_tc_end_ft(esw); 331 i++; 332 } else if (attr->dest_chain) { 333 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 334 ft = mlx5_esw_chains_get_table(esw, attr->dest_chain, 335 1, 0); 336 if (IS_ERR(ft)) { 337 rule = ERR_CAST(ft); 338 goto err_create_goto_table; 339 } 340 341 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 342 dest[i].ft = ft; 343 i++; 344 } else { 345 for (j = attr->split_count; j < attr->out_count; j++) { 346 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 347 dest[i].vport.num = attr->dests[j].rep->vport; 348 dest[i].vport.vhca_id = 349 MLX5_CAP_GEN(attr->dests[j].mdev, vhca_id); 350 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 351 dest[i].vport.flags |= 352 MLX5_FLOW_DEST_VPORT_VHCA_ID; 353 if (attr->dests[j].flags & MLX5_ESW_DEST_ENCAP) { 354 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 355 flow_act.pkt_reformat = attr->dests[j].pkt_reformat; 356 dest[i].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 357 dest[i].vport.pkt_reformat = 358 attr->dests[j].pkt_reformat; 359 } 360 i++; 361 } 362 } 363 } 364 365 if (attr->decap_pkt_reformat) 366 flow_act.pkt_reformat = attr->decap_pkt_reformat; 367 368 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 369 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 370 dest[i].counter_id = mlx5_fc_id(attr->counter); 371 i++; 372 } 373 374 if (attr->outer_match_level != MLX5_MATCH_NONE) 375 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 376 if (attr->inner_match_level != MLX5_MATCH_NONE) 377 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 378 379 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 380 flow_act.modify_hdr = attr->modify_hdr; 381 382 if (split) { 383 fdb = esw_vport_tbl_get(esw, attr); 384 } else { 385 if (attr->chain || attr->prio) 386 fdb = mlx5_esw_chains_get_table(esw, attr->chain, 387 attr->prio, 0); 388 else 389 fdb = attr->fdb; 390 391 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_NO_IN_PORT)) 392 mlx5_eswitch_set_rule_source_port(esw, spec, attr); 393 } 394 if (IS_ERR(fdb)) { 395 rule = ERR_CAST(fdb); 396 goto err_esw_get; 397 } 398 399 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 400 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, attr, 401 &flow_act, dest, i); 402 else 403 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 404 if (IS_ERR(rule)) 405 goto err_add_rule; 406 else 407 atomic64_inc(&esw->offloads.num_flows); 408 409 return rule; 410 411 err_add_rule: 412 if (split) 413 esw_vport_tbl_put(esw, attr); 414 else if (attr->chain || attr->prio) 415 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0); 416 err_esw_get: 417 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) && attr->dest_chain) 418 mlx5_esw_chains_put_table(esw, attr->dest_chain, 1, 0); 419 err_create_goto_table: 420 return rule; 421 } 422 423 struct mlx5_flow_handle * 424 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 425 struct mlx5_flow_spec *spec, 426 struct mlx5_esw_flow_attr *attr) 427 { 428 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {}; 429 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 430 struct mlx5_flow_table *fast_fdb; 431 struct mlx5_flow_table *fwd_fdb; 432 struct mlx5_flow_handle *rule; 433 int i; 434 435 fast_fdb = mlx5_esw_chains_get_table(esw, attr->chain, attr->prio, 0); 436 if (IS_ERR(fast_fdb)) { 437 rule = ERR_CAST(fast_fdb); 438 goto err_get_fast; 439 } 440 441 fwd_fdb = esw_vport_tbl_get(esw, attr); 442 if (IS_ERR(fwd_fdb)) { 443 rule = ERR_CAST(fwd_fdb); 444 goto err_get_fwd; 445 } 446 447 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 448 for (i = 0; i < attr->split_count; i++) { 449 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 450 dest[i].vport.num = attr->dests[i].rep->vport; 451 dest[i].vport.vhca_id = 452 MLX5_CAP_GEN(attr->dests[i].mdev, vhca_id); 453 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 454 dest[i].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 455 if (attr->dests[i].flags & MLX5_ESW_DEST_ENCAP) { 456 dest[i].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 457 dest[i].vport.pkt_reformat = attr->dests[i].pkt_reformat; 458 } 459 } 460 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 461 dest[i].ft = fwd_fdb, 462 i++; 463 464 mlx5_eswitch_set_rule_source_port(esw, spec, attr); 465 466 if (attr->outer_match_level != MLX5_MATCH_NONE) 467 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 468 469 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 470 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 471 472 if (IS_ERR(rule)) 473 goto add_err; 474 475 atomic64_inc(&esw->offloads.num_flows); 476 477 return rule; 478 add_err: 479 esw_vport_tbl_put(esw, attr); 480 err_get_fwd: 481 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0); 482 err_get_fast: 483 return rule; 484 } 485 486 static void 487 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 488 struct mlx5_flow_handle *rule, 489 struct mlx5_esw_flow_attr *attr, 490 bool fwd_rule) 491 { 492 bool split = (attr->split_count > 0); 493 int i; 494 495 mlx5_del_flow_rules(rule); 496 497 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) { 498 /* unref the term table */ 499 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 500 if (attr->dests[i].termtbl) 501 mlx5_eswitch_termtbl_put(esw, attr->dests[i].termtbl); 502 } 503 } 504 505 atomic64_dec(&esw->offloads.num_flows); 506 507 if (fwd_rule) { 508 esw_vport_tbl_put(esw, attr); 509 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 0); 510 } else { 511 if (split) 512 esw_vport_tbl_put(esw, attr); 513 else if (attr->chain || attr->prio) 514 mlx5_esw_chains_put_table(esw, attr->chain, attr->prio, 515 0); 516 if (attr->dest_chain) 517 mlx5_esw_chains_put_table(esw, attr->dest_chain, 1, 0); 518 } 519 } 520 521 void 522 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 523 struct mlx5_flow_handle *rule, 524 struct mlx5_esw_flow_attr *attr) 525 { 526 __mlx5_eswitch_del_rule(esw, rule, attr, false); 527 } 528 529 void 530 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 531 struct mlx5_flow_handle *rule, 532 struct mlx5_esw_flow_attr *attr) 533 { 534 __mlx5_eswitch_del_rule(esw, rule, attr, true); 535 } 536 537 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) 538 { 539 struct mlx5_eswitch_rep *rep; 540 int i, err = 0; 541 542 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); 543 mlx5_esw_for_each_host_func_rep(esw, i, rep, esw->esw_funcs.num_vfs) { 544 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 545 continue; 546 547 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); 548 if (err) 549 goto out; 550 } 551 552 out: 553 return err; 554 } 555 556 static struct mlx5_eswitch_rep * 557 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) 558 { 559 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; 560 561 in_rep = attr->in_rep; 562 out_rep = attr->dests[0].rep; 563 564 if (push) 565 vport = in_rep; 566 else if (pop) 567 vport = out_rep; 568 else 569 vport = in_rep; 570 571 return vport; 572 } 573 574 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, 575 bool push, bool pop, bool fwd) 576 { 577 struct mlx5_eswitch_rep *in_rep, *out_rep; 578 579 if ((push || pop) && !fwd) 580 goto out_notsupp; 581 582 in_rep = attr->in_rep; 583 out_rep = attr->dests[0].rep; 584 585 if (push && in_rep->vport == MLX5_VPORT_UPLINK) 586 goto out_notsupp; 587 588 if (pop && out_rep->vport == MLX5_VPORT_UPLINK) 589 goto out_notsupp; 590 591 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ 592 if (!push && !pop && fwd) 593 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK) 594 goto out_notsupp; 595 596 /* protects against (1) setting rules with different vlans to push and 597 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) 598 */ 599 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0])) 600 goto out_notsupp; 601 602 return 0; 603 604 out_notsupp: 605 return -EOPNOTSUPP; 606 } 607 608 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, 609 struct mlx5_esw_flow_attr *attr) 610 { 611 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 612 struct mlx5_eswitch_rep *vport = NULL; 613 bool push, pop, fwd; 614 int err = 0; 615 616 /* nop if we're on the vlan push/pop non emulation mode */ 617 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 618 return 0; 619 620 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 621 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 622 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && 623 !attr->dest_chain); 624 625 mutex_lock(&esw->state_lock); 626 627 err = esw_add_vlan_action_check(attr, push, pop, fwd); 628 if (err) 629 goto unlock; 630 631 attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 632 633 vport = esw_vlan_action_get_vport(attr, push, pop); 634 635 if (!push && !pop && fwd) { 636 /* tracks VF --> wire rules without vlan push action */ 637 if (attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) { 638 vport->vlan_refcount++; 639 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 640 } 641 642 goto unlock; 643 } 644 645 if (!push && !pop) 646 goto unlock; 647 648 if (!(offloads->vlan_push_pop_refcount)) { 649 /* it's the 1st vlan rule, apply global vlan pop policy */ 650 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); 651 if (err) 652 goto out; 653 } 654 offloads->vlan_push_pop_refcount++; 655 656 if (push) { 657 if (vport->vlan_refcount) 658 goto skip_set_push; 659 660 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, attr->vlan_vid[0], 0, 661 SET_VLAN_INSERT | SET_VLAN_STRIP); 662 if (err) 663 goto out; 664 vport->vlan = attr->vlan_vid[0]; 665 skip_set_push: 666 vport->vlan_refcount++; 667 } 668 out: 669 if (!err) 670 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 671 unlock: 672 mutex_unlock(&esw->state_lock); 673 return err; 674 } 675 676 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, 677 struct mlx5_esw_flow_attr *attr) 678 { 679 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 680 struct mlx5_eswitch_rep *vport = NULL; 681 bool push, pop, fwd; 682 int err = 0; 683 684 /* nop if we're on the vlan push/pop non emulation mode */ 685 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 686 return 0; 687 688 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED)) 689 return 0; 690 691 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 692 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 693 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 694 695 mutex_lock(&esw->state_lock); 696 697 vport = esw_vlan_action_get_vport(attr, push, pop); 698 699 if (!push && !pop && fwd) { 700 /* tracks VF --> wire rules without vlan push action */ 701 if (attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) 702 vport->vlan_refcount--; 703 704 goto out; 705 } 706 707 if (push) { 708 vport->vlan_refcount--; 709 if (vport->vlan_refcount) 710 goto skip_unset_push; 711 712 vport->vlan = 0; 713 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, 714 0, 0, SET_VLAN_STRIP); 715 if (err) 716 goto out; 717 } 718 719 skip_unset_push: 720 offloads->vlan_push_pop_refcount--; 721 if (offloads->vlan_push_pop_refcount) 722 goto out; 723 724 /* no more vlan rules, stop global vlan pop policy */ 725 err = esw_set_global_vlan_pop(esw, 0); 726 727 out: 728 mutex_unlock(&esw->state_lock); 729 return err; 730 } 731 732 struct mlx5_flow_handle * 733 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, u16 vport, 734 u32 sqn) 735 { 736 struct mlx5_flow_act flow_act = {0}; 737 struct mlx5_flow_destination dest = {}; 738 struct mlx5_flow_handle *flow_rule; 739 struct mlx5_flow_spec *spec; 740 void *misc; 741 742 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 743 if (!spec) { 744 flow_rule = ERR_PTR(-ENOMEM); 745 goto out; 746 } 747 748 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 749 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 750 /* source vport is the esw manager */ 751 MLX5_SET(fte_match_set_misc, misc, source_port, esw->manager_vport); 752 753 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 754 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 755 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 756 757 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 758 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 759 dest.vport.num = vport; 760 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 761 762 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 763 spec, &flow_act, &dest, 1); 764 if (IS_ERR(flow_rule)) 765 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule)); 766 out: 767 kvfree(spec); 768 return flow_rule; 769 } 770 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 771 772 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 773 { 774 mlx5_del_flow_rules(rule); 775 } 776 777 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 778 { 779 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 780 MLX5_FDB_TO_VPORT_REG_C_1; 781 } 782 783 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 784 { 785 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 786 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 787 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 788 u8 curr, wanted; 789 int err; 790 791 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 792 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 793 return 0; 794 795 MLX5_SET(query_esw_vport_context_in, in, opcode, 796 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 797 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 798 if (err) 799 return err; 800 801 curr = MLX5_GET(query_esw_vport_context_out, out, 802 esw_vport_context.fdb_to_vport_reg_c_id); 803 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 804 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 805 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 806 807 if (enable) 808 curr |= wanted; 809 else 810 curr &= ~wanted; 811 812 MLX5_SET(modify_esw_vport_context_in, min, 813 esw_vport_context.fdb_to_vport_reg_c_id, curr); 814 MLX5_SET(modify_esw_vport_context_in, min, 815 field_select.fdb_to_vport_reg_c_id, 1); 816 817 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 818 if (!err) { 819 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 820 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 821 else 822 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 823 } 824 825 return err; 826 } 827 828 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 829 struct mlx5_core_dev *peer_dev, 830 struct mlx5_flow_spec *spec, 831 struct mlx5_flow_destination *dest) 832 { 833 void *misc; 834 835 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 836 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 837 misc_parameters_2); 838 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 839 mlx5_eswitch_get_vport_metadata_mask()); 840 841 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 842 } else { 843 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 844 misc_parameters); 845 846 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 847 MLX5_CAP_GEN(peer_dev, vhca_id)); 848 849 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 850 851 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 852 misc_parameters); 853 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 854 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 855 source_eswitch_owner_vhca_id); 856 } 857 858 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 859 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 860 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 861 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 862 } 863 864 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 865 struct mlx5_eswitch *peer_esw, 866 struct mlx5_flow_spec *spec, 867 u16 vport) 868 { 869 void *misc; 870 871 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 872 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 873 misc_parameters_2); 874 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 875 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 876 vport)); 877 } else { 878 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 879 misc_parameters); 880 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 881 } 882 } 883 884 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 885 struct mlx5_core_dev *peer_dev) 886 { 887 struct mlx5_flow_destination dest = {}; 888 struct mlx5_flow_act flow_act = {0}; 889 struct mlx5_flow_handle **flows; 890 struct mlx5_flow_handle *flow; 891 struct mlx5_flow_spec *spec; 892 /* total vports is the same for both e-switches */ 893 int nvports = esw->total_vports; 894 void *misc; 895 int err, i; 896 897 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 898 if (!spec) 899 return -ENOMEM; 900 901 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 902 903 flows = kvzalloc(nvports * sizeof(*flows), GFP_KERNEL); 904 if (!flows) { 905 err = -ENOMEM; 906 goto alloc_flows_err; 907 } 908 909 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 910 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 911 misc_parameters); 912 913 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 914 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 915 spec, MLX5_VPORT_PF); 916 917 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 918 spec, &flow_act, &dest, 1); 919 if (IS_ERR(flow)) { 920 err = PTR_ERR(flow); 921 goto add_pf_flow_err; 922 } 923 flows[MLX5_VPORT_PF] = flow; 924 } 925 926 if (mlx5_ecpf_vport_exists(esw->dev)) { 927 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 928 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 929 spec, &flow_act, &dest, 1); 930 if (IS_ERR(flow)) { 931 err = PTR_ERR(flow); 932 goto add_ecpf_flow_err; 933 } 934 flows[mlx5_eswitch_ecpf_idx(esw)] = flow; 935 } 936 937 mlx5_esw_for_each_vf_vport_num(esw, i, mlx5_core_max_vfs(esw->dev)) { 938 esw_set_peer_miss_rule_source_port(esw, 939 peer_dev->priv.eswitch, 940 spec, i); 941 942 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 943 spec, &flow_act, &dest, 1); 944 if (IS_ERR(flow)) { 945 err = PTR_ERR(flow); 946 goto add_vf_flow_err; 947 } 948 flows[i] = flow; 949 } 950 951 esw->fdb_table.offloads.peer_miss_rules = flows; 952 953 kvfree(spec); 954 return 0; 955 956 add_vf_flow_err: 957 nvports = --i; 958 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, nvports) 959 mlx5_del_flow_rules(flows[i]); 960 961 if (mlx5_ecpf_vport_exists(esw->dev)) 962 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]); 963 add_ecpf_flow_err: 964 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 965 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]); 966 add_pf_flow_err: 967 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 968 kvfree(flows); 969 alloc_flows_err: 970 kvfree(spec); 971 return err; 972 } 973 974 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) 975 { 976 struct mlx5_flow_handle **flows; 977 int i; 978 979 flows = esw->fdb_table.offloads.peer_miss_rules; 980 981 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, 982 mlx5_core_max_vfs(esw->dev)) 983 mlx5_del_flow_rules(flows[i]); 984 985 if (mlx5_ecpf_vport_exists(esw->dev)) 986 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]); 987 988 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 989 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]); 990 991 kvfree(flows); 992 } 993 994 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 995 { 996 struct mlx5_flow_act flow_act = {0}; 997 struct mlx5_flow_destination dest = {}; 998 struct mlx5_flow_handle *flow_rule = NULL; 999 struct mlx5_flow_spec *spec; 1000 void *headers_c; 1001 void *headers_v; 1002 int err = 0; 1003 u8 *dmac_c; 1004 u8 *dmac_v; 1005 1006 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1007 if (!spec) { 1008 err = -ENOMEM; 1009 goto out; 1010 } 1011 1012 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1013 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1014 outer_headers); 1015 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1016 outer_headers.dmac_47_16); 1017 dmac_c[0] = 0x01; 1018 1019 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1020 dest.vport.num = esw->manager_vport; 1021 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1022 1023 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1024 spec, &flow_act, &dest, 1); 1025 if (IS_ERR(flow_rule)) { 1026 err = PTR_ERR(flow_rule); 1027 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1028 goto out; 1029 } 1030 1031 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1032 1033 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1034 outer_headers); 1035 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1036 outer_headers.dmac_47_16); 1037 dmac_v[0] = 0x01; 1038 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1039 spec, &flow_act, &dest, 1); 1040 if (IS_ERR(flow_rule)) { 1041 err = PTR_ERR(flow_rule); 1042 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1043 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1044 goto out; 1045 } 1046 1047 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1048 1049 out: 1050 kvfree(spec); 1051 return err; 1052 } 1053 1054 struct mlx5_flow_handle * 1055 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1056 { 1057 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1058 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1059 struct mlx5_flow_context *flow_context; 1060 struct mlx5_flow_handle *flow_rule; 1061 struct mlx5_flow_destination dest; 1062 struct mlx5_flow_spec *spec; 1063 void *misc; 1064 1065 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1066 return ERR_PTR(-EOPNOTSUPP); 1067 1068 spec = kzalloc(sizeof(*spec), GFP_KERNEL); 1069 if (!spec) 1070 return ERR_PTR(-ENOMEM); 1071 1072 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1073 misc_parameters_2); 1074 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1075 ESW_CHAIN_TAG_METADATA_MASK); 1076 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1077 misc_parameters_2); 1078 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1079 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1080 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1081 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1082 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1083 1084 flow_context = &spec->flow_context; 1085 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1086 flow_context->flow_tag = tag; 1087 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1088 dest.ft = esw->offloads.ft_offloads; 1089 1090 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1091 kfree(spec); 1092 1093 if (IS_ERR(flow_rule)) 1094 esw_warn(esw->dev, 1095 "Failed to create restore rule for tag: %d, err(%d)\n", 1096 tag, (int)PTR_ERR(flow_rule)); 1097 1098 return flow_rule; 1099 } 1100 1101 u32 1102 esw_get_max_restore_tag(struct mlx5_eswitch *esw) 1103 { 1104 return ESW_CHAIN_TAG_METADATA_MASK; 1105 } 1106 1107 #define MAX_PF_SQ 256 1108 #define MAX_SQ_NVPORTS 32 1109 1110 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1111 u32 *flow_group_in) 1112 { 1113 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1114 flow_group_in, 1115 match_criteria); 1116 1117 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1118 MLX5_SET(create_flow_group_in, flow_group_in, 1119 match_criteria_enable, 1120 MLX5_MATCH_MISC_PARAMETERS_2); 1121 1122 MLX5_SET(fte_match_param, match_criteria, 1123 misc_parameters_2.metadata_reg_c_0, 1124 mlx5_eswitch_get_vport_metadata_mask()); 1125 } else { 1126 MLX5_SET(create_flow_group_in, flow_group_in, 1127 match_criteria_enable, 1128 MLX5_MATCH_MISC_PARAMETERS); 1129 1130 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1131 misc_parameters.source_port); 1132 } 1133 } 1134 1135 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw, int nvports) 1136 { 1137 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1138 struct mlx5_flow_table_attr ft_attr = {}; 1139 struct mlx5_core_dev *dev = esw->dev; 1140 struct mlx5_flow_namespace *root_ns; 1141 struct mlx5_flow_table *fdb = NULL; 1142 u32 flags = 0, *flow_group_in; 1143 int table_size, ix, err = 0; 1144 struct mlx5_flow_group *g; 1145 void *match_criteria; 1146 u8 *dmac; 1147 1148 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1149 1150 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1151 if (!flow_group_in) 1152 return -ENOMEM; 1153 1154 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1155 if (!root_ns) { 1156 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1157 err = -EOPNOTSUPP; 1158 goto ns_err; 1159 } 1160 esw->fdb_table.offloads.ns = root_ns; 1161 err = mlx5_flow_namespace_set_mode(root_ns, 1162 esw->dev->priv.steering->mode); 1163 if (err) { 1164 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1165 goto ns_err; 1166 } 1167 1168 table_size = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1169 MLX5_ESW_MISS_FLOWS + esw->total_vports; 1170 1171 /* create the slow path fdb with encap set, so further table instances 1172 * can be created at run time while VFs are probed if the FW allows that. 1173 */ 1174 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1175 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1176 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1177 1178 ft_attr.flags = flags; 1179 ft_attr.max_fte = table_size; 1180 ft_attr.prio = FDB_SLOW_PATH; 1181 1182 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1183 if (IS_ERR(fdb)) { 1184 err = PTR_ERR(fdb); 1185 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1186 goto slow_fdb_err; 1187 } 1188 esw->fdb_table.offloads.slow_fdb = fdb; 1189 1190 err = mlx5_esw_chains_create(esw); 1191 if (err) { 1192 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1193 goto fdb_chains_err; 1194 } 1195 1196 /* create send-to-vport group */ 1197 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1198 MLX5_MATCH_MISC_PARAMETERS); 1199 1200 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1201 1202 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1203 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 1204 1205 ix = nvports * MAX_SQ_NVPORTS + MAX_PF_SQ; 1206 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1207 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); 1208 1209 g = mlx5_create_flow_group(fdb, flow_group_in); 1210 if (IS_ERR(g)) { 1211 err = PTR_ERR(g); 1212 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1213 goto send_vport_err; 1214 } 1215 esw->fdb_table.offloads.send_to_vport_grp = g; 1216 1217 /* create peer esw miss group */ 1218 memset(flow_group_in, 0, inlen); 1219 1220 esw_set_flow_group_source_port(esw, flow_group_in); 1221 1222 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1223 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1224 flow_group_in, 1225 match_criteria); 1226 1227 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1228 misc_parameters.source_eswitch_owner_vhca_id); 1229 1230 MLX5_SET(create_flow_group_in, flow_group_in, 1231 source_eswitch_owner_vhca_id_valid, 1); 1232 } 1233 1234 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1235 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1236 ix + esw->total_vports - 1); 1237 ix += esw->total_vports; 1238 1239 g = mlx5_create_flow_group(fdb, flow_group_in); 1240 if (IS_ERR(g)) { 1241 err = PTR_ERR(g); 1242 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err); 1243 goto peer_miss_err; 1244 } 1245 esw->fdb_table.offloads.peer_miss_grp = g; 1246 1247 /* create miss group */ 1248 memset(flow_group_in, 0, inlen); 1249 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1250 MLX5_MATCH_OUTER_HEADERS); 1251 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1252 match_criteria); 1253 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1254 outer_headers.dmac_47_16); 1255 dmac[0] = 0x01; 1256 1257 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1258 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1259 ix + MLX5_ESW_MISS_FLOWS); 1260 1261 g = mlx5_create_flow_group(fdb, flow_group_in); 1262 if (IS_ERR(g)) { 1263 err = PTR_ERR(g); 1264 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err); 1265 goto miss_err; 1266 } 1267 esw->fdb_table.offloads.miss_grp = g; 1268 1269 err = esw_add_fdb_miss_rule(esw); 1270 if (err) 1271 goto miss_rule_err; 1272 1273 esw->nvports = nvports; 1274 kvfree(flow_group_in); 1275 return 0; 1276 1277 miss_rule_err: 1278 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1279 miss_err: 1280 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1281 peer_miss_err: 1282 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1283 send_vport_err: 1284 mlx5_esw_chains_destroy(esw); 1285 fdb_chains_err: 1286 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1287 slow_fdb_err: 1288 /* Holds true only as long as DMFS is the default */ 1289 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1290 ns_err: 1291 kvfree(flow_group_in); 1292 return err; 1293 } 1294 1295 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1296 { 1297 if (!esw->fdb_table.offloads.slow_fdb) 1298 return; 1299 1300 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1301 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1302 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1303 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1304 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1305 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1306 1307 mlx5_esw_chains_destroy(esw); 1308 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1309 /* Holds true only as long as DMFS is the default */ 1310 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1311 MLX5_FLOW_STEERING_MODE_DMFS); 1312 } 1313 1314 static int esw_create_offloads_table(struct mlx5_eswitch *esw, int nvports) 1315 { 1316 struct mlx5_flow_table_attr ft_attr = {}; 1317 struct mlx5_core_dev *dev = esw->dev; 1318 struct mlx5_flow_table *ft_offloads; 1319 struct mlx5_flow_namespace *ns; 1320 int err = 0; 1321 1322 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1323 if (!ns) { 1324 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1325 return -EOPNOTSUPP; 1326 } 1327 1328 ft_attr.max_fte = nvports + MLX5_ESW_MISS_FLOWS; 1329 ft_attr.prio = 1; 1330 1331 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 1332 if (IS_ERR(ft_offloads)) { 1333 err = PTR_ERR(ft_offloads); 1334 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 1335 return err; 1336 } 1337 1338 esw->offloads.ft_offloads = ft_offloads; 1339 return 0; 1340 } 1341 1342 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 1343 { 1344 struct mlx5_esw_offload *offloads = &esw->offloads; 1345 1346 mlx5_destroy_flow_table(offloads->ft_offloads); 1347 } 1348 1349 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw, int nvports) 1350 { 1351 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1352 struct mlx5_flow_group *g; 1353 u32 *flow_group_in; 1354 int err = 0; 1355 1356 nvports = nvports + MLX5_ESW_MISS_FLOWS; 1357 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1358 if (!flow_group_in) 1359 return -ENOMEM; 1360 1361 /* create vport rx group */ 1362 esw_set_flow_group_source_port(esw, flow_group_in); 1363 1364 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1365 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 1366 1367 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 1368 1369 if (IS_ERR(g)) { 1370 err = PTR_ERR(g); 1371 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 1372 goto out; 1373 } 1374 1375 esw->offloads.vport_rx_group = g; 1376 out: 1377 kvfree(flow_group_in); 1378 return err; 1379 } 1380 1381 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 1382 { 1383 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 1384 } 1385 1386 struct mlx5_flow_handle * 1387 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 1388 struct mlx5_flow_destination *dest) 1389 { 1390 struct mlx5_flow_act flow_act = {0}; 1391 struct mlx5_flow_handle *flow_rule; 1392 struct mlx5_flow_spec *spec; 1393 void *misc; 1394 1395 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1396 if (!spec) { 1397 flow_rule = ERR_PTR(-ENOMEM); 1398 goto out; 1399 } 1400 1401 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1402 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 1403 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1404 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 1405 1406 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 1407 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1408 mlx5_eswitch_get_vport_metadata_mask()); 1409 1410 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1411 } else { 1412 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1413 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1414 1415 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1416 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1417 1418 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1419 } 1420 1421 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1422 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 1423 &flow_act, dest, 1); 1424 if (IS_ERR(flow_rule)) { 1425 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 1426 goto out; 1427 } 1428 1429 out: 1430 kvfree(spec); 1431 return flow_rule; 1432 } 1433 1434 1435 static int mlx5_eswitch_inline_mode_get(const struct mlx5_eswitch *esw, u8 *mode) 1436 { 1437 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 1438 struct mlx5_core_dev *dev = esw->dev; 1439 int vport; 1440 1441 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 1442 return -EOPNOTSUPP; 1443 1444 if (esw->mode == MLX5_ESWITCH_NONE) 1445 return -EOPNOTSUPP; 1446 1447 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 1448 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 1449 mlx5_mode = MLX5_INLINE_MODE_NONE; 1450 goto out; 1451 case MLX5_CAP_INLINE_MODE_L2: 1452 mlx5_mode = MLX5_INLINE_MODE_L2; 1453 goto out; 1454 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 1455 goto query_vports; 1456 } 1457 1458 query_vports: 1459 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 1460 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) { 1461 mlx5_query_nic_vport_min_inline(dev, vport, &mlx5_mode); 1462 if (prev_mlx5_mode != mlx5_mode) 1463 return -EINVAL; 1464 prev_mlx5_mode = mlx5_mode; 1465 } 1466 1467 out: 1468 *mode = mlx5_mode; 1469 return 0; 1470 } 1471 1472 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 1473 { 1474 struct mlx5_esw_offload *offloads = &esw->offloads; 1475 1476 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1477 return; 1478 1479 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 1480 mlx5_destroy_flow_group(offloads->restore_group); 1481 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 1482 } 1483 1484 static int esw_create_restore_table(struct mlx5_eswitch *esw) 1485 { 1486 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1487 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1488 struct mlx5_flow_table_attr ft_attr = {}; 1489 struct mlx5_core_dev *dev = esw->dev; 1490 struct mlx5_flow_namespace *ns; 1491 struct mlx5_modify_hdr *mod_hdr; 1492 void *match_criteria, *misc; 1493 struct mlx5_flow_table *ft; 1494 struct mlx5_flow_group *g; 1495 u32 *flow_group_in; 1496 int err = 0; 1497 1498 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1499 return 0; 1500 1501 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1502 if (!ns) { 1503 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1504 return -EOPNOTSUPP; 1505 } 1506 1507 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1508 if (!flow_group_in) { 1509 err = -ENOMEM; 1510 goto out_free; 1511 } 1512 1513 ft_attr.max_fte = 1 << ESW_CHAIN_TAG_METADATA_BITS; 1514 ft = mlx5_create_flow_table(ns, &ft_attr); 1515 if (IS_ERR(ft)) { 1516 err = PTR_ERR(ft); 1517 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 1518 err); 1519 goto out_free; 1520 } 1521 1522 memset(flow_group_in, 0, inlen); 1523 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1524 match_criteria); 1525 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 1526 misc_parameters_2); 1527 1528 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1529 ESW_CHAIN_TAG_METADATA_MASK); 1530 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1531 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1532 ft_attr.max_fte - 1); 1533 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1534 MLX5_MATCH_MISC_PARAMETERS_2); 1535 g = mlx5_create_flow_group(ft, flow_group_in); 1536 if (IS_ERR(g)) { 1537 err = PTR_ERR(g); 1538 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 1539 err); 1540 goto err_group; 1541 } 1542 1543 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 1544 MLX5_SET(copy_action_in, modact, src_field, 1545 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 1546 MLX5_SET(copy_action_in, modact, dst_field, 1547 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1548 mod_hdr = mlx5_modify_header_alloc(esw->dev, 1549 MLX5_FLOW_NAMESPACE_KERNEL, 1, 1550 modact); 1551 if (IS_ERR(mod_hdr)) { 1552 err = PTR_ERR(mod_hdr); 1553 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 1554 err); 1555 goto err_mod_hdr; 1556 } 1557 1558 esw->offloads.ft_offloads_restore = ft; 1559 esw->offloads.restore_group = g; 1560 esw->offloads.restore_copy_hdr_id = mod_hdr; 1561 1562 kvfree(flow_group_in); 1563 1564 return 0; 1565 1566 err_mod_hdr: 1567 mlx5_destroy_flow_group(g); 1568 err_group: 1569 mlx5_destroy_flow_table(ft); 1570 out_free: 1571 kvfree(flow_group_in); 1572 1573 return err; 1574 } 1575 1576 static int esw_offloads_start(struct mlx5_eswitch *esw, 1577 struct netlink_ext_ack *extack) 1578 { 1579 int err, err1; 1580 1581 if (esw->mode != MLX5_ESWITCH_LEGACY && 1582 !mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1583 NL_SET_ERR_MSG_MOD(extack, 1584 "Can't set offloads mode, SRIOV legacy not enabled"); 1585 return -EINVAL; 1586 } 1587 1588 mlx5_eswitch_disable_locked(esw, false); 1589 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, 1590 esw->dev->priv.sriov.num_vfs); 1591 if (err) { 1592 NL_SET_ERR_MSG_MOD(extack, 1593 "Failed setting eswitch to offloads"); 1594 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, 1595 MLX5_ESWITCH_IGNORE_NUM_VFS); 1596 if (err1) { 1597 NL_SET_ERR_MSG_MOD(extack, 1598 "Failed setting eswitch back to legacy"); 1599 } 1600 } 1601 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 1602 if (mlx5_eswitch_inline_mode_get(esw, 1603 &esw->offloads.inline_mode)) { 1604 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 1605 NL_SET_ERR_MSG_MOD(extack, 1606 "Inline mode is different between vports"); 1607 } 1608 } 1609 return err; 1610 } 1611 1612 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 1613 { 1614 kfree(esw->offloads.vport_reps); 1615 } 1616 1617 int esw_offloads_init_reps(struct mlx5_eswitch *esw) 1618 { 1619 int total_vports = esw->total_vports; 1620 struct mlx5_eswitch_rep *rep; 1621 int vport_index; 1622 u8 rep_type; 1623 1624 esw->offloads.vport_reps = kcalloc(total_vports, 1625 sizeof(struct mlx5_eswitch_rep), 1626 GFP_KERNEL); 1627 if (!esw->offloads.vport_reps) 1628 return -ENOMEM; 1629 1630 mlx5_esw_for_all_reps(esw, vport_index, rep) { 1631 rep->vport = mlx5_eswitch_index_to_vport_num(esw, vport_index); 1632 rep->vport_index = vport_index; 1633 1634 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 1635 atomic_set(&rep->rep_data[rep_type].state, 1636 REP_UNREGISTERED); 1637 } 1638 1639 return 0; 1640 } 1641 1642 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 1643 struct mlx5_eswitch_rep *rep, u8 rep_type) 1644 { 1645 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 1646 REP_LOADED, REP_REGISTERED) == REP_LOADED) 1647 esw->offloads.rep_ops[rep_type]->unload(rep); 1648 } 1649 1650 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 1651 { 1652 struct mlx5_eswitch_rep *rep; 1653 int i; 1654 1655 mlx5_esw_for_each_vf_rep_reverse(esw, i, rep, esw->esw_funcs.num_vfs) 1656 __esw_offloads_unload_rep(esw, rep, rep_type); 1657 1658 if (mlx5_ecpf_vport_exists(esw->dev)) { 1659 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF); 1660 __esw_offloads_unload_rep(esw, rep, rep_type); 1661 } 1662 1663 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1664 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF); 1665 __esw_offloads_unload_rep(esw, rep, rep_type); 1666 } 1667 1668 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 1669 __esw_offloads_unload_rep(esw, rep, rep_type); 1670 } 1671 1672 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) 1673 { 1674 struct mlx5_eswitch_rep *rep; 1675 int rep_type; 1676 int err; 1677 1678 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 1679 return 0; 1680 1681 rep = mlx5_eswitch_get_rep(esw, vport_num); 1682 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 1683 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 1684 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 1685 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 1686 if (err) 1687 goto err_reps; 1688 } 1689 1690 return 0; 1691 1692 err_reps: 1693 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 1694 for (--rep_type; rep_type >= 0; rep_type--) 1695 __esw_offloads_unload_rep(esw, rep, rep_type); 1696 return err; 1697 } 1698 1699 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) 1700 { 1701 struct mlx5_eswitch_rep *rep; 1702 int rep_type; 1703 1704 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 1705 return; 1706 1707 rep = mlx5_eswitch_get_rep(esw, vport_num); 1708 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 1709 __esw_offloads_unload_rep(esw, rep, rep_type); 1710 } 1711 1712 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 1713 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 1714 1715 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 1716 struct mlx5_eswitch *peer_esw) 1717 { 1718 int err; 1719 1720 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 1721 if (err) 1722 return err; 1723 1724 return 0; 1725 } 1726 1727 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) 1728 { 1729 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1730 mlx5e_tc_clean_fdb_peer_flows(esw); 1731 #endif 1732 esw_del_fdb_peer_miss_rules(esw); 1733 } 1734 1735 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 1736 struct mlx5_eswitch *peer_esw, 1737 bool pair) 1738 { 1739 struct mlx5_flow_root_namespace *peer_ns; 1740 struct mlx5_flow_root_namespace *ns; 1741 int err; 1742 1743 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 1744 ns = esw->dev->priv.steering->fdb_root_ns; 1745 1746 if (pair) { 1747 err = mlx5_flow_namespace_set_peer(ns, peer_ns); 1748 if (err) 1749 return err; 1750 1751 err = mlx5_flow_namespace_set_peer(peer_ns, ns); 1752 if (err) { 1753 mlx5_flow_namespace_set_peer(ns, NULL); 1754 return err; 1755 } 1756 } else { 1757 mlx5_flow_namespace_set_peer(ns, NULL); 1758 mlx5_flow_namespace_set_peer(peer_ns, NULL); 1759 } 1760 1761 return 0; 1762 } 1763 1764 static int mlx5_esw_offloads_devcom_event(int event, 1765 void *my_data, 1766 void *event_data) 1767 { 1768 struct mlx5_eswitch *esw = my_data; 1769 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 1770 struct mlx5_eswitch *peer_esw = event_data; 1771 int err; 1772 1773 switch (event) { 1774 case ESW_OFFLOADS_DEVCOM_PAIR: 1775 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 1776 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 1777 break; 1778 1779 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 1780 if (err) 1781 goto err_out; 1782 err = mlx5_esw_offloads_pair(esw, peer_esw); 1783 if (err) 1784 goto err_peer; 1785 1786 err = mlx5_esw_offloads_pair(peer_esw, esw); 1787 if (err) 1788 goto err_pair; 1789 1790 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); 1791 break; 1792 1793 case ESW_OFFLOADS_DEVCOM_UNPAIR: 1794 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) 1795 break; 1796 1797 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); 1798 mlx5_esw_offloads_unpair(peer_esw); 1799 mlx5_esw_offloads_unpair(esw); 1800 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 1801 break; 1802 } 1803 1804 return 0; 1805 1806 err_pair: 1807 mlx5_esw_offloads_unpair(esw); 1808 err_peer: 1809 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 1810 err_out: 1811 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 1812 event, err); 1813 return err; 1814 } 1815 1816 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw) 1817 { 1818 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 1819 1820 INIT_LIST_HEAD(&esw->offloads.peer_flows); 1821 mutex_init(&esw->offloads.peer_mutex); 1822 1823 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1824 return; 1825 1826 mlx5_devcom_register_component(devcom, 1827 MLX5_DEVCOM_ESW_OFFLOADS, 1828 mlx5_esw_offloads_devcom_event, 1829 esw); 1830 1831 mlx5_devcom_send_event(devcom, 1832 MLX5_DEVCOM_ESW_OFFLOADS, 1833 ESW_OFFLOADS_DEVCOM_PAIR, esw); 1834 } 1835 1836 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 1837 { 1838 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 1839 1840 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1841 return; 1842 1843 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, 1844 ESW_OFFLOADS_DEVCOM_UNPAIR, esw); 1845 1846 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); 1847 } 1848 1849 static bool 1850 esw_check_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 1851 { 1852 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 1853 return false; 1854 1855 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1856 MLX5_FDB_TO_VPORT_REG_C_0)) 1857 return false; 1858 1859 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 1860 return false; 1861 1862 if (mlx5_core_is_ecpf_esw_manager(esw->dev) || 1863 mlx5_ecpf_vport_exists(esw->dev)) 1864 return false; 1865 1866 return true; 1867 } 1868 1869 static bool 1870 esw_check_vport_match_metadata_mandatory(const struct mlx5_eswitch *esw) 1871 { 1872 return mlx5_core_mp_enabled(esw->dev); 1873 } 1874 1875 static bool esw_use_vport_metadata(const struct mlx5_eswitch *esw) 1876 { 1877 return esw_check_vport_match_metadata_mandatory(esw) && 1878 esw_check_vport_match_metadata_supported(esw); 1879 } 1880 1881 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 1882 { 1883 u32 num_vports = GENMASK(ESW_VPORT_BITS - 1, 0) - 1; 1884 u32 vhca_id_mask = GENMASK(ESW_VHCA_ID_BITS - 1, 0); 1885 u32 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 1886 u32 start; 1887 u32 end; 1888 int id; 1889 1890 /* Make sure the vhca_id fits the ESW_VHCA_ID_BITS */ 1891 WARN_ON_ONCE(vhca_id >= BIT(ESW_VHCA_ID_BITS)); 1892 1893 /* Trim vhca_id to ESW_VHCA_ID_BITS */ 1894 vhca_id &= vhca_id_mask; 1895 1896 start = (vhca_id << ESW_VPORT_BITS); 1897 end = start + num_vports; 1898 if (!vhca_id) 1899 start += 1; /* zero is reserved/invalid metadata */ 1900 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, start, end, GFP_KERNEL); 1901 1902 return (id < 0) ? 0 : id; 1903 } 1904 1905 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 1906 { 1907 ida_free(&esw->offloads.vport_metadata_ida, metadata); 1908 } 1909 1910 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 1911 struct mlx5_vport *vport) 1912 { 1913 if (vport->vport == MLX5_VPORT_UPLINK) 1914 return 0; 1915 1916 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 1917 vport->metadata = vport->default_metadata; 1918 return vport->metadata ? 0 : -ENOSPC; 1919 } 1920 1921 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 1922 struct mlx5_vport *vport) 1923 { 1924 if (vport->vport == MLX5_VPORT_UPLINK || !vport->default_metadata) 1925 return; 1926 1927 WARN_ON(vport->metadata != vport->default_metadata); 1928 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 1929 } 1930 1931 int 1932 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 1933 struct mlx5_vport *vport) 1934 { 1935 int err; 1936 1937 err = esw_offloads_vport_metadata_setup(esw, vport); 1938 if (err) 1939 goto metadata_err; 1940 1941 err = esw_acl_ingress_ofld_setup(esw, vport); 1942 if (err) 1943 goto ingress_err; 1944 1945 if (mlx5_eswitch_is_vf_vport(esw, vport->vport)) { 1946 err = esw_acl_egress_ofld_setup(esw, vport); 1947 if (err) 1948 goto egress_err; 1949 } 1950 1951 return 0; 1952 1953 egress_err: 1954 esw_acl_ingress_ofld_cleanup(esw, vport); 1955 ingress_err: 1956 esw_offloads_vport_metadata_cleanup(esw, vport); 1957 metadata_err: 1958 return err; 1959 } 1960 1961 void 1962 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 1963 struct mlx5_vport *vport) 1964 { 1965 esw_acl_egress_ofld_cleanup(vport); 1966 esw_acl_ingress_ofld_cleanup(esw, vport); 1967 esw_offloads_vport_metadata_cleanup(esw, vport); 1968 } 1969 1970 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 1971 { 1972 struct mlx5_vport *vport; 1973 int err; 1974 1975 if (esw_use_vport_metadata(esw)) 1976 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 1977 1978 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 1979 err = esw_vport_create_offloads_acl_tables(esw, vport); 1980 if (err) 1981 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 1982 return err; 1983 } 1984 1985 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 1986 { 1987 struct mlx5_vport *vport; 1988 1989 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 1990 esw_vport_destroy_offloads_acl_tables(esw, vport); 1991 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 1992 } 1993 1994 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 1995 { 1996 int num_vfs = esw->esw_funcs.num_vfs; 1997 int total_vports; 1998 int err; 1999 2000 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 2001 total_vports = esw->total_vports; 2002 else 2003 total_vports = num_vfs + MLX5_SPECIAL_VPORTS(esw->dev); 2004 2005 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 2006 mutex_init(&esw->fdb_table.offloads.vports.lock); 2007 hash_init(esw->fdb_table.offloads.vports.table); 2008 2009 err = esw_create_uplink_offloads_acl_tables(esw); 2010 if (err) 2011 goto create_acl_err; 2012 2013 err = esw_create_offloads_table(esw, total_vports); 2014 if (err) 2015 goto create_offloads_err; 2016 2017 err = esw_create_restore_table(esw); 2018 if (err) 2019 goto create_restore_err; 2020 2021 err = esw_create_offloads_fdb_tables(esw, total_vports); 2022 if (err) 2023 goto create_fdb_err; 2024 2025 err = esw_create_vport_rx_group(esw, total_vports); 2026 if (err) 2027 goto create_fg_err; 2028 2029 return 0; 2030 2031 create_fg_err: 2032 esw_destroy_offloads_fdb_tables(esw); 2033 create_fdb_err: 2034 esw_destroy_restore_table(esw); 2035 create_restore_err: 2036 esw_destroy_offloads_table(esw); 2037 create_offloads_err: 2038 esw_destroy_uplink_offloads_acl_tables(esw); 2039 create_acl_err: 2040 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 2041 return err; 2042 } 2043 2044 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 2045 { 2046 esw_destroy_vport_rx_group(esw); 2047 esw_destroy_offloads_fdb_tables(esw); 2048 esw_destroy_restore_table(esw); 2049 esw_destroy_offloads_table(esw); 2050 esw_destroy_uplink_offloads_acl_tables(esw); 2051 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 2052 } 2053 2054 static void 2055 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 2056 { 2057 bool host_pf_disabled; 2058 u16 new_num_vfs; 2059 2060 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 2061 host_params_context.host_num_of_vfs); 2062 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 2063 host_params_context.host_pf_disabled); 2064 2065 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 2066 return; 2067 2068 /* Number of VFs can only change from "0 to x" or "x to 0". */ 2069 if (esw->esw_funcs.num_vfs > 0) { 2070 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 2071 } else { 2072 int err; 2073 2074 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 2075 MLX5_VPORT_UC_ADDR_CHANGE); 2076 if (err) 2077 return; 2078 } 2079 esw->esw_funcs.num_vfs = new_num_vfs; 2080 } 2081 2082 static void esw_functions_changed_event_handler(struct work_struct *work) 2083 { 2084 struct mlx5_host_work *host_work; 2085 struct mlx5_eswitch *esw; 2086 const u32 *out; 2087 2088 host_work = container_of(work, struct mlx5_host_work, work); 2089 esw = host_work->esw; 2090 2091 out = mlx5_esw_query_functions(esw->dev); 2092 if (IS_ERR(out)) 2093 goto out; 2094 2095 esw_vfs_changed_event_handler(esw, out); 2096 kvfree(out); 2097 out: 2098 kfree(host_work); 2099 } 2100 2101 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 2102 { 2103 struct mlx5_esw_functions *esw_funcs; 2104 struct mlx5_host_work *host_work; 2105 struct mlx5_eswitch *esw; 2106 2107 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 2108 if (!host_work) 2109 return NOTIFY_DONE; 2110 2111 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 2112 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 2113 2114 host_work->esw = esw; 2115 2116 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 2117 queue_work(esw->work_queue, &host_work->work); 2118 2119 return NOTIFY_OK; 2120 } 2121 2122 int esw_offloads_enable(struct mlx5_eswitch *esw) 2123 { 2124 struct mlx5_vport *vport; 2125 int err, i; 2126 2127 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat) && 2128 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, decap)) 2129 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC; 2130 else 2131 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE; 2132 2133 mutex_init(&esw->offloads.termtbl_mutex); 2134 mlx5_rdma_enable_roce(esw->dev); 2135 2136 err = esw_set_passing_vport_metadata(esw, true); 2137 if (err) 2138 goto err_vport_metadata; 2139 2140 err = esw_offloads_steering_init(esw); 2141 if (err) 2142 goto err_steering_init; 2143 2144 /* Representor will control the vport link state */ 2145 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 2146 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 2147 2148 /* Uplink vport rep must load first. */ 2149 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); 2150 if (err) 2151 goto err_uplink; 2152 2153 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 2154 if (err) 2155 goto err_vports; 2156 2157 esw_offloads_devcom_init(esw); 2158 2159 return 0; 2160 2161 err_vports: 2162 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 2163 err_uplink: 2164 esw_offloads_steering_cleanup(esw); 2165 err_steering_init: 2166 esw_set_passing_vport_metadata(esw, false); 2167 err_vport_metadata: 2168 mlx5_rdma_disable_roce(esw->dev); 2169 mutex_destroy(&esw->offloads.termtbl_mutex); 2170 return err; 2171 } 2172 2173 static int esw_offloads_stop(struct mlx5_eswitch *esw, 2174 struct netlink_ext_ack *extack) 2175 { 2176 int err, err1; 2177 2178 mlx5_eswitch_disable_locked(esw, false); 2179 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, 2180 MLX5_ESWITCH_IGNORE_NUM_VFS); 2181 if (err) { 2182 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 2183 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, 2184 MLX5_ESWITCH_IGNORE_NUM_VFS); 2185 if (err1) { 2186 NL_SET_ERR_MSG_MOD(extack, 2187 "Failed setting eswitch back to offloads"); 2188 } 2189 } 2190 2191 return err; 2192 } 2193 2194 void esw_offloads_disable(struct mlx5_eswitch *esw) 2195 { 2196 esw_offloads_devcom_cleanup(esw); 2197 mlx5_eswitch_disable_pf_vf_vports(esw); 2198 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 2199 esw_set_passing_vport_metadata(esw, false); 2200 esw_offloads_steering_cleanup(esw); 2201 mlx5_rdma_disable_roce(esw->dev); 2202 mutex_destroy(&esw->offloads.termtbl_mutex); 2203 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE; 2204 } 2205 2206 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 2207 { 2208 switch (mode) { 2209 case DEVLINK_ESWITCH_MODE_LEGACY: 2210 *mlx5_mode = MLX5_ESWITCH_LEGACY; 2211 break; 2212 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 2213 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 2214 break; 2215 default: 2216 return -EINVAL; 2217 } 2218 2219 return 0; 2220 } 2221 2222 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 2223 { 2224 switch (mlx5_mode) { 2225 case MLX5_ESWITCH_LEGACY: 2226 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 2227 break; 2228 case MLX5_ESWITCH_OFFLOADS: 2229 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 2230 break; 2231 default: 2232 return -EINVAL; 2233 } 2234 2235 return 0; 2236 } 2237 2238 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 2239 { 2240 switch (mode) { 2241 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 2242 *mlx5_mode = MLX5_INLINE_MODE_NONE; 2243 break; 2244 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 2245 *mlx5_mode = MLX5_INLINE_MODE_L2; 2246 break; 2247 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 2248 *mlx5_mode = MLX5_INLINE_MODE_IP; 2249 break; 2250 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 2251 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 2252 break; 2253 default: 2254 return -EINVAL; 2255 } 2256 2257 return 0; 2258 } 2259 2260 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 2261 { 2262 switch (mlx5_mode) { 2263 case MLX5_INLINE_MODE_NONE: 2264 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 2265 break; 2266 case MLX5_INLINE_MODE_L2: 2267 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 2268 break; 2269 case MLX5_INLINE_MODE_IP: 2270 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 2271 break; 2272 case MLX5_INLINE_MODE_TCP_UDP: 2273 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 2274 break; 2275 default: 2276 return -EINVAL; 2277 } 2278 2279 return 0; 2280 } 2281 2282 static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) 2283 { 2284 /* devlink commands in NONE eswitch mode are currently supported only 2285 * on ECPF. 2286 */ 2287 return (esw->mode == MLX5_ESWITCH_NONE && 2288 !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0; 2289 } 2290 2291 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 2292 struct netlink_ext_ack *extack) 2293 { 2294 u16 cur_mlx5_mode, mlx5_mode = 0; 2295 struct mlx5_eswitch *esw; 2296 int err; 2297 2298 esw = mlx5_devlink_eswitch_get(devlink); 2299 if (IS_ERR(esw)) 2300 return PTR_ERR(esw); 2301 2302 if (esw_mode_from_devlink(mode, &mlx5_mode)) 2303 return -EINVAL; 2304 2305 mutex_lock(&esw->mode_lock); 2306 err = eswitch_devlink_esw_mode_check(esw); 2307 if (err) 2308 goto unlock; 2309 2310 cur_mlx5_mode = esw->mode; 2311 2312 if (cur_mlx5_mode == mlx5_mode) 2313 goto unlock; 2314 2315 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) 2316 err = esw_offloads_start(esw, extack); 2317 else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) 2318 err = esw_offloads_stop(esw, extack); 2319 else 2320 err = -EINVAL; 2321 2322 unlock: 2323 mutex_unlock(&esw->mode_lock); 2324 return err; 2325 } 2326 2327 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 2328 { 2329 struct mlx5_eswitch *esw; 2330 int err; 2331 2332 esw = mlx5_devlink_eswitch_get(devlink); 2333 if (IS_ERR(esw)) 2334 return PTR_ERR(esw); 2335 2336 mutex_lock(&esw->mode_lock); 2337 err = eswitch_devlink_esw_mode_check(esw); 2338 if (err) 2339 goto unlock; 2340 2341 err = esw_mode_to_devlink(esw->mode, mode); 2342 unlock: 2343 mutex_unlock(&esw->mode_lock); 2344 return err; 2345 } 2346 2347 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 2348 struct netlink_ext_ack *extack) 2349 { 2350 struct mlx5_core_dev *dev = devlink_priv(devlink); 2351 int err, vport, num_vport; 2352 struct mlx5_eswitch *esw; 2353 u8 mlx5_mode; 2354 2355 esw = mlx5_devlink_eswitch_get(devlink); 2356 if (IS_ERR(esw)) 2357 return PTR_ERR(esw); 2358 2359 mutex_lock(&esw->mode_lock); 2360 err = eswitch_devlink_esw_mode_check(esw); 2361 if (err) 2362 goto out; 2363 2364 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2365 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2366 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) 2367 goto out; 2368 /* fall through */ 2369 case MLX5_CAP_INLINE_MODE_L2: 2370 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 2371 err = -EOPNOTSUPP; 2372 goto out; 2373 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2374 break; 2375 } 2376 2377 if (atomic64_read(&esw->offloads.num_flows) > 0) { 2378 NL_SET_ERR_MSG_MOD(extack, 2379 "Can't set inline mode when flows are configured"); 2380 err = -EOPNOTSUPP; 2381 goto out; 2382 } 2383 2384 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 2385 if (err) 2386 goto out; 2387 2388 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) { 2389 err = mlx5_modify_nic_vport_min_inline(dev, vport, mlx5_mode); 2390 if (err) { 2391 NL_SET_ERR_MSG_MOD(extack, 2392 "Failed to set min inline on vport"); 2393 goto revert_inline_mode; 2394 } 2395 } 2396 2397 esw->offloads.inline_mode = mlx5_mode; 2398 mutex_unlock(&esw->mode_lock); 2399 return 0; 2400 2401 revert_inline_mode: 2402 num_vport = --vport; 2403 mlx5_esw_for_each_host_func_vport_reverse(esw, vport, num_vport) 2404 mlx5_modify_nic_vport_min_inline(dev, 2405 vport, 2406 esw->offloads.inline_mode); 2407 out: 2408 mutex_unlock(&esw->mode_lock); 2409 return err; 2410 } 2411 2412 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 2413 { 2414 struct mlx5_eswitch *esw; 2415 int err; 2416 2417 esw = mlx5_devlink_eswitch_get(devlink); 2418 if (IS_ERR(esw)) 2419 return PTR_ERR(esw); 2420 2421 mutex_lock(&esw->mode_lock); 2422 err = eswitch_devlink_esw_mode_check(esw); 2423 if (err) 2424 goto unlock; 2425 2426 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 2427 unlock: 2428 mutex_unlock(&esw->mode_lock); 2429 return err; 2430 } 2431 2432 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 2433 enum devlink_eswitch_encap_mode encap, 2434 struct netlink_ext_ack *extack) 2435 { 2436 struct mlx5_core_dev *dev = devlink_priv(devlink); 2437 struct mlx5_eswitch *esw; 2438 int err; 2439 2440 esw = mlx5_devlink_eswitch_get(devlink); 2441 if (IS_ERR(esw)) 2442 return PTR_ERR(esw); 2443 2444 mutex_lock(&esw->mode_lock); 2445 err = eswitch_devlink_esw_mode_check(esw); 2446 if (err) 2447 goto unlock; 2448 2449 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 2450 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 2451 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 2452 err = -EOPNOTSUPP; 2453 goto unlock; 2454 } 2455 2456 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 2457 err = -EOPNOTSUPP; 2458 goto unlock; 2459 } 2460 2461 if (esw->mode == MLX5_ESWITCH_LEGACY) { 2462 esw->offloads.encap = encap; 2463 goto unlock; 2464 } 2465 2466 if (esw->offloads.encap == encap) 2467 goto unlock; 2468 2469 if (atomic64_read(&esw->offloads.num_flows) > 0) { 2470 NL_SET_ERR_MSG_MOD(extack, 2471 "Can't set encapsulation when flows are configured"); 2472 err = -EOPNOTSUPP; 2473 goto unlock; 2474 } 2475 2476 esw_destroy_offloads_fdb_tables(esw); 2477 2478 esw->offloads.encap = encap; 2479 2480 err = esw_create_offloads_fdb_tables(esw, esw->nvports); 2481 2482 if (err) { 2483 NL_SET_ERR_MSG_MOD(extack, 2484 "Failed re-creating fast FDB table"); 2485 esw->offloads.encap = !encap; 2486 (void)esw_create_offloads_fdb_tables(esw, esw->nvports); 2487 } 2488 2489 unlock: 2490 mutex_unlock(&esw->mode_lock); 2491 return err; 2492 } 2493 2494 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 2495 enum devlink_eswitch_encap_mode *encap) 2496 { 2497 struct mlx5_eswitch *esw; 2498 int err; 2499 2500 esw = mlx5_devlink_eswitch_get(devlink); 2501 if (IS_ERR(esw)) 2502 return PTR_ERR(esw); 2503 2504 2505 mutex_lock(&esw->mode_lock); 2506 err = eswitch_devlink_esw_mode_check(esw); 2507 if (err) 2508 goto unlock; 2509 2510 *encap = esw->offloads.encap; 2511 unlock: 2512 mutex_unlock(&esw->mode_lock); 2513 return 0; 2514 } 2515 2516 static bool 2517 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 2518 { 2519 /* Currently, only ECPF based device has representor for host PF. */ 2520 if (vport_num == MLX5_VPORT_PF && 2521 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 2522 return false; 2523 2524 if (vport_num == MLX5_VPORT_ECPF && 2525 !mlx5_ecpf_vport_exists(esw->dev)) 2526 return false; 2527 2528 return true; 2529 } 2530 2531 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 2532 const struct mlx5_eswitch_rep_ops *ops, 2533 u8 rep_type) 2534 { 2535 struct mlx5_eswitch_rep_data *rep_data; 2536 struct mlx5_eswitch_rep *rep; 2537 int i; 2538 2539 esw->offloads.rep_ops[rep_type] = ops; 2540 mlx5_esw_for_all_reps(esw, i, rep) { 2541 if (likely(mlx5_eswitch_vport_has_rep(esw, i))) { 2542 rep_data = &rep->rep_data[rep_type]; 2543 atomic_set(&rep_data->state, REP_REGISTERED); 2544 } 2545 } 2546 } 2547 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 2548 2549 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 2550 { 2551 struct mlx5_eswitch_rep *rep; 2552 int i; 2553 2554 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 2555 __unload_reps_all_vport(esw, rep_type); 2556 2557 mlx5_esw_for_all_reps(esw, i, rep) 2558 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2559 } 2560 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 2561 2562 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 2563 { 2564 struct mlx5_eswitch_rep *rep; 2565 2566 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 2567 return rep->rep_data[rep_type].priv; 2568 } 2569 2570 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 2571 u16 vport, 2572 u8 rep_type) 2573 { 2574 struct mlx5_eswitch_rep *rep; 2575 2576 rep = mlx5_eswitch_get_rep(esw, vport); 2577 2578 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2579 esw->offloads.rep_ops[rep_type]->get_proto_dev) 2580 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 2581 return NULL; 2582 } 2583 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 2584 2585 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 2586 { 2587 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 2588 } 2589 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 2590 2591 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 2592 u16 vport) 2593 { 2594 return mlx5_eswitch_get_rep(esw, vport); 2595 } 2596 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 2597 2598 bool mlx5_eswitch_is_vf_vport(const struct mlx5_eswitch *esw, u16 vport_num) 2599 { 2600 return vport_num >= MLX5_VPORT_FIRST_VF && 2601 vport_num <= esw->dev->priv.sriov.max_vfs; 2602 } 2603 2604 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 2605 { 2606 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 2607 } 2608 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 2609 2610 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 2611 { 2612 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 2613 } 2614 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 2615 2616 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 2617 u16 vport_num) 2618 { 2619 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 2620 2621 if (WARN_ON_ONCE(IS_ERR(vport))) 2622 return 0; 2623 2624 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 2625 } 2626 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 2627