1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/acl/ofld.h" 42 #include "rdma.h" 43 #include "en.h" 44 #include "fs_core.h" 45 #include "lib/devcom.h" 46 #include "lib/eq.h" 47 #include "lib/fs_chains.h" 48 #include "en_tc.h" 49 50 /* There are two match-all miss flows, one for unicast dst mac and 51 * one for multicast. 52 */ 53 #define MLX5_ESW_MISS_FLOWS (2) 54 #define UPLINK_REP_INDEX 0 55 56 /* Per vport tables */ 57 58 #define MLX5_ESW_VPORT_TABLE_SIZE 128 59 60 /* This struct is used as a key to the hash table and we need it to be packed 61 * so hash result is consistent 62 */ 63 struct mlx5_vport_key { 64 u32 chain; 65 u16 prio; 66 u16 vport; 67 u16 vhca_id; 68 } __packed; 69 70 struct mlx5_vport_tbl_attr { 71 u16 chain; 72 u16 prio; 73 u16 vport; 74 }; 75 76 struct mlx5_vport_table { 77 struct hlist_node hlist; 78 struct mlx5_flow_table *fdb; 79 u32 num_rules; 80 struct mlx5_vport_key key; 81 }; 82 83 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 84 85 static struct mlx5_flow_table * 86 esw_vport_tbl_create(struct mlx5_eswitch *esw, struct mlx5_flow_namespace *ns) 87 { 88 struct mlx5_flow_table_attr ft_attr = {}; 89 struct mlx5_flow_table *fdb; 90 91 ft_attr.autogroup.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS; 92 ft_attr.max_fte = MLX5_ESW_VPORT_TABLE_SIZE; 93 ft_attr.prio = FDB_PER_VPORT; 94 fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 95 if (IS_ERR(fdb)) { 96 esw_warn(esw->dev, "Failed to create per vport FDB Table err %ld\n", 97 PTR_ERR(fdb)); 98 } 99 100 return fdb; 101 } 102 103 static u32 flow_attr_to_vport_key(struct mlx5_eswitch *esw, 104 struct mlx5_vport_tbl_attr *attr, 105 struct mlx5_vport_key *key) 106 { 107 key->vport = attr->vport; 108 key->chain = attr->chain; 109 key->prio = attr->prio; 110 key->vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 111 return jhash(key, sizeof(*key), 0); 112 } 113 114 /* caller must hold vports.lock */ 115 static struct mlx5_vport_table * 116 esw_vport_tbl_lookup(struct mlx5_eswitch *esw, struct mlx5_vport_key *skey, u32 key) 117 { 118 struct mlx5_vport_table *e; 119 120 hash_for_each_possible(esw->fdb_table.offloads.vports.table, e, hlist, key) 121 if (!memcmp(&e->key, skey, sizeof(*skey))) 122 return e; 123 124 return NULL; 125 } 126 127 static void 128 esw_vport_tbl_put(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr) 129 { 130 struct mlx5_vport_table *e; 131 struct mlx5_vport_key key; 132 u32 hkey; 133 134 mutex_lock(&esw->fdb_table.offloads.vports.lock); 135 hkey = flow_attr_to_vport_key(esw, attr, &key); 136 e = esw_vport_tbl_lookup(esw, &key, hkey); 137 if (!e || --e->num_rules) 138 goto out; 139 140 hash_del(&e->hlist); 141 mlx5_destroy_flow_table(e->fdb); 142 kfree(e); 143 out: 144 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 145 } 146 147 static struct mlx5_flow_table * 148 esw_vport_tbl_get(struct mlx5_eswitch *esw, struct mlx5_vport_tbl_attr *attr) 149 { 150 struct mlx5_core_dev *dev = esw->dev; 151 struct mlx5_flow_namespace *ns; 152 struct mlx5_flow_table *fdb; 153 struct mlx5_vport_table *e; 154 struct mlx5_vport_key skey; 155 u32 hkey; 156 157 mutex_lock(&esw->fdb_table.offloads.vports.lock); 158 hkey = flow_attr_to_vport_key(esw, attr, &skey); 159 e = esw_vport_tbl_lookup(esw, &skey, hkey); 160 if (e) { 161 e->num_rules++; 162 goto out; 163 } 164 165 e = kzalloc(sizeof(*e), GFP_KERNEL); 166 if (!e) { 167 fdb = ERR_PTR(-ENOMEM); 168 goto err_alloc; 169 } 170 171 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 172 if (!ns) { 173 esw_warn(dev, "Failed to get FDB namespace\n"); 174 fdb = ERR_PTR(-ENOENT); 175 goto err_ns; 176 } 177 178 fdb = esw_vport_tbl_create(esw, ns); 179 if (IS_ERR(fdb)) 180 goto err_ns; 181 182 e->fdb = fdb; 183 e->num_rules = 1; 184 e->key = skey; 185 hash_add(esw->fdb_table.offloads.vports.table, &e->hlist, hkey); 186 out: 187 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 188 return e->fdb; 189 190 err_ns: 191 kfree(e); 192 err_alloc: 193 mutex_unlock(&esw->fdb_table.offloads.vports.lock); 194 return fdb; 195 } 196 197 int mlx5_esw_vport_tbl_get(struct mlx5_eswitch *esw) 198 { 199 struct mlx5_vport_tbl_attr attr; 200 struct mlx5_flow_table *fdb; 201 struct mlx5_vport *vport; 202 int i; 203 204 attr.chain = 0; 205 attr.prio = 1; 206 mlx5_esw_for_all_vports(esw, i, vport) { 207 attr.vport = vport->vport; 208 fdb = esw_vport_tbl_get(esw, &attr); 209 if (IS_ERR(fdb)) 210 goto out; 211 } 212 return 0; 213 214 out: 215 mlx5_esw_vport_tbl_put(esw); 216 return PTR_ERR(fdb); 217 } 218 219 void mlx5_esw_vport_tbl_put(struct mlx5_eswitch *esw) 220 { 221 struct mlx5_vport_tbl_attr attr; 222 struct mlx5_vport *vport; 223 int i; 224 225 attr.chain = 0; 226 attr.prio = 1; 227 mlx5_esw_for_all_vports(esw, i, vport) { 228 attr.vport = vport->vport; 229 esw_vport_tbl_put(esw, &attr); 230 } 231 } 232 233 /* End: Per vport tables */ 234 235 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 236 u16 vport_num) 237 { 238 int idx = mlx5_eswitch_vport_num_to_index(esw, vport_num); 239 240 WARN_ON(idx > esw->total_vports - 1); 241 return &esw->offloads.vport_reps[idx]; 242 } 243 244 static void 245 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 246 struct mlx5_flow_spec *spec, 247 struct mlx5_esw_flow_attr *attr) 248 { 249 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) && 250 attr && attr->in_rep) 251 spec->flow_context.flow_source = 252 attr->in_rep->vport == MLX5_VPORT_UPLINK ? 253 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 254 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 255 } 256 257 static void 258 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 259 struct mlx5_flow_spec *spec, 260 struct mlx5_esw_flow_attr *attr) 261 { 262 void *misc2; 263 void *misc; 264 265 /* Use metadata matching because vport is not represented by single 266 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 267 */ 268 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 269 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 270 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 271 mlx5_eswitch_get_vport_metadata_for_match(attr->in_mdev->priv.eswitch, 272 attr->in_rep->vport)); 273 274 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 275 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 276 mlx5_eswitch_get_vport_metadata_mask()); 277 278 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 279 } else { 280 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 281 MLX5_SET(fte_match_set_misc, misc, source_port, attr->in_rep->vport); 282 283 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 284 MLX5_SET(fte_match_set_misc, misc, 285 source_eswitch_owner_vhca_id, 286 MLX5_CAP_GEN(attr->in_mdev, vhca_id)); 287 288 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 289 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 290 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 291 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 292 source_eswitch_owner_vhca_id); 293 294 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 295 } 296 } 297 298 struct mlx5_flow_handle * 299 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 300 struct mlx5_flow_spec *spec, 301 struct mlx5_flow_attr *attr) 302 { 303 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {}; 304 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 305 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 306 struct mlx5_fs_chains *chains = esw_chains(esw); 307 bool split = !!(esw_attr->split_count); 308 struct mlx5_vport_tbl_attr fwd_attr; 309 struct mlx5_flow_handle *rule; 310 struct mlx5_flow_table *fdb; 311 int j, i = 0; 312 313 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 314 return ERR_PTR(-EOPNOTSUPP); 315 316 flow_act.action = attr->action; 317 /* if per flow vlan pop/push is emulated, don't set that into the firmware */ 318 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 319 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH | 320 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 321 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 322 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 323 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 324 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 325 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 326 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 327 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 328 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 329 } 330 } 331 332 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 333 struct mlx5_flow_table *ft; 334 335 if (attr->dest_ft) { 336 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 337 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 338 dest[i].ft = attr->dest_ft; 339 i++; 340 } else if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) { 341 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 342 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 343 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 344 i++; 345 } else if (attr->dest_chain) { 346 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 347 ft = mlx5_chains_get_table(chains, attr->dest_chain, 348 1, 0); 349 if (IS_ERR(ft)) { 350 rule = ERR_CAST(ft); 351 goto err_create_goto_table; 352 } 353 354 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 355 dest[i].ft = ft; 356 i++; 357 } else { 358 for (j = esw_attr->split_count; j < esw_attr->out_count; j++) { 359 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 360 dest[i].vport.num = esw_attr->dests[j].rep->vport; 361 dest[i].vport.vhca_id = 362 MLX5_CAP_GEN(esw_attr->dests[j].mdev, vhca_id); 363 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 364 dest[i].vport.flags |= 365 MLX5_FLOW_DEST_VPORT_VHCA_ID; 366 if (esw_attr->dests[j].flags & MLX5_ESW_DEST_ENCAP) { 367 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 368 flow_act.pkt_reformat = 369 esw_attr->dests[j].pkt_reformat; 370 dest[i].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 371 dest[i].vport.pkt_reformat = 372 esw_attr->dests[j].pkt_reformat; 373 } 374 i++; 375 } 376 } 377 } 378 379 if (esw_attr->decap_pkt_reformat) 380 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 381 382 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 383 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 384 dest[i].counter_id = mlx5_fc_id(attr->counter); 385 i++; 386 } 387 388 if (attr->outer_match_level != MLX5_MATCH_NONE) 389 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 390 if (attr->inner_match_level != MLX5_MATCH_NONE) 391 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 392 393 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 394 flow_act.modify_hdr = attr->modify_hdr; 395 396 if (split) { 397 fwd_attr.chain = attr->chain; 398 fwd_attr.prio = attr->prio; 399 fwd_attr.vport = esw_attr->in_rep->vport; 400 401 fdb = esw_vport_tbl_get(esw, &fwd_attr); 402 } else { 403 if (attr->chain || attr->prio) 404 fdb = mlx5_chains_get_table(chains, attr->chain, 405 attr->prio, 0); 406 else 407 fdb = attr->ft; 408 409 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_NO_IN_PORT)) 410 mlx5_eswitch_set_rule_source_port(esw, spec, esw_attr); 411 } 412 if (IS_ERR(fdb)) { 413 rule = ERR_CAST(fdb); 414 goto err_esw_get; 415 } 416 417 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 418 419 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 420 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 421 &flow_act, dest, i); 422 else 423 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 424 if (IS_ERR(rule)) 425 goto err_add_rule; 426 else 427 atomic64_inc(&esw->offloads.num_flows); 428 429 return rule; 430 431 err_add_rule: 432 if (split) 433 esw_vport_tbl_put(esw, &fwd_attr); 434 else if (attr->chain || attr->prio) 435 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 436 err_esw_get: 437 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) && attr->dest_chain) 438 mlx5_chains_put_table(chains, attr->dest_chain, 1, 0); 439 err_create_goto_table: 440 return rule; 441 } 442 443 struct mlx5_flow_handle * 444 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 445 struct mlx5_flow_spec *spec, 446 struct mlx5_flow_attr *attr) 447 { 448 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {}; 449 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 450 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 451 struct mlx5_fs_chains *chains = esw_chains(esw); 452 struct mlx5_vport_tbl_attr fwd_attr; 453 struct mlx5_flow_table *fast_fdb; 454 struct mlx5_flow_table *fwd_fdb; 455 struct mlx5_flow_handle *rule; 456 int i; 457 458 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 459 if (IS_ERR(fast_fdb)) { 460 rule = ERR_CAST(fast_fdb); 461 goto err_get_fast; 462 } 463 464 fwd_attr.chain = attr->chain; 465 fwd_attr.prio = attr->prio; 466 fwd_attr.vport = esw_attr->in_rep->vport; 467 fwd_fdb = esw_vport_tbl_get(esw, &fwd_attr); 468 if (IS_ERR(fwd_fdb)) { 469 rule = ERR_CAST(fwd_fdb); 470 goto err_get_fwd; 471 } 472 473 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 474 for (i = 0; i < esw_attr->split_count; i++) { 475 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 476 dest[i].vport.num = esw_attr->dests[i].rep->vport; 477 dest[i].vport.vhca_id = 478 MLX5_CAP_GEN(esw_attr->dests[i].mdev, vhca_id); 479 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 480 dest[i].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 481 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_ENCAP) { 482 dest[i].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 483 dest[i].vport.pkt_reformat = esw_attr->dests[i].pkt_reformat; 484 } 485 } 486 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 487 dest[i].ft = fwd_fdb, 488 i++; 489 490 mlx5_eswitch_set_rule_source_port(esw, spec, esw_attr); 491 492 if (attr->outer_match_level != MLX5_MATCH_NONE) 493 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 494 495 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 496 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 497 498 if (IS_ERR(rule)) 499 goto add_err; 500 501 atomic64_inc(&esw->offloads.num_flows); 502 503 return rule; 504 add_err: 505 esw_vport_tbl_put(esw, &fwd_attr); 506 err_get_fwd: 507 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 508 err_get_fast: 509 return rule; 510 } 511 512 static void 513 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 514 struct mlx5_flow_handle *rule, 515 struct mlx5_flow_attr *attr, 516 bool fwd_rule) 517 { 518 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 519 struct mlx5_fs_chains *chains = esw_chains(esw); 520 bool split = (esw_attr->split_count > 0); 521 struct mlx5_vport_tbl_attr fwd_attr; 522 int i; 523 524 mlx5_del_flow_rules(rule); 525 526 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) { 527 /* unref the term table */ 528 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 529 if (esw_attr->dests[i].termtbl) 530 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 531 } 532 } 533 534 atomic64_dec(&esw->offloads.num_flows); 535 536 if (fwd_rule || split) { 537 fwd_attr.chain = attr->chain; 538 fwd_attr.prio = attr->prio; 539 fwd_attr.vport = esw_attr->in_rep->vport; 540 } 541 542 if (fwd_rule) { 543 esw_vport_tbl_put(esw, &fwd_attr); 544 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 545 } else { 546 if (split) 547 esw_vport_tbl_put(esw, &fwd_attr); 548 else if (attr->chain || attr->prio) 549 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 550 if (attr->dest_chain) 551 mlx5_chains_put_table(chains, attr->dest_chain, 1, 0); 552 } 553 } 554 555 void 556 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 557 struct mlx5_flow_handle *rule, 558 struct mlx5_flow_attr *attr) 559 { 560 __mlx5_eswitch_del_rule(esw, rule, attr, false); 561 } 562 563 void 564 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 565 struct mlx5_flow_handle *rule, 566 struct mlx5_flow_attr *attr) 567 { 568 __mlx5_eswitch_del_rule(esw, rule, attr, true); 569 } 570 571 static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val) 572 { 573 struct mlx5_eswitch_rep *rep; 574 int i, err = 0; 575 576 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none"); 577 mlx5_esw_for_each_host_func_rep(esw, i, rep, esw->esw_funcs.num_vfs) { 578 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 579 continue; 580 581 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val); 582 if (err) 583 goto out; 584 } 585 586 out: 587 return err; 588 } 589 590 static struct mlx5_eswitch_rep * 591 esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop) 592 { 593 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL; 594 595 in_rep = attr->in_rep; 596 out_rep = attr->dests[0].rep; 597 598 if (push) 599 vport = in_rep; 600 else if (pop) 601 vport = out_rep; 602 else 603 vport = in_rep; 604 605 return vport; 606 } 607 608 static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr, 609 bool push, bool pop, bool fwd) 610 { 611 struct mlx5_eswitch_rep *in_rep, *out_rep; 612 613 if ((push || pop) && !fwd) 614 goto out_notsupp; 615 616 in_rep = attr->in_rep; 617 out_rep = attr->dests[0].rep; 618 619 if (push && in_rep->vport == MLX5_VPORT_UPLINK) 620 goto out_notsupp; 621 622 if (pop && out_rep->vport == MLX5_VPORT_UPLINK) 623 goto out_notsupp; 624 625 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */ 626 if (!push && !pop && fwd) 627 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK) 628 goto out_notsupp; 629 630 /* protects against (1) setting rules with different vlans to push and 631 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0) 632 */ 633 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0])) 634 goto out_notsupp; 635 636 return 0; 637 638 out_notsupp: 639 return -EOPNOTSUPP; 640 } 641 642 int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw, 643 struct mlx5_flow_attr *attr) 644 { 645 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 646 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 647 struct mlx5_eswitch_rep *vport = NULL; 648 bool push, pop, fwd; 649 int err = 0; 650 651 /* nop if we're on the vlan push/pop non emulation mode */ 652 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 653 return 0; 654 655 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 656 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 657 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) && 658 !attr->dest_chain); 659 660 mutex_lock(&esw->state_lock); 661 662 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd); 663 if (err) 664 goto unlock; 665 666 attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 667 668 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 669 670 if (!push && !pop && fwd) { 671 /* tracks VF --> wire rules without vlan push action */ 672 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) { 673 vport->vlan_refcount++; 674 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 675 } 676 677 goto unlock; 678 } 679 680 if (!push && !pop) 681 goto unlock; 682 683 if (!(offloads->vlan_push_pop_refcount)) { 684 /* it's the 1st vlan rule, apply global vlan pop policy */ 685 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP); 686 if (err) 687 goto out; 688 } 689 offloads->vlan_push_pop_refcount++; 690 691 if (push) { 692 if (vport->vlan_refcount) 693 goto skip_set_push; 694 695 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0], 696 0, SET_VLAN_INSERT | SET_VLAN_STRIP); 697 if (err) 698 goto out; 699 vport->vlan = esw_attr->vlan_vid[0]; 700 skip_set_push: 701 vport->vlan_refcount++; 702 } 703 out: 704 if (!err) 705 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED; 706 unlock: 707 mutex_unlock(&esw->state_lock); 708 return err; 709 } 710 711 int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, 712 struct mlx5_flow_attr *attr) 713 { 714 struct offloads_fdb *offloads = &esw->fdb_table.offloads; 715 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 716 struct mlx5_eswitch_rep *vport = NULL; 717 bool push, pop, fwd; 718 int err = 0; 719 720 /* nop if we're on the vlan push/pop non emulation mode */ 721 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 722 return 0; 723 724 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED)) 725 return 0; 726 727 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH); 728 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP); 729 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST); 730 731 mutex_lock(&esw->state_lock); 732 733 vport = esw_vlan_action_get_vport(esw_attr, push, pop); 734 735 if (!push && !pop && fwd) { 736 /* tracks VF --> wire rules without vlan push action */ 737 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) 738 vport->vlan_refcount--; 739 740 goto out; 741 } 742 743 if (push) { 744 vport->vlan_refcount--; 745 if (vport->vlan_refcount) 746 goto skip_unset_push; 747 748 vport->vlan = 0; 749 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, 750 0, 0, SET_VLAN_STRIP); 751 if (err) 752 goto out; 753 } 754 755 skip_unset_push: 756 offloads->vlan_push_pop_refcount--; 757 if (offloads->vlan_push_pop_refcount) 758 goto out; 759 760 /* no more vlan rules, stop global vlan pop policy */ 761 err = esw_set_global_vlan_pop(esw, 0); 762 763 out: 764 mutex_unlock(&esw->state_lock); 765 return err; 766 } 767 768 struct mlx5_flow_handle * 769 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, u16 vport, 770 u32 sqn) 771 { 772 struct mlx5_flow_act flow_act = {0}; 773 struct mlx5_flow_destination dest = {}; 774 struct mlx5_flow_handle *flow_rule; 775 struct mlx5_flow_spec *spec; 776 void *misc; 777 778 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 779 if (!spec) { 780 flow_rule = ERR_PTR(-ENOMEM); 781 goto out; 782 } 783 784 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 785 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 786 /* source vport is the esw manager */ 787 MLX5_SET(fte_match_set_misc, misc, source_port, esw->manager_vport); 788 789 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 790 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 791 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 792 793 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 794 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 795 dest.vport.num = vport; 796 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 797 798 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 799 spec, &flow_act, &dest, 1); 800 if (IS_ERR(flow_rule)) 801 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule)); 802 out: 803 kvfree(spec); 804 return flow_rule; 805 } 806 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 807 808 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 809 { 810 mlx5_del_flow_rules(rule); 811 } 812 813 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 814 { 815 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 816 MLX5_FDB_TO_VPORT_REG_C_1; 817 } 818 819 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 820 { 821 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 822 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 823 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 824 u8 curr, wanted; 825 int err; 826 827 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 828 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 829 return 0; 830 831 MLX5_SET(query_esw_vport_context_in, in, opcode, 832 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 833 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 834 if (err) 835 return err; 836 837 curr = MLX5_GET(query_esw_vport_context_out, out, 838 esw_vport_context.fdb_to_vport_reg_c_id); 839 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 840 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 841 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 842 843 if (enable) 844 curr |= wanted; 845 else 846 curr &= ~wanted; 847 848 MLX5_SET(modify_esw_vport_context_in, min, 849 esw_vport_context.fdb_to_vport_reg_c_id, curr); 850 MLX5_SET(modify_esw_vport_context_in, min, 851 field_select.fdb_to_vport_reg_c_id, 1); 852 853 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 854 if (!err) { 855 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 856 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 857 else 858 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 859 } 860 861 return err; 862 } 863 864 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 865 struct mlx5_core_dev *peer_dev, 866 struct mlx5_flow_spec *spec, 867 struct mlx5_flow_destination *dest) 868 { 869 void *misc; 870 871 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 872 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 873 misc_parameters_2); 874 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 875 mlx5_eswitch_get_vport_metadata_mask()); 876 877 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 878 } else { 879 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 880 misc_parameters); 881 882 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 883 MLX5_CAP_GEN(peer_dev, vhca_id)); 884 885 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 886 887 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 888 misc_parameters); 889 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 890 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 891 source_eswitch_owner_vhca_id); 892 } 893 894 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 895 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 896 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 897 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 898 } 899 900 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 901 struct mlx5_eswitch *peer_esw, 902 struct mlx5_flow_spec *spec, 903 u16 vport) 904 { 905 void *misc; 906 907 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 908 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 909 misc_parameters_2); 910 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 911 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 912 vport)); 913 } else { 914 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 915 misc_parameters); 916 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 917 } 918 } 919 920 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 921 struct mlx5_core_dev *peer_dev) 922 { 923 struct mlx5_flow_destination dest = {}; 924 struct mlx5_flow_act flow_act = {0}; 925 struct mlx5_flow_handle **flows; 926 struct mlx5_flow_handle *flow; 927 struct mlx5_flow_spec *spec; 928 /* total vports is the same for both e-switches */ 929 int nvports = esw->total_vports; 930 void *misc; 931 int err, i; 932 933 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 934 if (!spec) 935 return -ENOMEM; 936 937 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 938 939 flows = kvzalloc(nvports * sizeof(*flows), GFP_KERNEL); 940 if (!flows) { 941 err = -ENOMEM; 942 goto alloc_flows_err; 943 } 944 945 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 946 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 947 misc_parameters); 948 949 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 950 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 951 spec, MLX5_VPORT_PF); 952 953 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 954 spec, &flow_act, &dest, 1); 955 if (IS_ERR(flow)) { 956 err = PTR_ERR(flow); 957 goto add_pf_flow_err; 958 } 959 flows[MLX5_VPORT_PF] = flow; 960 } 961 962 if (mlx5_ecpf_vport_exists(esw->dev)) { 963 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 964 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 965 spec, &flow_act, &dest, 1); 966 if (IS_ERR(flow)) { 967 err = PTR_ERR(flow); 968 goto add_ecpf_flow_err; 969 } 970 flows[mlx5_eswitch_ecpf_idx(esw)] = flow; 971 } 972 973 mlx5_esw_for_each_vf_vport_num(esw, i, mlx5_core_max_vfs(esw->dev)) { 974 esw_set_peer_miss_rule_source_port(esw, 975 peer_dev->priv.eswitch, 976 spec, i); 977 978 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 979 spec, &flow_act, &dest, 1); 980 if (IS_ERR(flow)) { 981 err = PTR_ERR(flow); 982 goto add_vf_flow_err; 983 } 984 flows[i] = flow; 985 } 986 987 esw->fdb_table.offloads.peer_miss_rules = flows; 988 989 kvfree(spec); 990 return 0; 991 992 add_vf_flow_err: 993 nvports = --i; 994 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, nvports) 995 mlx5_del_flow_rules(flows[i]); 996 997 if (mlx5_ecpf_vport_exists(esw->dev)) 998 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]); 999 add_ecpf_flow_err: 1000 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 1001 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]); 1002 add_pf_flow_err: 1003 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1004 kvfree(flows); 1005 alloc_flows_err: 1006 kvfree(spec); 1007 return err; 1008 } 1009 1010 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw) 1011 { 1012 struct mlx5_flow_handle **flows; 1013 int i; 1014 1015 flows = esw->fdb_table.offloads.peer_miss_rules; 1016 1017 mlx5_esw_for_each_vf_vport_num_reverse(esw, i, 1018 mlx5_core_max_vfs(esw->dev)) 1019 mlx5_del_flow_rules(flows[i]); 1020 1021 if (mlx5_ecpf_vport_exists(esw->dev)) 1022 mlx5_del_flow_rules(flows[mlx5_eswitch_ecpf_idx(esw)]); 1023 1024 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) 1025 mlx5_del_flow_rules(flows[MLX5_VPORT_PF]); 1026 1027 kvfree(flows); 1028 } 1029 1030 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1031 { 1032 struct mlx5_flow_act flow_act = {0}; 1033 struct mlx5_flow_destination dest = {}; 1034 struct mlx5_flow_handle *flow_rule = NULL; 1035 struct mlx5_flow_spec *spec; 1036 void *headers_c; 1037 void *headers_v; 1038 int err = 0; 1039 u8 *dmac_c; 1040 u8 *dmac_v; 1041 1042 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1043 if (!spec) { 1044 err = -ENOMEM; 1045 goto out; 1046 } 1047 1048 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1049 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1050 outer_headers); 1051 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1052 outer_headers.dmac_47_16); 1053 dmac_c[0] = 0x01; 1054 1055 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1056 dest.vport.num = esw->manager_vport; 1057 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1058 1059 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1060 spec, &flow_act, &dest, 1); 1061 if (IS_ERR(flow_rule)) { 1062 err = PTR_ERR(flow_rule); 1063 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1064 goto out; 1065 } 1066 1067 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1068 1069 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1070 outer_headers); 1071 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1072 outer_headers.dmac_47_16); 1073 dmac_v[0] = 0x01; 1074 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1075 spec, &flow_act, &dest, 1); 1076 if (IS_ERR(flow_rule)) { 1077 err = PTR_ERR(flow_rule); 1078 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1079 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1080 goto out; 1081 } 1082 1083 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1084 1085 out: 1086 kvfree(spec); 1087 return err; 1088 } 1089 1090 struct mlx5_flow_handle * 1091 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1092 { 1093 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1094 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1095 struct mlx5_flow_context *flow_context; 1096 struct mlx5_flow_handle *flow_rule; 1097 struct mlx5_flow_destination dest; 1098 struct mlx5_flow_spec *spec; 1099 void *misc; 1100 1101 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1102 return ERR_PTR(-EOPNOTSUPP); 1103 1104 spec = kzalloc(sizeof(*spec), GFP_KERNEL); 1105 if (!spec) 1106 return ERR_PTR(-ENOMEM); 1107 1108 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1109 misc_parameters_2); 1110 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1111 ESW_CHAIN_TAG_METADATA_MASK); 1112 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1113 misc_parameters_2); 1114 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1115 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1116 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1117 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1118 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1119 1120 flow_context = &spec->flow_context; 1121 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1122 flow_context->flow_tag = tag; 1123 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1124 dest.ft = esw->offloads.ft_offloads; 1125 1126 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1127 kfree(spec); 1128 1129 if (IS_ERR(flow_rule)) 1130 esw_warn(esw->dev, 1131 "Failed to create restore rule for tag: %d, err(%d)\n", 1132 tag, (int)PTR_ERR(flow_rule)); 1133 1134 return flow_rule; 1135 } 1136 1137 u32 1138 esw_get_max_restore_tag(struct mlx5_eswitch *esw) 1139 { 1140 return ESW_CHAIN_TAG_METADATA_MASK; 1141 } 1142 1143 #define MAX_PF_SQ 256 1144 #define MAX_SQ_NVPORTS 32 1145 1146 static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1147 u32 *flow_group_in) 1148 { 1149 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1150 flow_group_in, 1151 match_criteria); 1152 1153 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1154 MLX5_SET(create_flow_group_in, flow_group_in, 1155 match_criteria_enable, 1156 MLX5_MATCH_MISC_PARAMETERS_2); 1157 1158 MLX5_SET(fte_match_param, match_criteria, 1159 misc_parameters_2.metadata_reg_c_0, 1160 mlx5_eswitch_get_vport_metadata_mask()); 1161 } else { 1162 MLX5_SET(create_flow_group_in, flow_group_in, 1163 match_criteria_enable, 1164 MLX5_MATCH_MISC_PARAMETERS); 1165 1166 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1167 misc_parameters.source_port); 1168 } 1169 } 1170 1171 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1172 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1173 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1174 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1175 { 1176 struct mlx5_core_dev *dev = esw->dev; 1177 1178 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1179 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1180 1181 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1182 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1183 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1184 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1185 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1186 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1187 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1188 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1189 /* Disabled when ttl workaround is needed, e.g 1190 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1191 */ 1192 esw_warn(dev, 1193 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1194 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1195 } else { 1196 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1197 esw_info(dev, "Supported tc chains and prios offload\n"); 1198 } 1199 1200 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1201 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1202 } 1203 1204 static int 1205 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1206 { 1207 struct mlx5_core_dev *dev = esw->dev; 1208 struct mlx5_flow_table *nf_ft, *ft; 1209 struct mlx5_chains_attr attr = {}; 1210 struct mlx5_fs_chains *chains; 1211 u32 fdb_max; 1212 int err; 1213 1214 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size); 1215 1216 esw_init_chains_offload_flags(esw, &attr.flags); 1217 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1218 attr.max_ft_sz = fdb_max; 1219 attr.max_grp_num = esw->params.large_group_num; 1220 attr.default_ft = miss_fdb; 1221 attr.max_restore_tag = esw_get_max_restore_tag(esw); 1222 1223 chains = mlx5_chains_create(dev, &attr); 1224 if (IS_ERR(chains)) { 1225 err = PTR_ERR(chains); 1226 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1227 return err; 1228 } 1229 1230 esw->fdb_table.offloads.esw_chains_priv = chains; 1231 1232 /* Create tc_end_ft which is the always created ft chain */ 1233 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1234 1, 0); 1235 if (IS_ERR(nf_ft)) { 1236 err = PTR_ERR(nf_ft); 1237 goto nf_ft_err; 1238 } 1239 1240 /* Always open the root for fast path */ 1241 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1242 if (IS_ERR(ft)) { 1243 err = PTR_ERR(ft); 1244 goto level_0_err; 1245 } 1246 1247 /* Open level 1 for split fdb rules now if prios isn't supported */ 1248 if (!mlx5_chains_prios_supported(chains)) { 1249 err = mlx5_esw_vport_tbl_get(esw); 1250 if (err) 1251 goto level_1_err; 1252 } 1253 1254 mlx5_chains_set_end_ft(chains, nf_ft); 1255 1256 return 0; 1257 1258 level_1_err: 1259 mlx5_chains_put_table(chains, 0, 1, 0); 1260 level_0_err: 1261 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1262 nf_ft_err: 1263 mlx5_chains_destroy(chains); 1264 esw->fdb_table.offloads.esw_chains_priv = NULL; 1265 1266 return err; 1267 } 1268 1269 static void 1270 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1271 { 1272 if (!mlx5_chains_prios_supported(chains)) 1273 mlx5_esw_vport_tbl_put(esw); 1274 mlx5_chains_put_table(chains, 0, 1, 0); 1275 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1276 mlx5_chains_destroy(chains); 1277 } 1278 1279 #else /* CONFIG_MLX5_CLS_ACT */ 1280 1281 static int 1282 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1283 { return 0; } 1284 1285 static void 1286 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1287 {} 1288 1289 #endif 1290 1291 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1292 { 1293 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1294 struct mlx5_flow_table_attr ft_attr = {}; 1295 struct mlx5_core_dev *dev = esw->dev; 1296 struct mlx5_flow_namespace *root_ns; 1297 struct mlx5_flow_table *fdb = NULL; 1298 u32 flags = 0, *flow_group_in; 1299 int table_size, ix, err = 0; 1300 struct mlx5_flow_group *g; 1301 void *match_criteria; 1302 u8 *dmac; 1303 1304 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1305 1306 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1307 if (!flow_group_in) 1308 return -ENOMEM; 1309 1310 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1311 if (!root_ns) { 1312 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1313 err = -EOPNOTSUPP; 1314 goto ns_err; 1315 } 1316 esw->fdb_table.offloads.ns = root_ns; 1317 err = mlx5_flow_namespace_set_mode(root_ns, 1318 esw->dev->priv.steering->mode); 1319 if (err) { 1320 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1321 goto ns_err; 1322 } 1323 1324 table_size = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1325 MLX5_ESW_MISS_FLOWS + esw->total_vports; 1326 1327 /* create the slow path fdb with encap set, so further table instances 1328 * can be created at run time while VFs are probed if the FW allows that. 1329 */ 1330 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1331 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1332 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1333 1334 ft_attr.flags = flags; 1335 ft_attr.max_fte = table_size; 1336 ft_attr.prio = FDB_SLOW_PATH; 1337 1338 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1339 if (IS_ERR(fdb)) { 1340 err = PTR_ERR(fdb); 1341 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1342 goto slow_fdb_err; 1343 } 1344 esw->fdb_table.offloads.slow_fdb = fdb; 1345 1346 err = esw_chains_create(esw, fdb); 1347 if (err) { 1348 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1349 goto fdb_chains_err; 1350 } 1351 1352 /* create send-to-vport group */ 1353 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1354 MLX5_MATCH_MISC_PARAMETERS); 1355 1356 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1357 1358 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1359 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port); 1360 1361 ix = esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ; 1362 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1363 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1); 1364 1365 g = mlx5_create_flow_group(fdb, flow_group_in); 1366 if (IS_ERR(g)) { 1367 err = PTR_ERR(g); 1368 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1369 goto send_vport_err; 1370 } 1371 esw->fdb_table.offloads.send_to_vport_grp = g; 1372 1373 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1374 /* create peer esw miss group */ 1375 memset(flow_group_in, 0, inlen); 1376 1377 esw_set_flow_group_source_port(esw, flow_group_in); 1378 1379 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1380 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1381 flow_group_in, 1382 match_criteria); 1383 1384 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1385 misc_parameters.source_eswitch_owner_vhca_id); 1386 1387 MLX5_SET(create_flow_group_in, flow_group_in, 1388 source_eswitch_owner_vhca_id_valid, 1); 1389 } 1390 1391 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1392 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1393 ix + esw->total_vports - 1); 1394 ix += esw->total_vports; 1395 1396 g = mlx5_create_flow_group(fdb, flow_group_in); 1397 if (IS_ERR(g)) { 1398 err = PTR_ERR(g); 1399 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err); 1400 goto peer_miss_err; 1401 } 1402 esw->fdb_table.offloads.peer_miss_grp = g; 1403 } 1404 1405 /* create miss group */ 1406 memset(flow_group_in, 0, inlen); 1407 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1408 MLX5_MATCH_OUTER_HEADERS); 1409 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1410 match_criteria); 1411 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1412 outer_headers.dmac_47_16); 1413 dmac[0] = 0x01; 1414 1415 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix); 1416 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1417 ix + MLX5_ESW_MISS_FLOWS); 1418 1419 g = mlx5_create_flow_group(fdb, flow_group_in); 1420 if (IS_ERR(g)) { 1421 err = PTR_ERR(g); 1422 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err); 1423 goto miss_err; 1424 } 1425 esw->fdb_table.offloads.miss_grp = g; 1426 1427 err = esw_add_fdb_miss_rule(esw); 1428 if (err) 1429 goto miss_rule_err; 1430 1431 kvfree(flow_group_in); 1432 return 0; 1433 1434 miss_rule_err: 1435 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1436 miss_err: 1437 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1438 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1439 peer_miss_err: 1440 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1441 send_vport_err: 1442 esw_chains_destroy(esw, esw_chains(esw)); 1443 fdb_chains_err: 1444 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1445 slow_fdb_err: 1446 /* Holds true only as long as DMFS is the default */ 1447 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1448 ns_err: 1449 kvfree(flow_group_in); 1450 return err; 1451 } 1452 1453 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1454 { 1455 if (!esw->fdb_table.offloads.slow_fdb) 1456 return; 1457 1458 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1459 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1460 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1461 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1462 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1463 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1464 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1465 1466 esw_chains_destroy(esw, esw_chains(esw)); 1467 1468 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb); 1469 /* Holds true only as long as DMFS is the default */ 1470 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1471 MLX5_FLOW_STEERING_MODE_DMFS); 1472 } 1473 1474 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1475 { 1476 struct mlx5_flow_table_attr ft_attr = {}; 1477 struct mlx5_core_dev *dev = esw->dev; 1478 struct mlx5_flow_table *ft_offloads; 1479 struct mlx5_flow_namespace *ns; 1480 int err = 0; 1481 1482 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1483 if (!ns) { 1484 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1485 return -EOPNOTSUPP; 1486 } 1487 1488 ft_attr.max_fte = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1489 ft_attr.prio = 1; 1490 1491 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 1492 if (IS_ERR(ft_offloads)) { 1493 err = PTR_ERR(ft_offloads); 1494 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 1495 return err; 1496 } 1497 1498 esw->offloads.ft_offloads = ft_offloads; 1499 return 0; 1500 } 1501 1502 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 1503 { 1504 struct mlx5_esw_offload *offloads = &esw->offloads; 1505 1506 mlx5_destroy_flow_table(offloads->ft_offloads); 1507 } 1508 1509 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 1510 { 1511 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1512 struct mlx5_flow_group *g; 1513 u32 *flow_group_in; 1514 int nvports; 1515 int err = 0; 1516 1517 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1518 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1519 if (!flow_group_in) 1520 return -ENOMEM; 1521 1522 /* create vport rx group */ 1523 esw_set_flow_group_source_port(esw, flow_group_in); 1524 1525 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1526 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 1527 1528 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 1529 1530 if (IS_ERR(g)) { 1531 err = PTR_ERR(g); 1532 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 1533 goto out; 1534 } 1535 1536 esw->offloads.vport_rx_group = g; 1537 out: 1538 kvfree(flow_group_in); 1539 return err; 1540 } 1541 1542 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 1543 { 1544 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 1545 } 1546 1547 struct mlx5_flow_handle * 1548 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 1549 struct mlx5_flow_destination *dest) 1550 { 1551 struct mlx5_flow_act flow_act = {0}; 1552 struct mlx5_flow_handle *flow_rule; 1553 struct mlx5_flow_spec *spec; 1554 void *misc; 1555 1556 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1557 if (!spec) { 1558 flow_rule = ERR_PTR(-ENOMEM); 1559 goto out; 1560 } 1561 1562 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1563 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 1564 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1565 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 1566 1567 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 1568 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1569 mlx5_eswitch_get_vport_metadata_mask()); 1570 1571 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1572 } else { 1573 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 1574 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1575 1576 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 1577 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1578 1579 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1580 } 1581 1582 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1583 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 1584 &flow_act, dest, 1); 1585 if (IS_ERR(flow_rule)) { 1586 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 1587 goto out; 1588 } 1589 1590 out: 1591 kvfree(spec); 1592 return flow_rule; 1593 } 1594 1595 1596 static int mlx5_eswitch_inline_mode_get(const struct mlx5_eswitch *esw, u8 *mode) 1597 { 1598 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 1599 struct mlx5_core_dev *dev = esw->dev; 1600 int vport; 1601 1602 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 1603 return -EOPNOTSUPP; 1604 1605 if (esw->mode == MLX5_ESWITCH_NONE) 1606 return -EOPNOTSUPP; 1607 1608 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 1609 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 1610 mlx5_mode = MLX5_INLINE_MODE_NONE; 1611 goto out; 1612 case MLX5_CAP_INLINE_MODE_L2: 1613 mlx5_mode = MLX5_INLINE_MODE_L2; 1614 goto out; 1615 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 1616 goto query_vports; 1617 } 1618 1619 query_vports: 1620 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 1621 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) { 1622 mlx5_query_nic_vport_min_inline(dev, vport, &mlx5_mode); 1623 if (prev_mlx5_mode != mlx5_mode) 1624 return -EINVAL; 1625 prev_mlx5_mode = mlx5_mode; 1626 } 1627 1628 out: 1629 *mode = mlx5_mode; 1630 return 0; 1631 } 1632 1633 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 1634 { 1635 struct mlx5_esw_offload *offloads = &esw->offloads; 1636 1637 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1638 return; 1639 1640 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 1641 mlx5_destroy_flow_group(offloads->restore_group); 1642 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 1643 } 1644 1645 static int esw_create_restore_table(struct mlx5_eswitch *esw) 1646 { 1647 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1648 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1649 struct mlx5_flow_table_attr ft_attr = {}; 1650 struct mlx5_core_dev *dev = esw->dev; 1651 struct mlx5_flow_namespace *ns; 1652 struct mlx5_modify_hdr *mod_hdr; 1653 void *match_criteria, *misc; 1654 struct mlx5_flow_table *ft; 1655 struct mlx5_flow_group *g; 1656 u32 *flow_group_in; 1657 int err = 0; 1658 1659 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1660 return 0; 1661 1662 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1663 if (!ns) { 1664 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1665 return -EOPNOTSUPP; 1666 } 1667 1668 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1669 if (!flow_group_in) { 1670 err = -ENOMEM; 1671 goto out_free; 1672 } 1673 1674 ft_attr.max_fte = 1 << ESW_CHAIN_TAG_METADATA_BITS; 1675 ft = mlx5_create_flow_table(ns, &ft_attr); 1676 if (IS_ERR(ft)) { 1677 err = PTR_ERR(ft); 1678 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 1679 err); 1680 goto out_free; 1681 } 1682 1683 memset(flow_group_in, 0, inlen); 1684 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1685 match_criteria); 1686 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 1687 misc_parameters_2); 1688 1689 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1690 ESW_CHAIN_TAG_METADATA_MASK); 1691 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1692 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1693 ft_attr.max_fte - 1); 1694 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1695 MLX5_MATCH_MISC_PARAMETERS_2); 1696 g = mlx5_create_flow_group(ft, flow_group_in); 1697 if (IS_ERR(g)) { 1698 err = PTR_ERR(g); 1699 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 1700 err); 1701 goto err_group; 1702 } 1703 1704 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 1705 MLX5_SET(copy_action_in, modact, src_field, 1706 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 1707 MLX5_SET(copy_action_in, modact, dst_field, 1708 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1709 mod_hdr = mlx5_modify_header_alloc(esw->dev, 1710 MLX5_FLOW_NAMESPACE_KERNEL, 1, 1711 modact); 1712 if (IS_ERR(mod_hdr)) { 1713 err = PTR_ERR(mod_hdr); 1714 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 1715 err); 1716 goto err_mod_hdr; 1717 } 1718 1719 esw->offloads.ft_offloads_restore = ft; 1720 esw->offloads.restore_group = g; 1721 esw->offloads.restore_copy_hdr_id = mod_hdr; 1722 1723 kvfree(flow_group_in); 1724 1725 return 0; 1726 1727 err_mod_hdr: 1728 mlx5_destroy_flow_group(g); 1729 err_group: 1730 mlx5_destroy_flow_table(ft); 1731 out_free: 1732 kvfree(flow_group_in); 1733 1734 return err; 1735 } 1736 1737 static int esw_offloads_start(struct mlx5_eswitch *esw, 1738 struct netlink_ext_ack *extack) 1739 { 1740 int err, err1; 1741 1742 mlx5_eswitch_disable_locked(esw, false); 1743 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, 1744 esw->dev->priv.sriov.num_vfs); 1745 if (err) { 1746 NL_SET_ERR_MSG_MOD(extack, 1747 "Failed setting eswitch to offloads"); 1748 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, 1749 MLX5_ESWITCH_IGNORE_NUM_VFS); 1750 if (err1) { 1751 NL_SET_ERR_MSG_MOD(extack, 1752 "Failed setting eswitch back to legacy"); 1753 } 1754 } 1755 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 1756 if (mlx5_eswitch_inline_mode_get(esw, 1757 &esw->offloads.inline_mode)) { 1758 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 1759 NL_SET_ERR_MSG_MOD(extack, 1760 "Inline mode is different between vports"); 1761 } 1762 } 1763 return err; 1764 } 1765 1766 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 1767 { 1768 kfree(esw->offloads.vport_reps); 1769 } 1770 1771 int esw_offloads_init_reps(struct mlx5_eswitch *esw) 1772 { 1773 int total_vports = esw->total_vports; 1774 struct mlx5_eswitch_rep *rep; 1775 int vport_index; 1776 u8 rep_type; 1777 1778 esw->offloads.vport_reps = kcalloc(total_vports, 1779 sizeof(struct mlx5_eswitch_rep), 1780 GFP_KERNEL); 1781 if (!esw->offloads.vport_reps) 1782 return -ENOMEM; 1783 1784 mlx5_esw_for_all_reps(esw, vport_index, rep) { 1785 rep->vport = mlx5_eswitch_index_to_vport_num(esw, vport_index); 1786 rep->vport_index = vport_index; 1787 1788 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 1789 atomic_set(&rep->rep_data[rep_type].state, 1790 REP_UNREGISTERED); 1791 } 1792 1793 return 0; 1794 } 1795 1796 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 1797 struct mlx5_eswitch_rep *rep, u8 rep_type) 1798 { 1799 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 1800 REP_LOADED, REP_REGISTERED) == REP_LOADED) 1801 esw->offloads.rep_ops[rep_type]->unload(rep); 1802 } 1803 1804 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 1805 { 1806 struct mlx5_eswitch_rep *rep; 1807 int i; 1808 1809 mlx5_esw_for_each_vf_rep_reverse(esw, i, rep, esw->esw_funcs.num_vfs) 1810 __esw_offloads_unload_rep(esw, rep, rep_type); 1811 1812 if (mlx5_ecpf_vport_exists(esw->dev)) { 1813 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF); 1814 __esw_offloads_unload_rep(esw, rep, rep_type); 1815 } 1816 1817 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1818 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF); 1819 __esw_offloads_unload_rep(esw, rep, rep_type); 1820 } 1821 1822 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 1823 __esw_offloads_unload_rep(esw, rep, rep_type); 1824 } 1825 1826 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 1827 { 1828 struct mlx5_eswitch_rep *rep; 1829 int rep_type; 1830 int err; 1831 1832 rep = mlx5_eswitch_get_rep(esw, vport_num); 1833 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 1834 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 1835 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 1836 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 1837 if (err) 1838 goto err_reps; 1839 } 1840 1841 return 0; 1842 1843 err_reps: 1844 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 1845 for (--rep_type; rep_type >= 0; rep_type--) 1846 __esw_offloads_unload_rep(esw, rep, rep_type); 1847 return err; 1848 } 1849 1850 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 1851 { 1852 struct mlx5_eswitch_rep *rep; 1853 int rep_type; 1854 1855 rep = mlx5_eswitch_get_rep(esw, vport_num); 1856 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 1857 __esw_offloads_unload_rep(esw, rep, rep_type); 1858 } 1859 1860 int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num) 1861 { 1862 int err; 1863 1864 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 1865 return 0; 1866 1867 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num); 1868 if (err) 1869 return err; 1870 1871 err = mlx5_esw_offloads_rep_load(esw, vport_num); 1872 if (err) 1873 goto load_err; 1874 return err; 1875 1876 load_err: 1877 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 1878 return err; 1879 } 1880 1881 void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num) 1882 { 1883 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 1884 return; 1885 1886 mlx5_esw_offloads_rep_unload(esw, vport_num); 1887 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num); 1888 } 1889 1890 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 1891 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 1892 1893 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 1894 struct mlx5_eswitch *peer_esw) 1895 { 1896 int err; 1897 1898 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 1899 if (err) 1900 return err; 1901 1902 return 0; 1903 } 1904 1905 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw) 1906 { 1907 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1908 mlx5e_tc_clean_fdb_peer_flows(esw); 1909 #endif 1910 esw_del_fdb_peer_miss_rules(esw); 1911 } 1912 1913 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 1914 struct mlx5_eswitch *peer_esw, 1915 bool pair) 1916 { 1917 struct mlx5_flow_root_namespace *peer_ns; 1918 struct mlx5_flow_root_namespace *ns; 1919 int err; 1920 1921 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 1922 ns = esw->dev->priv.steering->fdb_root_ns; 1923 1924 if (pair) { 1925 err = mlx5_flow_namespace_set_peer(ns, peer_ns); 1926 if (err) 1927 return err; 1928 1929 err = mlx5_flow_namespace_set_peer(peer_ns, ns); 1930 if (err) { 1931 mlx5_flow_namespace_set_peer(ns, NULL); 1932 return err; 1933 } 1934 } else { 1935 mlx5_flow_namespace_set_peer(ns, NULL); 1936 mlx5_flow_namespace_set_peer(peer_ns, NULL); 1937 } 1938 1939 return 0; 1940 } 1941 1942 static int mlx5_esw_offloads_devcom_event(int event, 1943 void *my_data, 1944 void *event_data) 1945 { 1946 struct mlx5_eswitch *esw = my_data; 1947 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 1948 struct mlx5_eswitch *peer_esw = event_data; 1949 int err; 1950 1951 switch (event) { 1952 case ESW_OFFLOADS_DEVCOM_PAIR: 1953 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 1954 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 1955 break; 1956 1957 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 1958 if (err) 1959 goto err_out; 1960 err = mlx5_esw_offloads_pair(esw, peer_esw); 1961 if (err) 1962 goto err_peer; 1963 1964 err = mlx5_esw_offloads_pair(peer_esw, esw); 1965 if (err) 1966 goto err_pair; 1967 1968 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true); 1969 break; 1970 1971 case ESW_OFFLOADS_DEVCOM_UNPAIR: 1972 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS)) 1973 break; 1974 1975 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false); 1976 mlx5_esw_offloads_unpair(peer_esw); 1977 mlx5_esw_offloads_unpair(esw); 1978 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 1979 break; 1980 } 1981 1982 return 0; 1983 1984 err_pair: 1985 mlx5_esw_offloads_unpair(esw); 1986 err_peer: 1987 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 1988 err_out: 1989 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 1990 event, err); 1991 return err; 1992 } 1993 1994 static void esw_offloads_devcom_init(struct mlx5_eswitch *esw) 1995 { 1996 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 1997 1998 INIT_LIST_HEAD(&esw->offloads.peer_flows); 1999 mutex_init(&esw->offloads.peer_mutex); 2000 2001 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2002 return; 2003 2004 mlx5_devcom_register_component(devcom, 2005 MLX5_DEVCOM_ESW_OFFLOADS, 2006 mlx5_esw_offloads_devcom_event, 2007 esw); 2008 2009 mlx5_devcom_send_event(devcom, 2010 MLX5_DEVCOM_ESW_OFFLOADS, 2011 ESW_OFFLOADS_DEVCOM_PAIR, esw); 2012 } 2013 2014 static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 2015 { 2016 struct mlx5_devcom *devcom = esw->dev->priv.devcom; 2017 2018 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 2019 return; 2020 2021 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS, 2022 ESW_OFFLOADS_DEVCOM_UNPAIR, esw); 2023 2024 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS); 2025 } 2026 2027 static bool 2028 esw_check_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 2029 { 2030 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 2031 return false; 2032 2033 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 2034 MLX5_FDB_TO_VPORT_REG_C_0)) 2035 return false; 2036 2037 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source)) 2038 return false; 2039 2040 if (mlx5_core_is_ecpf_esw_manager(esw->dev) || 2041 mlx5_ecpf_vport_exists(esw->dev)) 2042 return false; 2043 2044 return true; 2045 } 2046 2047 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 2048 { 2049 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 2050 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 1; 2051 u32 pf_num; 2052 int id; 2053 2054 /* Only 4 bits of pf_num */ 2055 pf_num = PCI_FUNC(esw->dev->pdev->devfn); 2056 if (pf_num > max_pf_num) 2057 return 0; 2058 2059 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 2060 /* Use only non-zero vport_id (1-4095) for all PF's */ 2061 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 1, vport_end_ida, GFP_KERNEL); 2062 if (id < 0) 2063 return 0; 2064 id = (pf_num << ESW_VPORT_BITS) | id; 2065 return id; 2066 } 2067 2068 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 2069 { 2070 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 2071 2072 /* Metadata contains only 12 bits of actual ida id */ 2073 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 2074 } 2075 2076 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 2077 struct mlx5_vport *vport) 2078 { 2079 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 2080 vport->metadata = vport->default_metadata; 2081 return vport->metadata ? 0 : -ENOSPC; 2082 } 2083 2084 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 2085 struct mlx5_vport *vport) 2086 { 2087 if (!vport->default_metadata) 2088 return; 2089 2090 WARN_ON(vport->metadata != vport->default_metadata); 2091 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 2092 } 2093 2094 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 2095 { 2096 struct mlx5_vport *vport; 2097 int i; 2098 2099 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2100 return; 2101 2102 mlx5_esw_for_all_vports_reverse(esw, i, vport) 2103 esw_offloads_vport_metadata_cleanup(esw, vport); 2104 } 2105 2106 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 2107 { 2108 struct mlx5_vport *vport; 2109 int err; 2110 int i; 2111 2112 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 2113 return 0; 2114 2115 mlx5_esw_for_all_vports(esw, i, vport) { 2116 err = esw_offloads_vport_metadata_setup(esw, vport); 2117 if (err) 2118 goto metadata_err; 2119 } 2120 2121 return 0; 2122 2123 metadata_err: 2124 esw_offloads_metadata_uninit(esw); 2125 return err; 2126 } 2127 2128 int 2129 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 2130 struct mlx5_vport *vport) 2131 { 2132 int err; 2133 2134 err = esw_acl_ingress_ofld_setup(esw, vport); 2135 if (err) 2136 return err; 2137 2138 err = esw_acl_egress_ofld_setup(esw, vport); 2139 if (err) 2140 goto egress_err; 2141 2142 return 0; 2143 2144 egress_err: 2145 esw_acl_ingress_ofld_cleanup(esw, vport); 2146 return err; 2147 } 2148 2149 void 2150 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 2151 struct mlx5_vport *vport) 2152 { 2153 esw_acl_egress_ofld_cleanup(vport); 2154 esw_acl_ingress_ofld_cleanup(esw, vport); 2155 } 2156 2157 static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2158 { 2159 struct mlx5_vport *vport; 2160 2161 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2162 return esw_vport_create_offloads_acl_tables(esw, vport); 2163 } 2164 2165 static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw) 2166 { 2167 struct mlx5_vport *vport; 2168 2169 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 2170 esw_vport_destroy_offloads_acl_tables(esw, vport); 2171 } 2172 2173 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 2174 { 2175 int err; 2176 2177 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 2178 mutex_init(&esw->fdb_table.offloads.vports.lock); 2179 hash_init(esw->fdb_table.offloads.vports.table); 2180 2181 err = esw_create_uplink_offloads_acl_tables(esw); 2182 if (err) 2183 goto create_acl_err; 2184 2185 err = esw_create_offloads_table(esw); 2186 if (err) 2187 goto create_offloads_err; 2188 2189 err = esw_create_restore_table(esw); 2190 if (err) 2191 goto create_restore_err; 2192 2193 err = esw_create_offloads_fdb_tables(esw); 2194 if (err) 2195 goto create_fdb_err; 2196 2197 err = esw_create_vport_rx_group(esw); 2198 if (err) 2199 goto create_fg_err; 2200 2201 return 0; 2202 2203 create_fg_err: 2204 esw_destroy_offloads_fdb_tables(esw); 2205 create_fdb_err: 2206 esw_destroy_restore_table(esw); 2207 create_restore_err: 2208 esw_destroy_offloads_table(esw); 2209 create_offloads_err: 2210 esw_destroy_uplink_offloads_acl_tables(esw); 2211 create_acl_err: 2212 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 2213 return err; 2214 } 2215 2216 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 2217 { 2218 esw_destroy_vport_rx_group(esw); 2219 esw_destroy_offloads_fdb_tables(esw); 2220 esw_destroy_restore_table(esw); 2221 esw_destroy_offloads_table(esw); 2222 esw_destroy_uplink_offloads_acl_tables(esw); 2223 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 2224 } 2225 2226 static void 2227 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 2228 { 2229 bool host_pf_disabled; 2230 u16 new_num_vfs; 2231 2232 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 2233 host_params_context.host_num_of_vfs); 2234 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 2235 host_params_context.host_pf_disabled); 2236 2237 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 2238 return; 2239 2240 /* Number of VFs can only change from "0 to x" or "x to 0". */ 2241 if (esw->esw_funcs.num_vfs > 0) { 2242 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 2243 } else { 2244 int err; 2245 2246 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 2247 MLX5_VPORT_UC_ADDR_CHANGE); 2248 if (err) 2249 return; 2250 } 2251 esw->esw_funcs.num_vfs = new_num_vfs; 2252 } 2253 2254 static void esw_functions_changed_event_handler(struct work_struct *work) 2255 { 2256 struct mlx5_host_work *host_work; 2257 struct mlx5_eswitch *esw; 2258 const u32 *out; 2259 2260 host_work = container_of(work, struct mlx5_host_work, work); 2261 esw = host_work->esw; 2262 2263 out = mlx5_esw_query_functions(esw->dev); 2264 if (IS_ERR(out)) 2265 goto out; 2266 2267 esw_vfs_changed_event_handler(esw, out); 2268 kvfree(out); 2269 out: 2270 kfree(host_work); 2271 } 2272 2273 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 2274 { 2275 struct mlx5_esw_functions *esw_funcs; 2276 struct mlx5_host_work *host_work; 2277 struct mlx5_eswitch *esw; 2278 2279 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 2280 if (!host_work) 2281 return NOTIFY_DONE; 2282 2283 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 2284 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 2285 2286 host_work->esw = esw; 2287 2288 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 2289 queue_work(esw->work_queue, &host_work->work); 2290 2291 return NOTIFY_OK; 2292 } 2293 2294 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 2295 { 2296 const u32 *query_host_out; 2297 2298 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 2299 return 0; 2300 2301 query_host_out = mlx5_esw_query_functions(esw->dev); 2302 if (IS_ERR(query_host_out)) 2303 return PTR_ERR(query_host_out); 2304 2305 /* Mark non local controller with non zero controller number. */ 2306 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 2307 host_params_context.host_number); 2308 kvfree(query_host_out); 2309 return 0; 2310 } 2311 2312 int esw_offloads_enable(struct mlx5_eswitch *esw) 2313 { 2314 struct mlx5_vport *vport; 2315 int err, i; 2316 2317 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat) && 2318 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, decap)) 2319 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC; 2320 else 2321 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE; 2322 2323 mutex_init(&esw->offloads.termtbl_mutex); 2324 mlx5_rdma_enable_roce(esw->dev); 2325 2326 err = mlx5_esw_host_number_init(esw); 2327 if (err) 2328 goto err_metadata; 2329 2330 if (esw_check_vport_match_metadata_supported(esw)) 2331 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2332 2333 err = esw_offloads_metadata_init(esw); 2334 if (err) 2335 goto err_metadata; 2336 2337 err = esw_set_passing_vport_metadata(esw, true); 2338 if (err) 2339 goto err_vport_metadata; 2340 2341 err = esw_offloads_steering_init(esw); 2342 if (err) 2343 goto err_steering_init; 2344 2345 /* Representor will control the vport link state */ 2346 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 2347 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 2348 2349 /* Uplink vport rep must load first. */ 2350 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK); 2351 if (err) 2352 goto err_uplink; 2353 2354 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 2355 if (err) 2356 goto err_vports; 2357 2358 esw_offloads_devcom_init(esw); 2359 2360 return 0; 2361 2362 err_vports: 2363 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 2364 err_uplink: 2365 esw_offloads_steering_cleanup(esw); 2366 err_steering_init: 2367 esw_set_passing_vport_metadata(esw, false); 2368 err_vport_metadata: 2369 esw_offloads_metadata_uninit(esw); 2370 err_metadata: 2371 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2372 mlx5_rdma_disable_roce(esw->dev); 2373 mutex_destroy(&esw->offloads.termtbl_mutex); 2374 return err; 2375 } 2376 2377 static int esw_offloads_stop(struct mlx5_eswitch *esw, 2378 struct netlink_ext_ack *extack) 2379 { 2380 int err, err1; 2381 2382 mlx5_eswitch_disable_locked(esw, false); 2383 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, 2384 MLX5_ESWITCH_IGNORE_NUM_VFS); 2385 if (err) { 2386 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 2387 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, 2388 MLX5_ESWITCH_IGNORE_NUM_VFS); 2389 if (err1) { 2390 NL_SET_ERR_MSG_MOD(extack, 2391 "Failed setting eswitch back to offloads"); 2392 } 2393 } 2394 2395 return err; 2396 } 2397 2398 void esw_offloads_disable(struct mlx5_eswitch *esw) 2399 { 2400 esw_offloads_devcom_cleanup(esw); 2401 mlx5_eswitch_disable_pf_vf_vports(esw); 2402 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK); 2403 esw_set_passing_vport_metadata(esw, false); 2404 esw_offloads_steering_cleanup(esw); 2405 esw_offloads_metadata_uninit(esw); 2406 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2407 mlx5_rdma_disable_roce(esw->dev); 2408 mutex_destroy(&esw->offloads.termtbl_mutex); 2409 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE; 2410 } 2411 2412 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 2413 { 2414 switch (mode) { 2415 case DEVLINK_ESWITCH_MODE_LEGACY: 2416 *mlx5_mode = MLX5_ESWITCH_LEGACY; 2417 break; 2418 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 2419 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 2420 break; 2421 default: 2422 return -EINVAL; 2423 } 2424 2425 return 0; 2426 } 2427 2428 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 2429 { 2430 switch (mlx5_mode) { 2431 case MLX5_ESWITCH_LEGACY: 2432 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 2433 break; 2434 case MLX5_ESWITCH_OFFLOADS: 2435 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 2436 break; 2437 default: 2438 return -EINVAL; 2439 } 2440 2441 return 0; 2442 } 2443 2444 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 2445 { 2446 switch (mode) { 2447 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 2448 *mlx5_mode = MLX5_INLINE_MODE_NONE; 2449 break; 2450 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 2451 *mlx5_mode = MLX5_INLINE_MODE_L2; 2452 break; 2453 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 2454 *mlx5_mode = MLX5_INLINE_MODE_IP; 2455 break; 2456 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 2457 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 2458 break; 2459 default: 2460 return -EINVAL; 2461 } 2462 2463 return 0; 2464 } 2465 2466 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 2467 { 2468 switch (mlx5_mode) { 2469 case MLX5_INLINE_MODE_NONE: 2470 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 2471 break; 2472 case MLX5_INLINE_MODE_L2: 2473 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 2474 break; 2475 case MLX5_INLINE_MODE_IP: 2476 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 2477 break; 2478 case MLX5_INLINE_MODE_TCP_UDP: 2479 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 2480 break; 2481 default: 2482 return -EINVAL; 2483 } 2484 2485 return 0; 2486 } 2487 2488 static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) 2489 { 2490 /* devlink commands in NONE eswitch mode are currently supported only 2491 * on ECPF. 2492 */ 2493 return (esw->mode == MLX5_ESWITCH_NONE && 2494 !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0; 2495 } 2496 2497 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 2498 struct netlink_ext_ack *extack) 2499 { 2500 u16 cur_mlx5_mode, mlx5_mode = 0; 2501 struct mlx5_eswitch *esw; 2502 int err = 0; 2503 2504 esw = mlx5_devlink_eswitch_get(devlink); 2505 if (IS_ERR(esw)) 2506 return PTR_ERR(esw); 2507 2508 if (esw_mode_from_devlink(mode, &mlx5_mode)) 2509 return -EINVAL; 2510 2511 mutex_lock(&esw->mode_lock); 2512 cur_mlx5_mode = esw->mode; 2513 if (cur_mlx5_mode == mlx5_mode) 2514 goto unlock; 2515 2516 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) 2517 err = esw_offloads_start(esw, extack); 2518 else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) 2519 err = esw_offloads_stop(esw, extack); 2520 else 2521 err = -EINVAL; 2522 2523 unlock: 2524 mutex_unlock(&esw->mode_lock); 2525 return err; 2526 } 2527 2528 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 2529 { 2530 struct mlx5_eswitch *esw; 2531 int err; 2532 2533 esw = mlx5_devlink_eswitch_get(devlink); 2534 if (IS_ERR(esw)) 2535 return PTR_ERR(esw); 2536 2537 mutex_lock(&esw->mode_lock); 2538 err = eswitch_devlink_esw_mode_check(esw); 2539 if (err) 2540 goto unlock; 2541 2542 err = esw_mode_to_devlink(esw->mode, mode); 2543 unlock: 2544 mutex_unlock(&esw->mode_lock); 2545 return err; 2546 } 2547 2548 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 2549 struct netlink_ext_ack *extack) 2550 { 2551 struct mlx5_core_dev *dev = devlink_priv(devlink); 2552 int err, vport, num_vport; 2553 struct mlx5_eswitch *esw; 2554 u8 mlx5_mode; 2555 2556 esw = mlx5_devlink_eswitch_get(devlink); 2557 if (IS_ERR(esw)) 2558 return PTR_ERR(esw); 2559 2560 mutex_lock(&esw->mode_lock); 2561 err = eswitch_devlink_esw_mode_check(esw); 2562 if (err) 2563 goto out; 2564 2565 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2566 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2567 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) 2568 goto out; 2569 fallthrough; 2570 case MLX5_CAP_INLINE_MODE_L2: 2571 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 2572 err = -EOPNOTSUPP; 2573 goto out; 2574 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2575 break; 2576 } 2577 2578 if (atomic64_read(&esw->offloads.num_flows) > 0) { 2579 NL_SET_ERR_MSG_MOD(extack, 2580 "Can't set inline mode when flows are configured"); 2581 err = -EOPNOTSUPP; 2582 goto out; 2583 } 2584 2585 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 2586 if (err) 2587 goto out; 2588 2589 mlx5_esw_for_each_host_func_vport(esw, vport, esw->esw_funcs.num_vfs) { 2590 err = mlx5_modify_nic_vport_min_inline(dev, vport, mlx5_mode); 2591 if (err) { 2592 NL_SET_ERR_MSG_MOD(extack, 2593 "Failed to set min inline on vport"); 2594 goto revert_inline_mode; 2595 } 2596 } 2597 2598 esw->offloads.inline_mode = mlx5_mode; 2599 mutex_unlock(&esw->mode_lock); 2600 return 0; 2601 2602 revert_inline_mode: 2603 num_vport = --vport; 2604 mlx5_esw_for_each_host_func_vport_reverse(esw, vport, num_vport) 2605 mlx5_modify_nic_vport_min_inline(dev, 2606 vport, 2607 esw->offloads.inline_mode); 2608 out: 2609 mutex_unlock(&esw->mode_lock); 2610 return err; 2611 } 2612 2613 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 2614 { 2615 struct mlx5_eswitch *esw; 2616 int err; 2617 2618 esw = mlx5_devlink_eswitch_get(devlink); 2619 if (IS_ERR(esw)) 2620 return PTR_ERR(esw); 2621 2622 mutex_lock(&esw->mode_lock); 2623 err = eswitch_devlink_esw_mode_check(esw); 2624 if (err) 2625 goto unlock; 2626 2627 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 2628 unlock: 2629 mutex_unlock(&esw->mode_lock); 2630 return err; 2631 } 2632 2633 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 2634 enum devlink_eswitch_encap_mode encap, 2635 struct netlink_ext_ack *extack) 2636 { 2637 struct mlx5_core_dev *dev = devlink_priv(devlink); 2638 struct mlx5_eswitch *esw; 2639 int err; 2640 2641 esw = mlx5_devlink_eswitch_get(devlink); 2642 if (IS_ERR(esw)) 2643 return PTR_ERR(esw); 2644 2645 mutex_lock(&esw->mode_lock); 2646 err = eswitch_devlink_esw_mode_check(esw); 2647 if (err) 2648 goto unlock; 2649 2650 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 2651 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 2652 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 2653 err = -EOPNOTSUPP; 2654 goto unlock; 2655 } 2656 2657 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 2658 err = -EOPNOTSUPP; 2659 goto unlock; 2660 } 2661 2662 if (esw->mode == MLX5_ESWITCH_LEGACY) { 2663 esw->offloads.encap = encap; 2664 goto unlock; 2665 } 2666 2667 if (esw->offloads.encap == encap) 2668 goto unlock; 2669 2670 if (atomic64_read(&esw->offloads.num_flows) > 0) { 2671 NL_SET_ERR_MSG_MOD(extack, 2672 "Can't set encapsulation when flows are configured"); 2673 err = -EOPNOTSUPP; 2674 goto unlock; 2675 } 2676 2677 esw_destroy_offloads_fdb_tables(esw); 2678 2679 esw->offloads.encap = encap; 2680 2681 err = esw_create_offloads_fdb_tables(esw); 2682 2683 if (err) { 2684 NL_SET_ERR_MSG_MOD(extack, 2685 "Failed re-creating fast FDB table"); 2686 esw->offloads.encap = !encap; 2687 (void)esw_create_offloads_fdb_tables(esw); 2688 } 2689 2690 unlock: 2691 mutex_unlock(&esw->mode_lock); 2692 return err; 2693 } 2694 2695 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 2696 enum devlink_eswitch_encap_mode *encap) 2697 { 2698 struct mlx5_eswitch *esw; 2699 int err; 2700 2701 esw = mlx5_devlink_eswitch_get(devlink); 2702 if (IS_ERR(esw)) 2703 return PTR_ERR(esw); 2704 2705 2706 mutex_lock(&esw->mode_lock); 2707 err = eswitch_devlink_esw_mode_check(esw); 2708 if (err) 2709 goto unlock; 2710 2711 *encap = esw->offloads.encap; 2712 unlock: 2713 mutex_unlock(&esw->mode_lock); 2714 return 0; 2715 } 2716 2717 static bool 2718 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 2719 { 2720 /* Currently, only ECPF based device has representor for host PF. */ 2721 if (vport_num == MLX5_VPORT_PF && 2722 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 2723 return false; 2724 2725 if (vport_num == MLX5_VPORT_ECPF && 2726 !mlx5_ecpf_vport_exists(esw->dev)) 2727 return false; 2728 2729 return true; 2730 } 2731 2732 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 2733 const struct mlx5_eswitch_rep_ops *ops, 2734 u8 rep_type) 2735 { 2736 struct mlx5_eswitch_rep_data *rep_data; 2737 struct mlx5_eswitch_rep *rep; 2738 int i; 2739 2740 esw->offloads.rep_ops[rep_type] = ops; 2741 mlx5_esw_for_all_reps(esw, i, rep) { 2742 if (likely(mlx5_eswitch_vport_has_rep(esw, i))) { 2743 rep_data = &rep->rep_data[rep_type]; 2744 atomic_set(&rep_data->state, REP_REGISTERED); 2745 } 2746 } 2747 } 2748 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 2749 2750 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 2751 { 2752 struct mlx5_eswitch_rep *rep; 2753 int i; 2754 2755 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 2756 __unload_reps_all_vport(esw, rep_type); 2757 2758 mlx5_esw_for_all_reps(esw, i, rep) 2759 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2760 } 2761 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 2762 2763 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 2764 { 2765 struct mlx5_eswitch_rep *rep; 2766 2767 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 2768 return rep->rep_data[rep_type].priv; 2769 } 2770 2771 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 2772 u16 vport, 2773 u8 rep_type) 2774 { 2775 struct mlx5_eswitch_rep *rep; 2776 2777 rep = mlx5_eswitch_get_rep(esw, vport); 2778 2779 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2780 esw->offloads.rep_ops[rep_type]->get_proto_dev) 2781 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 2782 return NULL; 2783 } 2784 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 2785 2786 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 2787 { 2788 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 2789 } 2790 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 2791 2792 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 2793 u16 vport) 2794 { 2795 return mlx5_eswitch_get_rep(esw, vport); 2796 } 2797 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 2798 2799 bool mlx5_eswitch_is_vf_vport(const struct mlx5_eswitch *esw, u16 vport_num) 2800 { 2801 return vport_num >= MLX5_VPORT_FIRST_VF && 2802 vport_num <= esw->dev->priv.sriov.max_vfs; 2803 } 2804 2805 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 2806 { 2807 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 2808 } 2809 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 2810 2811 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 2812 { 2813 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 2814 } 2815 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 2816 2817 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 2818 u16 vport_num) 2819 { 2820 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 2821 2822 if (WARN_ON_ONCE(IS_ERR(vport))) 2823 return 0; 2824 2825 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 2826 } 2827 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 2828