1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/debugfs.h> 34 #include <linux/mlx5/fs.h> 35 #include <net/switchdev.h> 36 #include <net/pkt_cls.h> 37 #include <net/act_api.h> 38 #include <net/devlink.h> 39 #include <net/ipv6_stubs.h> 40 41 #include "eswitch.h" 42 #include "en.h" 43 #include "en_rep.h" 44 #include "en/params.h" 45 #include "en/txrx.h" 46 #include "en_tc.h" 47 #include "en/rep/tc.h" 48 #include "en/rep/neigh.h" 49 #include "en/rep/bridge.h" 50 #include "en/devlink.h" 51 #include "fs_core.h" 52 #include "lib/mlx5.h" 53 #include "lib/devcom.h" 54 #include "lib/vxlan.h" 55 #define CREATE_TRACE_POINTS 56 #include "diag/en_rep_tracepoint.h" 57 #include "diag/reporter_vnic.h" 58 #include "en_accel/ipsec.h" 59 #include "en/tc/int_port.h" 60 #include "en/ptp.h" 61 #include "en/fs_ethtool.h" 62 63 #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \ 64 max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE) 65 #define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1 66 67 static const char mlx5e_rep_driver_name[] = "mlx5e_rep"; 68 69 static void mlx5e_rep_get_drvinfo(struct net_device *dev, 70 struct ethtool_drvinfo *drvinfo) 71 { 72 struct mlx5e_priv *priv = netdev_priv(dev); 73 struct mlx5_core_dev *mdev = priv->mdev; 74 int count; 75 76 strscpy(drvinfo->driver, mlx5e_rep_driver_name, 77 sizeof(drvinfo->driver)); 78 count = snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), 79 "%d.%d.%04d (%.16s)", fw_rev_maj(mdev), 80 fw_rev_min(mdev), fw_rev_sub(mdev), mdev->board_id); 81 if (count >= sizeof(drvinfo->fw_version)) 82 snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), 83 "%d.%d.%04d", fw_rev_maj(mdev), 84 fw_rev_min(mdev), fw_rev_sub(mdev)); 85 } 86 87 static const struct counter_desc sw_rep_stats_desc[] = { 88 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) }, 89 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_bytes) }, 90 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_packets) }, 91 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) }, 92 }; 93 94 static const struct counter_desc vport_rep_stats_desc[] = { 95 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_packets) }, 96 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_bytes) }, 97 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_packets) }, 98 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_bytes) }, 99 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 100 rx_vport_rdma_unicast_packets) }, 101 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, rx_vport_rdma_unicast_bytes) }, 102 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 103 tx_vport_rdma_unicast_packets) }, 104 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, tx_vport_rdma_unicast_bytes) }, 105 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 106 rx_vport_rdma_multicast_packets) }, 107 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 108 rx_vport_rdma_multicast_bytes) }, 109 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 110 tx_vport_rdma_multicast_packets) }, 111 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 112 tx_vport_rdma_multicast_bytes) }, 113 }; 114 115 #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc) 116 #define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc) 117 118 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep) 119 { 120 return NUM_VPORT_REP_SW_COUNTERS; 121 } 122 123 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep) 124 { 125 int i; 126 127 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++) 128 strcpy(data + (idx++) * ETH_GSTRING_LEN, 129 sw_rep_stats_desc[i].format); 130 return idx; 131 } 132 133 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep) 134 { 135 int i; 136 137 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++) 138 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.sw, 139 sw_rep_stats_desc, i); 140 return idx; 141 } 142 143 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep) 144 { 145 struct mlx5e_sw_stats *s = &priv->stats.sw; 146 struct rtnl_link_stats64 stats64 = {}; 147 148 memset(s, 0, sizeof(*s)); 149 mlx5e_fold_sw_stats64(priv, &stats64); 150 151 s->rx_packets = stats64.rx_packets; 152 s->rx_bytes = stats64.rx_bytes; 153 s->tx_packets = stats64.tx_packets; 154 s->tx_bytes = stats64.tx_bytes; 155 s->tx_queue_dropped = stats64.tx_dropped; 156 } 157 158 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep) 159 { 160 return NUM_VPORT_REP_HW_COUNTERS; 161 } 162 163 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep) 164 { 165 int i; 166 167 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) 168 strcpy(data + (idx++) * ETH_GSTRING_LEN, vport_rep_stats_desc[i].format); 169 return idx; 170 } 171 172 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep) 173 { 174 int i; 175 176 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) 177 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats, 178 vport_rep_stats_desc, i); 179 return idx; 180 } 181 182 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep) 183 { 184 struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats; 185 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out); 186 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 187 struct mlx5e_rep_priv *rpriv = priv->ppriv; 188 struct mlx5_eswitch_rep *rep = rpriv->rep; 189 u32 *out; 190 int err; 191 192 out = kvzalloc(outlen, GFP_KERNEL); 193 if (!out) 194 return; 195 196 err = mlx5_core_query_vport_counter(esw->dev, 1, rep->vport - 1, 0, out); 197 if (err) { 198 netdev_warn(priv->netdev, "vport %d error %d reading stats\n", 199 rep->vport, err); 200 goto out; 201 } 202 203 #define MLX5_GET_CTR(p, x) \ 204 MLX5_GET64(query_vport_counter_out, p, x) 205 /* flip tx/rx as we are reporting the counters for the switch vport */ 206 rep_stats->vport_rx_packets = 207 MLX5_GET_CTR(out, transmitted_ib_unicast.packets) + 208 MLX5_GET_CTR(out, transmitted_eth_unicast.packets) + 209 MLX5_GET_CTR(out, transmitted_ib_multicast.packets) + 210 MLX5_GET_CTR(out, transmitted_eth_multicast.packets) + 211 MLX5_GET_CTR(out, transmitted_eth_broadcast.packets); 212 213 rep_stats->vport_tx_packets = 214 MLX5_GET_CTR(out, received_ib_unicast.packets) + 215 MLX5_GET_CTR(out, received_eth_unicast.packets) + 216 MLX5_GET_CTR(out, received_ib_multicast.packets) + 217 MLX5_GET_CTR(out, received_eth_multicast.packets) + 218 MLX5_GET_CTR(out, received_eth_broadcast.packets); 219 220 rep_stats->vport_rx_bytes = 221 MLX5_GET_CTR(out, transmitted_ib_unicast.octets) + 222 MLX5_GET_CTR(out, transmitted_eth_unicast.octets) + 223 MLX5_GET_CTR(out, transmitted_ib_multicast.octets) + 224 MLX5_GET_CTR(out, transmitted_eth_broadcast.octets); 225 226 rep_stats->vport_tx_bytes = 227 MLX5_GET_CTR(out, received_ib_unicast.octets) + 228 MLX5_GET_CTR(out, received_eth_unicast.octets) + 229 MLX5_GET_CTR(out, received_ib_multicast.octets) + 230 MLX5_GET_CTR(out, received_eth_multicast.octets) + 231 MLX5_GET_CTR(out, received_eth_broadcast.octets); 232 233 rep_stats->rx_vport_rdma_unicast_packets = 234 MLX5_GET_CTR(out, transmitted_ib_unicast.packets); 235 rep_stats->tx_vport_rdma_unicast_packets = 236 MLX5_GET_CTR(out, received_ib_unicast.packets); 237 rep_stats->rx_vport_rdma_unicast_bytes = 238 MLX5_GET_CTR(out, transmitted_ib_unicast.octets); 239 rep_stats->tx_vport_rdma_unicast_bytes = 240 MLX5_GET_CTR(out, received_ib_unicast.octets); 241 rep_stats->rx_vport_rdma_multicast_packets = 242 MLX5_GET_CTR(out, transmitted_ib_multicast.packets); 243 rep_stats->tx_vport_rdma_multicast_packets = 244 MLX5_GET_CTR(out, received_ib_multicast.packets); 245 rep_stats->rx_vport_rdma_multicast_bytes = 246 MLX5_GET_CTR(out, transmitted_ib_multicast.octets); 247 rep_stats->tx_vport_rdma_multicast_bytes = 248 MLX5_GET_CTR(out, received_ib_multicast.octets); 249 250 out: 251 kvfree(out); 252 } 253 254 static void mlx5e_rep_get_strings(struct net_device *dev, 255 u32 stringset, uint8_t *data) 256 { 257 struct mlx5e_priv *priv = netdev_priv(dev); 258 259 switch (stringset) { 260 case ETH_SS_STATS: 261 mlx5e_stats_fill_strings(priv, data); 262 break; 263 } 264 } 265 266 static void mlx5e_rep_get_ethtool_stats(struct net_device *dev, 267 struct ethtool_stats *stats, u64 *data) 268 { 269 struct mlx5e_priv *priv = netdev_priv(dev); 270 271 mlx5e_ethtool_get_ethtool_stats(priv, stats, data); 272 } 273 274 static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset) 275 { 276 struct mlx5e_priv *priv = netdev_priv(dev); 277 278 switch (sset) { 279 case ETH_SS_STATS: 280 return mlx5e_stats_total_num(priv); 281 default: 282 return -EOPNOTSUPP; 283 } 284 } 285 286 static void 287 mlx5e_rep_get_ringparam(struct net_device *dev, 288 struct ethtool_ringparam *param, 289 struct kernel_ethtool_ringparam *kernel_param, 290 struct netlink_ext_ack *extack) 291 { 292 struct mlx5e_priv *priv = netdev_priv(dev); 293 294 mlx5e_ethtool_get_ringparam(priv, param, kernel_param); 295 } 296 297 static int 298 mlx5e_rep_set_ringparam(struct net_device *dev, 299 struct ethtool_ringparam *param, 300 struct kernel_ethtool_ringparam *kernel_param, 301 struct netlink_ext_ack *extack) 302 { 303 struct mlx5e_priv *priv = netdev_priv(dev); 304 305 return mlx5e_ethtool_set_ringparam(priv, param); 306 } 307 308 static void mlx5e_rep_get_channels(struct net_device *dev, 309 struct ethtool_channels *ch) 310 { 311 struct mlx5e_priv *priv = netdev_priv(dev); 312 313 mlx5e_ethtool_get_channels(priv, ch); 314 } 315 316 static int mlx5e_rep_set_channels(struct net_device *dev, 317 struct ethtool_channels *ch) 318 { 319 struct mlx5e_priv *priv = netdev_priv(dev); 320 321 return mlx5e_ethtool_set_channels(priv, ch); 322 } 323 324 static int mlx5e_rep_get_coalesce(struct net_device *netdev, 325 struct ethtool_coalesce *coal, 326 struct kernel_ethtool_coalesce *kernel_coal, 327 struct netlink_ext_ack *extack) 328 { 329 struct mlx5e_priv *priv = netdev_priv(netdev); 330 331 return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal); 332 } 333 334 static int mlx5e_rep_set_coalesce(struct net_device *netdev, 335 struct ethtool_coalesce *coal, 336 struct kernel_ethtool_coalesce *kernel_coal, 337 struct netlink_ext_ack *extack) 338 { 339 struct mlx5e_priv *priv = netdev_priv(netdev); 340 341 return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack); 342 } 343 344 static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev) 345 { 346 struct mlx5e_priv *priv = netdev_priv(netdev); 347 348 return mlx5e_ethtool_get_rxfh_key_size(priv); 349 } 350 351 static u32 mlx5e_rep_get_rxfh_indir_size(struct net_device *netdev) 352 { 353 struct mlx5e_priv *priv = netdev_priv(netdev); 354 355 return mlx5e_ethtool_get_rxfh_indir_size(priv); 356 } 357 358 static const struct ethtool_ops mlx5e_rep_ethtool_ops = { 359 .supported_coalesce_params = ETHTOOL_COALESCE_USECS | 360 ETHTOOL_COALESCE_MAX_FRAMES | 361 ETHTOOL_COALESCE_USE_ADAPTIVE, 362 .get_drvinfo = mlx5e_rep_get_drvinfo, 363 .get_link = ethtool_op_get_link, 364 .get_strings = mlx5e_rep_get_strings, 365 .get_sset_count = mlx5e_rep_get_sset_count, 366 .get_ethtool_stats = mlx5e_rep_get_ethtool_stats, 367 .get_ringparam = mlx5e_rep_get_ringparam, 368 .set_ringparam = mlx5e_rep_set_ringparam, 369 .get_channels = mlx5e_rep_get_channels, 370 .set_channels = mlx5e_rep_set_channels, 371 .get_coalesce = mlx5e_rep_get_coalesce, 372 .set_coalesce = mlx5e_rep_set_coalesce, 373 .get_rxfh_key_size = mlx5e_rep_get_rxfh_key_size, 374 .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size, 375 }; 376 377 static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw, 378 struct mlx5_eswitch_rep *rep) 379 { 380 struct mlx5e_rep_sq *rep_sq, *tmp; 381 struct mlx5e_rep_sq_peer *sq_peer; 382 struct mlx5e_rep_priv *rpriv; 383 unsigned long i; 384 385 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 386 return; 387 388 rpriv = mlx5e_rep_to_rep_priv(rep); 389 list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) { 390 mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); 391 xa_for_each(&rep_sq->sq_peer, i, sq_peer) { 392 if (sq_peer->rule) 393 mlx5_eswitch_del_send_to_vport_rule(sq_peer->rule); 394 395 xa_erase(&rep_sq->sq_peer, i); 396 kfree(sq_peer); 397 } 398 399 xa_destroy(&rep_sq->sq_peer); 400 list_del(&rep_sq->list); 401 kfree(rep_sq); 402 } 403 } 404 405 static int mlx5e_sqs2vport_add_peers_rules(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep, 406 struct mlx5e_rep_sq *rep_sq, int i) 407 { 408 struct mlx5_flow_handle *flow_rule; 409 struct mlx5_devcom_comp_dev *tmp; 410 struct mlx5_eswitch *peer_esw; 411 412 mlx5_devcom_for_each_peer_entry(esw->devcom, peer_esw, tmp) { 413 u16 peer_rule_idx = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 414 struct mlx5e_rep_sq_peer *sq_peer; 415 int err; 416 417 sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL); 418 if (!sq_peer) 419 return -ENOMEM; 420 421 flow_rule = mlx5_eswitch_add_send_to_vport_rule(peer_esw, esw, 422 rep, rep_sq->sqn); 423 if (IS_ERR(flow_rule)) { 424 kfree(sq_peer); 425 return PTR_ERR(flow_rule); 426 } 427 428 sq_peer->rule = flow_rule; 429 sq_peer->peer = peer_esw; 430 err = xa_insert(&rep_sq->sq_peer, peer_rule_idx, sq_peer, GFP_KERNEL); 431 if (err) { 432 kfree(sq_peer); 433 mlx5_eswitch_del_send_to_vport_rule(flow_rule); 434 return err; 435 } 436 } 437 438 return 0; 439 } 440 441 static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, 442 struct mlx5_eswitch_rep *rep, 443 u32 *sqns_array, int sqns_num) 444 { 445 struct mlx5_flow_handle *flow_rule; 446 struct mlx5e_rep_priv *rpriv; 447 struct mlx5e_rep_sq *rep_sq; 448 bool devcom_locked = false; 449 int err; 450 int i; 451 452 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 453 return 0; 454 455 rpriv = mlx5e_rep_to_rep_priv(rep); 456 457 if (mlx5_devcom_comp_is_ready(esw->devcom) && 458 mlx5_devcom_for_each_peer_begin(esw->devcom)) 459 devcom_locked = true; 460 461 for (i = 0; i < sqns_num; i++) { 462 rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL); 463 if (!rep_sq) { 464 err = -ENOMEM; 465 goto out_err; 466 } 467 468 /* Add re-inject rule to the PF/representor sqs */ 469 flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw, esw, rep, 470 sqns_array[i]); 471 if (IS_ERR(flow_rule)) { 472 err = PTR_ERR(flow_rule); 473 kfree(rep_sq); 474 goto out_err; 475 } 476 rep_sq->send_to_vport_rule = flow_rule; 477 rep_sq->sqn = sqns_array[i]; 478 479 xa_init(&rep_sq->sq_peer); 480 if (devcom_locked) { 481 err = mlx5e_sqs2vport_add_peers_rules(esw, rep, rep_sq, i); 482 if (err) { 483 mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); 484 xa_destroy(&rep_sq->sq_peer); 485 kfree(rep_sq); 486 goto out_err; 487 } 488 } 489 490 list_add(&rep_sq->list, &rpriv->vport_sqs_list); 491 } 492 493 if (devcom_locked) 494 mlx5_devcom_for_each_peer_end(esw->devcom); 495 496 return 0; 497 498 out_err: 499 mlx5e_sqs2vport_stop(esw, rep); 500 501 if (devcom_locked) 502 mlx5_devcom_for_each_peer_end(esw->devcom); 503 504 return err; 505 } 506 507 static int 508 mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) 509 { 510 int sqs_per_channel = mlx5e_get_dcb_num_tc(&priv->channels.params); 511 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 512 bool is_uplink_rep = mlx5e_is_uplink_rep(priv); 513 struct mlx5e_rep_priv *rpriv = priv->ppriv; 514 struct mlx5_eswitch_rep *rep = rpriv->rep; 515 int n, tc, nch, num_sqs = 0; 516 struct mlx5e_channel *c; 517 int err = -ENOMEM; 518 bool ptp_sq; 519 u32 *sqs; 520 521 ptp_sq = !!(priv->channels.ptp && 522 MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)); 523 nch = priv->channels.num + ptp_sq; 524 /* +2 for xdpsqs, they don't exist on the ptp channel but will not be 525 * counted for by num_sqs. 526 */ 527 if (is_uplink_rep) 528 sqs_per_channel += 2; 529 530 sqs = kvcalloc(nch * sqs_per_channel, sizeof(*sqs), GFP_KERNEL); 531 if (!sqs) 532 goto out; 533 534 for (n = 0; n < priv->channels.num; n++) { 535 c = priv->channels.c[n]; 536 for (tc = 0; tc < c->num_tc; tc++) 537 sqs[num_sqs++] = c->sq[tc].sqn; 538 539 if (is_uplink_rep) { 540 if (c->xdp) 541 sqs[num_sqs++] = c->rq_xdpsq.sqn; 542 543 sqs[num_sqs++] = c->xdpsq.sqn; 544 } 545 } 546 if (ptp_sq) { 547 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 548 549 for (tc = 0; tc < ptp_ch->num_tc; tc++) 550 sqs[num_sqs++] = ptp_ch->ptpsq[tc].txqsq.sqn; 551 } 552 553 err = mlx5e_sqs2vport_start(esw, rep, sqs, num_sqs); 554 kvfree(sqs); 555 556 out: 557 if (err) 558 netdev_warn(priv->netdev, "Failed to add SQs FWD rules %d\n", err); 559 return err; 560 } 561 562 static void 563 mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) 564 { 565 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 566 struct mlx5e_rep_priv *rpriv = priv->ppriv; 567 struct mlx5_eswitch_rep *rep = rpriv->rep; 568 569 mlx5e_sqs2vport_stop(esw, rep); 570 } 571 572 static int 573 mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv *priv) 574 { 575 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 576 struct mlx5e_rep_priv *rpriv = priv->ppriv; 577 struct mlx5_eswitch_rep *rep = rpriv->rep; 578 struct mlx5_flow_handle *flow_rule; 579 struct mlx5_flow_group *g; 580 581 g = esw->fdb_table.offloads.send_to_vport_meta_grp; 582 if (!g) 583 return 0; 584 585 flow_rule = mlx5_eswitch_add_send_to_vport_meta_rule(esw, rep->vport); 586 if (IS_ERR(flow_rule)) 587 return PTR_ERR(flow_rule); 588 589 rpriv->send_to_vport_meta_rule = flow_rule; 590 591 return 0; 592 } 593 594 static void 595 mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv *priv) 596 { 597 struct mlx5e_rep_priv *rpriv = priv->ppriv; 598 599 if (rpriv->send_to_vport_meta_rule) 600 mlx5_eswitch_del_send_to_vport_meta_rule(rpriv->send_to_vport_meta_rule); 601 } 602 603 void mlx5e_rep_activate_channels(struct mlx5e_priv *priv) 604 { 605 mlx5e_add_sqs_fwd_rules(priv); 606 mlx5e_rep_add_meta_tunnel_rule(priv); 607 } 608 609 void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv) 610 { 611 mlx5e_rep_del_meta_tunnel_rule(priv); 612 mlx5e_remove_sqs_fwd_rules(priv); 613 } 614 615 static int mlx5e_rep_open(struct net_device *dev) 616 { 617 struct mlx5e_priv *priv = netdev_priv(dev); 618 struct mlx5e_rep_priv *rpriv = priv->ppriv; 619 struct mlx5_eswitch_rep *rep = rpriv->rep; 620 int err; 621 622 mutex_lock(&priv->state_lock); 623 err = mlx5e_open_locked(dev); 624 if (err) 625 goto unlock; 626 627 if (!mlx5_modify_vport_admin_state(priv->mdev, 628 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 629 rep->vport, 1, 630 MLX5_VPORT_ADMIN_STATE_UP)) 631 netif_carrier_on(dev); 632 633 unlock: 634 mutex_unlock(&priv->state_lock); 635 return err; 636 } 637 638 static int mlx5e_rep_close(struct net_device *dev) 639 { 640 struct mlx5e_priv *priv = netdev_priv(dev); 641 struct mlx5e_rep_priv *rpriv = priv->ppriv; 642 struct mlx5_eswitch_rep *rep = rpriv->rep; 643 int ret; 644 645 mutex_lock(&priv->state_lock); 646 mlx5_modify_vport_admin_state(priv->mdev, 647 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 648 rep->vport, 1, 649 MLX5_VPORT_ADMIN_STATE_DOWN); 650 ret = mlx5e_close_locked(dev); 651 mutex_unlock(&priv->state_lock); 652 return ret; 653 } 654 655 bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv) 656 { 657 struct mlx5e_rep_priv *rpriv = priv->ppriv; 658 struct mlx5_eswitch_rep *rep; 659 660 if (!MLX5_ESWITCH_MANAGER(priv->mdev)) 661 return false; 662 663 if (!rpriv) /* non vport rep mlx5e instances don't use this field */ 664 return false; 665 666 rep = rpriv->rep; 667 return (rep->vport == MLX5_VPORT_UPLINK); 668 } 669 670 bool mlx5e_rep_has_offload_stats(const struct net_device *dev, int attr_id) 671 { 672 switch (attr_id) { 673 case IFLA_OFFLOAD_XSTATS_CPU_HIT: 674 return true; 675 } 676 677 return false; 678 } 679 680 static int 681 mlx5e_get_sw_stats64(const struct net_device *dev, 682 struct rtnl_link_stats64 *stats) 683 { 684 struct mlx5e_priv *priv = netdev_priv(dev); 685 686 mlx5e_fold_sw_stats64(priv, stats); 687 return 0; 688 } 689 690 int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev, 691 void *sp) 692 { 693 switch (attr_id) { 694 case IFLA_OFFLOAD_XSTATS_CPU_HIT: 695 return mlx5e_get_sw_stats64(dev, sp); 696 } 697 698 return -EINVAL; 699 } 700 701 static void 702 mlx5e_rep_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 703 { 704 struct mlx5e_priv *priv = netdev_priv(dev); 705 706 /* update HW stats in background for next time */ 707 mlx5e_queue_update_stats(priv); 708 mlx5e_stats_copy_rep_stats(stats, &priv->stats.rep_stats); 709 } 710 711 static int mlx5e_rep_change_mtu(struct net_device *netdev, int new_mtu) 712 { 713 return mlx5e_change_mtu(netdev, new_mtu, NULL); 714 } 715 716 static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) 717 { 718 struct mlx5e_priv *priv = netdev_priv(dev); 719 struct mlx5e_rep_priv *rpriv = priv->ppriv; 720 struct mlx5_eswitch_rep *rep = rpriv->rep; 721 int err; 722 723 if (new_carrier) { 724 err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 725 rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP); 726 if (err) 727 return err; 728 netif_carrier_on(dev); 729 } else { 730 err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 731 rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); 732 if (err) 733 return err; 734 netif_carrier_off(dev); 735 } 736 return 0; 737 } 738 739 static const struct net_device_ops mlx5e_netdev_ops_rep = { 740 .ndo_open = mlx5e_rep_open, 741 .ndo_stop = mlx5e_rep_close, 742 .ndo_start_xmit = mlx5e_xmit, 743 .ndo_setup_tc = mlx5e_rep_setup_tc, 744 .ndo_get_stats64 = mlx5e_rep_get_stats, 745 .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, 746 .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, 747 .ndo_change_mtu = mlx5e_rep_change_mtu, 748 .ndo_change_carrier = mlx5e_rep_change_carrier, 749 }; 750 751 bool mlx5e_eswitch_uplink_rep(const struct net_device *netdev) 752 { 753 return netdev->netdev_ops == &mlx5e_netdev_ops && 754 mlx5e_is_uplink_rep(netdev_priv(netdev)); 755 } 756 757 bool mlx5e_eswitch_vf_rep(const struct net_device *netdev) 758 { 759 return netdev->netdev_ops == &mlx5e_netdev_ops_rep; 760 } 761 762 /* One indirect TIR set for outer. Inner not supported in reps. */ 763 #define REP_NUM_INDIR_TIRS MLX5E_NUM_INDIR_TIRS 764 765 static int mlx5e_rep_max_nch_limit(struct mlx5_core_dev *mdev) 766 { 767 int max_tir_num = 1 << MLX5_CAP_GEN(mdev, log_max_tir); 768 int num_vports = mlx5_eswitch_get_total_vports(mdev); 769 770 return (max_tir_num - mlx5e_get_pf_num_tirs(mdev) 771 - (num_vports * REP_NUM_INDIR_TIRS)) / num_vports; 772 } 773 774 static void mlx5e_build_rep_params(struct net_device *netdev) 775 { 776 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED; 777 struct mlx5e_priv *priv = netdev_priv(netdev); 778 struct mlx5e_rep_priv *rpriv = priv->ppriv; 779 struct mlx5_eswitch_rep *rep = rpriv->rep; 780 struct mlx5_core_dev *mdev = priv->mdev; 781 struct mlx5e_params *params; 782 783 u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 784 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 785 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 786 787 params = &priv->channels.params; 788 789 params->num_channels = MLX5E_REP_PARAMS_DEF_NUM_CHANNELS; 790 params->hard_mtu = MLX5E_ETH_HARD_MTU; 791 params->sw_mtu = netdev->mtu; 792 793 /* SQ */ 794 if (rep->vport == MLX5_VPORT_UPLINK) 795 params->log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 796 else 797 params->log_sq_size = MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE; 798 799 /* RQ */ 800 mlx5e_build_rq_params(mdev, params); 801 802 /* If netdev is already registered (e.g. move from nic profile to uplink, 803 * RTNL lock must be held before triggering netdev notifiers. 804 */ 805 if (take_rtnl) 806 rtnl_lock(); 807 /* update XDP supported features */ 808 mlx5e_set_xdp_feature(netdev); 809 if (take_rtnl) 810 rtnl_unlock(); 811 812 /* CQ moderation params */ 813 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 814 mlx5e_set_rx_cq_mode_params(params, cq_period_mode); 815 816 params->mqprio.num_tc = 1; 817 if (rep->vport != MLX5_VPORT_UPLINK) 818 params->vlan_strip_disable = true; 819 820 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 821 } 822 823 static void mlx5e_build_rep_netdev(struct net_device *netdev, 824 struct mlx5_core_dev *mdev) 825 { 826 SET_NETDEV_DEV(netdev, mdev->device); 827 netdev->netdev_ops = &mlx5e_netdev_ops_rep; 828 eth_hw_addr_random(netdev); 829 netdev->ethtool_ops = &mlx5e_rep_ethtool_ops; 830 831 netdev->watchdog_timeo = 15 * HZ; 832 833 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 834 netdev->hw_features |= NETIF_F_HW_TC; 835 #endif 836 netdev->hw_features |= NETIF_F_SG; 837 netdev->hw_features |= NETIF_F_IP_CSUM; 838 netdev->hw_features |= NETIF_F_IPV6_CSUM; 839 netdev->hw_features |= NETIF_F_GRO; 840 netdev->hw_features |= NETIF_F_TSO; 841 netdev->hw_features |= NETIF_F_TSO6; 842 netdev->hw_features |= NETIF_F_RXCSUM; 843 844 netdev->features |= netdev->hw_features; 845 netdev->features |= NETIF_F_NETNS_LOCAL; 846 } 847 848 static int mlx5e_init_rep(struct mlx5_core_dev *mdev, 849 struct net_device *netdev) 850 { 851 struct mlx5e_priv *priv = netdev_priv(netdev); 852 853 priv->fs = 854 mlx5e_fs_init(priv->profile, mdev, 855 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 856 priv->dfs_root); 857 if (!priv->fs) { 858 netdev_err(priv->netdev, "FS allocation failed\n"); 859 return -ENOMEM; 860 } 861 862 mlx5e_build_rep_params(netdev); 863 mlx5e_timestamp_init(priv); 864 865 return 0; 866 } 867 868 static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, 869 struct net_device *netdev) 870 { 871 struct mlx5e_priv *priv = netdev_priv(netdev); 872 873 priv->dfs_root = debugfs_create_dir("nic", 874 mlx5_debugfs_get_dev_root(mdev)); 875 876 priv->fs = mlx5e_fs_init(priv->profile, mdev, 877 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 878 priv->dfs_root); 879 if (!priv->fs) { 880 netdev_err(priv->netdev, "FS allocation failed\n"); 881 debugfs_remove_recursive(priv->dfs_root); 882 return -ENOMEM; 883 } 884 885 mlx5e_vxlan_set_netdev_info(priv); 886 mlx5e_build_rep_params(netdev); 887 mlx5e_timestamp_init(priv); 888 return 0; 889 } 890 891 static void mlx5e_cleanup_rep(struct mlx5e_priv *priv) 892 { 893 mlx5e_fs_cleanup(priv->fs); 894 debugfs_remove_recursive(priv->dfs_root); 895 priv->fs = NULL; 896 } 897 898 static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) 899 { 900 struct mlx5e_rep_priv *rpriv = priv->ppriv; 901 struct mlx5_eswitch_rep *rep = rpriv->rep; 902 struct ttc_params ttc_params = {}; 903 int err; 904 905 mlx5e_fs_set_ns(priv->fs, 906 mlx5_get_flow_namespace(priv->mdev, 907 MLX5_FLOW_NAMESPACE_KERNEL), false); 908 909 /* The inner_ttc in the ttc params is intentionally not set */ 910 mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false); 911 912 if (rep->vport != MLX5_VPORT_UPLINK) 913 /* To give uplik rep TTC a lower level for chaining from root ft */ 914 ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1; 915 916 mlx5e_fs_set_ttc(priv->fs, mlx5_create_ttc_table(priv->mdev, &ttc_params), false); 917 if (IS_ERR(mlx5e_fs_get_ttc(priv->fs, false))) { 918 err = PTR_ERR(mlx5e_fs_get_ttc(priv->fs, false)); 919 netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n", 920 err); 921 return err; 922 } 923 return 0; 924 } 925 926 static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv) 927 { 928 struct mlx5e_rep_priv *rpriv = priv->ppriv; 929 struct mlx5_eswitch_rep *rep = rpriv->rep; 930 struct mlx5_flow_table_attr ft_attr = {}; 931 struct mlx5_flow_namespace *ns; 932 int err = 0; 933 934 if (rep->vport != MLX5_VPORT_UPLINK) { 935 /* non uplik reps will skip any bypass tables and go directly to 936 * their own ttc 937 */ 938 rpriv->root_ft = mlx5_get_ttc_flow_table(mlx5e_fs_get_ttc(priv->fs, false)); 939 return 0; 940 } 941 942 /* uplink root ft will be used to auto chain, to ethtool or ttc tables */ 943 ns = mlx5_get_flow_namespace(priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS); 944 if (!ns) { 945 netdev_err(priv->netdev, "Failed to get reps offloads namespace\n"); 946 return -EOPNOTSUPP; 947 } 948 949 ft_attr.max_fte = 0; /* Empty table, miss rule will always point to next table */ 950 ft_attr.prio = 1; 951 ft_attr.level = 1; 952 953 rpriv->root_ft = mlx5_create_flow_table(ns, &ft_attr); 954 if (IS_ERR(rpriv->root_ft)) { 955 err = PTR_ERR(rpriv->root_ft); 956 rpriv->root_ft = NULL; 957 } 958 959 return err; 960 } 961 962 static void mlx5e_destroy_rep_root_ft(struct mlx5e_priv *priv) 963 { 964 struct mlx5e_rep_priv *rpriv = priv->ppriv; 965 struct mlx5_eswitch_rep *rep = rpriv->rep; 966 967 if (rep->vport != MLX5_VPORT_UPLINK) 968 return; 969 mlx5_destroy_flow_table(rpriv->root_ft); 970 } 971 972 static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv *priv) 973 { 974 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 975 struct mlx5e_rep_priv *rpriv = priv->ppriv; 976 struct mlx5_eswitch_rep *rep = rpriv->rep; 977 struct mlx5_flow_handle *flow_rule; 978 struct mlx5_flow_destination dest; 979 980 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 981 dest.ft = rpriv->root_ft; 982 983 flow_rule = mlx5_eswitch_create_vport_rx_rule(esw, rep->vport, &dest); 984 if (IS_ERR(flow_rule)) 985 return PTR_ERR(flow_rule); 986 rpriv->vport_rx_rule = flow_rule; 987 return 0; 988 } 989 990 static void rep_vport_rx_rule_destroy(struct mlx5e_priv *priv) 991 { 992 struct mlx5e_rep_priv *rpriv = priv->ppriv; 993 994 if (!rpriv->vport_rx_rule) 995 return; 996 997 mlx5_del_flow_rules(rpriv->vport_rx_rule); 998 rpriv->vport_rx_rule = NULL; 999 } 1000 1001 int mlx5e_rep_bond_update(struct mlx5e_priv *priv, bool cleanup) 1002 { 1003 rep_vport_rx_rule_destroy(priv); 1004 1005 return cleanup ? 0 : mlx5e_create_rep_vport_rx_rule(priv); 1006 } 1007 1008 static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) 1009 { 1010 struct mlx5_core_dev *mdev = priv->mdev; 1011 int err; 1012 1013 priv->rx_res = mlx5e_rx_res_alloc(); 1014 if (!priv->rx_res) { 1015 err = -ENOMEM; 1016 goto err_free_fs; 1017 } 1018 1019 mlx5e_fs_init_l2_addr(priv->fs, priv->netdev); 1020 1021 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 1022 if (err) { 1023 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 1024 goto err_rx_res_free; 1025 } 1026 1027 err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, 0, 1028 priv->max_nch, priv->drop_rq.rqn, 1029 &priv->channels.params.packet_merge, 1030 priv->channels.params.num_channels); 1031 if (err) 1032 goto err_close_drop_rq; 1033 1034 err = mlx5e_create_rep_ttc_table(priv); 1035 if (err) 1036 goto err_destroy_rx_res; 1037 1038 err = mlx5e_create_rep_root_ft(priv); 1039 if (err) 1040 goto err_destroy_ttc_table; 1041 1042 err = mlx5e_create_rep_vport_rx_rule(priv); 1043 if (err) 1044 goto err_destroy_root_ft; 1045 1046 mlx5e_ethtool_init_steering(priv->fs); 1047 1048 return 0; 1049 1050 err_destroy_root_ft: 1051 mlx5e_destroy_rep_root_ft(priv); 1052 err_destroy_ttc_table: 1053 mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false)); 1054 err_destroy_rx_res: 1055 mlx5e_rx_res_destroy(priv->rx_res); 1056 err_close_drop_rq: 1057 mlx5e_close_drop_rq(&priv->drop_rq); 1058 err_rx_res_free: 1059 mlx5e_rx_res_free(priv->rx_res); 1060 priv->rx_res = NULL; 1061 err_free_fs: 1062 mlx5e_fs_cleanup(priv->fs); 1063 priv->fs = NULL; 1064 return err; 1065 } 1066 1067 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) 1068 { 1069 mlx5e_ethtool_cleanup_steering(priv->fs); 1070 rep_vport_rx_rule_destroy(priv); 1071 mlx5e_destroy_rep_root_ft(priv); 1072 mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false)); 1073 mlx5e_rx_res_destroy(priv->rx_res); 1074 mlx5e_close_drop_rq(&priv->drop_rq); 1075 mlx5e_rx_res_free(priv->rx_res); 1076 priv->rx_res = NULL; 1077 } 1078 1079 static void mlx5e_rep_mpesw_work(struct work_struct *work) 1080 { 1081 struct mlx5_rep_uplink_priv *uplink_priv = 1082 container_of(work, struct mlx5_rep_uplink_priv, 1083 mpesw_work); 1084 struct mlx5e_rep_priv *rpriv = 1085 container_of(uplink_priv, struct mlx5e_rep_priv, 1086 uplink_priv); 1087 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); 1088 1089 rep_vport_rx_rule_destroy(priv); 1090 mlx5e_create_rep_vport_rx_rule(priv); 1091 } 1092 1093 static int mlx5e_init_ul_rep_rx(struct mlx5e_priv *priv) 1094 { 1095 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1096 int err; 1097 1098 mlx5e_create_q_counters(priv); 1099 err = mlx5e_init_rep_rx(priv); 1100 if (err) 1101 goto out; 1102 1103 mlx5e_tc_int_port_init_rep_rx(priv); 1104 1105 INIT_WORK(&rpriv->uplink_priv.mpesw_work, mlx5e_rep_mpesw_work); 1106 1107 out: 1108 return err; 1109 } 1110 1111 static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv *priv) 1112 { 1113 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1114 1115 cancel_work_sync(&rpriv->uplink_priv.mpesw_work); 1116 mlx5e_tc_int_port_cleanup_rep_rx(priv); 1117 mlx5e_cleanup_rep_rx(priv); 1118 mlx5e_destroy_q_counters(priv); 1119 } 1120 1121 static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv *rpriv) 1122 { 1123 struct mlx5_rep_uplink_priv *uplink_priv; 1124 struct net_device *netdev; 1125 struct mlx5e_priv *priv; 1126 int err; 1127 1128 netdev = rpriv->netdev; 1129 priv = netdev_priv(netdev); 1130 uplink_priv = &rpriv->uplink_priv; 1131 1132 err = mlx5e_rep_tc_init(rpriv); 1133 if (err) 1134 return err; 1135 1136 mlx5_init_port_tun_entropy(&uplink_priv->tun_entropy, priv->mdev); 1137 1138 mlx5e_rep_bond_init(rpriv); 1139 err = mlx5e_rep_tc_netdevice_event_register(rpriv); 1140 if (err) { 1141 mlx5_core_err(priv->mdev, "Failed to register netdev notifier, err: %d\n", 1142 err); 1143 goto err_event_reg; 1144 } 1145 1146 return 0; 1147 1148 err_event_reg: 1149 mlx5e_rep_bond_cleanup(rpriv); 1150 mlx5e_rep_tc_cleanup(rpriv); 1151 return err; 1152 } 1153 1154 static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv *rpriv) 1155 { 1156 mlx5e_rep_tc_netdevice_event_unregister(rpriv); 1157 mlx5e_rep_bond_cleanup(rpriv); 1158 mlx5e_rep_tc_cleanup(rpriv); 1159 } 1160 1161 static int mlx5e_init_rep_tx(struct mlx5e_priv *priv) 1162 { 1163 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1164 int err; 1165 1166 err = mlx5e_create_tises(priv); 1167 if (err) { 1168 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 1169 return err; 1170 } 1171 1172 err = mlx5e_rep_neigh_init(rpriv); 1173 if (err) 1174 goto err_neigh_init; 1175 1176 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) { 1177 err = mlx5e_init_uplink_rep_tx(rpriv); 1178 if (err) 1179 goto err_init_tx; 1180 } 1181 1182 err = mlx5e_tc_ht_init(&rpriv->tc_ht); 1183 if (err) 1184 goto err_ht_init; 1185 1186 return 0; 1187 1188 err_ht_init: 1189 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) 1190 mlx5e_cleanup_uplink_rep_tx(rpriv); 1191 err_init_tx: 1192 mlx5e_rep_neigh_cleanup(rpriv); 1193 err_neigh_init: 1194 mlx5e_destroy_tises(priv); 1195 return err; 1196 } 1197 1198 static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) 1199 { 1200 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1201 1202 mlx5e_tc_ht_cleanup(&rpriv->tc_ht); 1203 1204 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) 1205 mlx5e_cleanup_uplink_rep_tx(rpriv); 1206 1207 mlx5e_rep_neigh_cleanup(rpriv); 1208 mlx5e_destroy_tises(priv); 1209 } 1210 1211 static void mlx5e_rep_enable(struct mlx5e_priv *priv) 1212 { 1213 mlx5e_set_netdev_mtu_boundaries(priv); 1214 } 1215 1216 static void mlx5e_rep_disable(struct mlx5e_priv *priv) 1217 { 1218 } 1219 1220 static int mlx5e_update_rep_rx(struct mlx5e_priv *priv) 1221 { 1222 return 0; 1223 } 1224 1225 static int mlx5e_rep_event_mpesw(struct mlx5e_priv *priv) 1226 { 1227 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1228 struct mlx5_eswitch_rep *rep = rpriv->rep; 1229 1230 if (rep->vport != MLX5_VPORT_UPLINK) 1231 return NOTIFY_DONE; 1232 1233 queue_work(priv->wq, &rpriv->uplink_priv.mpesw_work); 1234 1235 return NOTIFY_OK; 1236 } 1237 1238 static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data) 1239 { 1240 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 1241 1242 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 1243 struct mlx5_eqe *eqe = data; 1244 1245 switch (eqe->sub_type) { 1246 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 1247 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 1248 queue_work(priv->wq, &priv->update_carrier_work); 1249 break; 1250 default: 1251 return NOTIFY_DONE; 1252 } 1253 1254 return NOTIFY_OK; 1255 } 1256 1257 if (event == MLX5_DEV_EVENT_PORT_AFFINITY) 1258 return mlx5e_rep_tc_event_port_affinity(priv); 1259 else if (event == MLX5_DEV_EVENT_MULTIPORT_ESW) 1260 return mlx5e_rep_event_mpesw(priv); 1261 1262 return NOTIFY_DONE; 1263 } 1264 1265 static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) 1266 { 1267 struct net_device *netdev = priv->netdev; 1268 struct mlx5_core_dev *mdev = priv->mdev; 1269 u16 max_mtu; 1270 1271 mlx5e_ipsec_init(priv); 1272 1273 netdev->min_mtu = ETH_MIN_MTU; 1274 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1); 1275 netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu); 1276 mlx5e_set_dev_port_mtu(priv); 1277 1278 mlx5e_rep_tc_enable(priv); 1279 1280 if (MLX5_CAP_GEN(mdev, uplink_follow)) 1281 mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, 1282 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); 1283 mlx5_lag_add_netdev(mdev, netdev); 1284 priv->events_nb.notifier_call = uplink_rep_async_event; 1285 mlx5_notifier_register(mdev, &priv->events_nb); 1286 mlx5e_dcbnl_initialize(priv); 1287 mlx5e_dcbnl_init_app(priv); 1288 mlx5e_rep_bridge_init(priv); 1289 1290 netdev->wanted_features |= NETIF_F_HW_TC; 1291 1292 rtnl_lock(); 1293 if (netif_running(netdev)) 1294 mlx5e_open(netdev); 1295 udp_tunnel_nic_reset_ntf(priv->netdev); 1296 netif_device_attach(netdev); 1297 rtnl_unlock(); 1298 } 1299 1300 static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv) 1301 { 1302 struct mlx5_core_dev *mdev = priv->mdev; 1303 1304 rtnl_lock(); 1305 if (netif_running(priv->netdev)) 1306 mlx5e_close(priv->netdev); 1307 netif_device_detach(priv->netdev); 1308 rtnl_unlock(); 1309 1310 mlx5e_rep_bridge_cleanup(priv); 1311 mlx5e_dcbnl_delete_app(priv); 1312 mlx5_notifier_unregister(mdev, &priv->events_nb); 1313 mlx5e_rep_tc_disable(priv); 1314 mlx5_lag_remove_netdev(mdev, priv->netdev); 1315 mlx5_vxlan_reset_to_default(mdev->vxlan); 1316 1317 mlx5e_ipsec_cleanup(priv); 1318 } 1319 1320 static MLX5E_DEFINE_STATS_GRP(sw_rep, 0); 1321 static MLX5E_DEFINE_STATS_GRP(vport_rep, MLX5E_NDO_UPDATE_STATS); 1322 1323 /* The stats groups order is opposite to the update_stats() order calls */ 1324 static mlx5e_stats_grp_t mlx5e_rep_stats_grps[] = { 1325 &MLX5E_STATS_GRP(sw_rep), 1326 &MLX5E_STATS_GRP(vport_rep), 1327 }; 1328 1329 static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv *priv) 1330 { 1331 return ARRAY_SIZE(mlx5e_rep_stats_grps); 1332 } 1333 1334 /* The stats groups order is opposite to the update_stats() order calls */ 1335 static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = { 1336 &MLX5E_STATS_GRP(sw), 1337 &MLX5E_STATS_GRP(qcnt), 1338 &MLX5E_STATS_GRP(vnic_env), 1339 &MLX5E_STATS_GRP(vport), 1340 &MLX5E_STATS_GRP(802_3), 1341 &MLX5E_STATS_GRP(2863), 1342 &MLX5E_STATS_GRP(2819), 1343 &MLX5E_STATS_GRP(phy), 1344 &MLX5E_STATS_GRP(eth_ext), 1345 &MLX5E_STATS_GRP(pcie), 1346 &MLX5E_STATS_GRP(per_prio), 1347 &MLX5E_STATS_GRP(pme), 1348 &MLX5E_STATS_GRP(channels), 1349 &MLX5E_STATS_GRP(per_port_buff_congest), 1350 #ifdef CONFIG_MLX5_EN_IPSEC 1351 &MLX5E_STATS_GRP(ipsec_hw), 1352 &MLX5E_STATS_GRP(ipsec_sw), 1353 #endif 1354 &MLX5E_STATS_GRP(ptp), 1355 }; 1356 1357 static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv) 1358 { 1359 return ARRAY_SIZE(mlx5e_ul_rep_stats_grps); 1360 } 1361 1362 static int 1363 mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter, 1364 struct devlink_fmsg *fmsg, 1365 struct netlink_ext_ack *extack) 1366 { 1367 struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter); 1368 struct mlx5_eswitch_rep *rep = rpriv->rep; 1369 1370 return mlx5_reporter_vnic_diagnose_counters(rep->esw->dev, fmsg, 1371 rep->vport, true); 1372 } 1373 1374 static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = { 1375 .name = "vnic", 1376 .diagnose = mlx5e_rep_vnic_reporter_diagnose, 1377 }; 1378 1379 static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv, 1380 struct devlink_port *dl_port) 1381 { 1382 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1383 struct devlink_health_reporter *reporter; 1384 1385 reporter = devl_port_health_reporter_create(dl_port, 1386 &mlx5_rep_vnic_reporter_ops, 1387 0, rpriv); 1388 if (IS_ERR(reporter)) { 1389 mlx5_core_err(priv->mdev, 1390 "Failed to create representor vnic reporter, err = %ld\n", 1391 PTR_ERR(reporter)); 1392 return; 1393 } 1394 1395 rpriv->rep_vnic_reporter = reporter; 1396 } 1397 1398 static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv) 1399 { 1400 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1401 1402 if (!IS_ERR_OR_NULL(rpriv->rep_vnic_reporter)) 1403 devl_health_reporter_destroy(rpriv->rep_vnic_reporter); 1404 } 1405 1406 static const struct mlx5e_profile mlx5e_rep_profile = { 1407 .init = mlx5e_init_rep, 1408 .cleanup = mlx5e_cleanup_rep, 1409 .init_rx = mlx5e_init_rep_rx, 1410 .cleanup_rx = mlx5e_cleanup_rep_rx, 1411 .init_tx = mlx5e_init_rep_tx, 1412 .cleanup_tx = mlx5e_cleanup_rep_tx, 1413 .enable = mlx5e_rep_enable, 1414 .disable = mlx5e_rep_disable, 1415 .update_rx = mlx5e_update_rep_rx, 1416 .update_stats = mlx5e_stats_update_ndo_stats, 1417 .rx_handlers = &mlx5e_rx_handlers_rep, 1418 .max_tc = 1, 1419 .stats_grps = mlx5e_rep_stats_grps, 1420 .stats_grps_num = mlx5e_rep_stats_grps_num, 1421 .max_nch_limit = mlx5e_rep_max_nch_limit, 1422 }; 1423 1424 static const struct mlx5e_profile mlx5e_uplink_rep_profile = { 1425 .init = mlx5e_init_ul_rep, 1426 .cleanup = mlx5e_cleanup_rep, 1427 .init_rx = mlx5e_init_ul_rep_rx, 1428 .cleanup_rx = mlx5e_cleanup_ul_rep_rx, 1429 .init_tx = mlx5e_init_rep_tx, 1430 .cleanup_tx = mlx5e_cleanup_rep_tx, 1431 .enable = mlx5e_uplink_rep_enable, 1432 .disable = mlx5e_uplink_rep_disable, 1433 .update_rx = mlx5e_update_rep_rx, 1434 .update_stats = mlx5e_stats_update_ndo_stats, 1435 .update_carrier = mlx5e_update_carrier, 1436 .rx_handlers = &mlx5e_rx_handlers_rep, 1437 .max_tc = MLX5E_MAX_NUM_TC, 1438 .stats_grps = mlx5e_ul_rep_stats_grps, 1439 .stats_grps_num = mlx5e_ul_rep_stats_grps_num, 1440 }; 1441 1442 /* e-Switch vport representors */ 1443 static int 1444 mlx5e_vport_uplink_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 1445 { 1446 struct mlx5e_priv *priv = netdev_priv(mlx5_uplink_netdev_get(dev)); 1447 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 1448 1449 rpriv->netdev = priv->netdev; 1450 return mlx5e_netdev_change_profile(priv, &mlx5e_uplink_rep_profile, 1451 rpriv); 1452 } 1453 1454 static void 1455 mlx5e_vport_uplink_rep_unload(struct mlx5e_rep_priv *rpriv) 1456 { 1457 struct net_device *netdev = rpriv->netdev; 1458 struct mlx5e_priv *priv; 1459 1460 priv = netdev_priv(netdev); 1461 1462 mlx5e_netdev_attach_nic_profile(priv); 1463 } 1464 1465 static int 1466 mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 1467 { 1468 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 1469 const struct mlx5e_profile *profile; 1470 struct devlink_port *dl_port; 1471 struct net_device *netdev; 1472 struct mlx5e_priv *priv; 1473 int err; 1474 1475 profile = &mlx5e_rep_profile; 1476 netdev = mlx5e_create_netdev(dev, profile); 1477 if (!netdev) { 1478 mlx5_core_warn(dev, 1479 "Failed to create representor netdev for vport %d\n", 1480 rep->vport); 1481 return -EINVAL; 1482 } 1483 1484 mlx5e_build_rep_netdev(netdev, dev); 1485 rpriv->netdev = netdev; 1486 1487 priv = netdev_priv(netdev); 1488 priv->profile = profile; 1489 priv->ppriv = rpriv; 1490 err = profile->init(dev, netdev); 1491 if (err) { 1492 netdev_warn(netdev, "rep profile init failed, %d\n", err); 1493 goto err_destroy_netdev; 1494 } 1495 1496 err = mlx5e_attach_netdev(netdev_priv(netdev)); 1497 if (err) { 1498 netdev_warn(netdev, 1499 "Failed to attach representor netdev for vport %d\n", 1500 rep->vport); 1501 goto err_cleanup_profile; 1502 } 1503 1504 dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch, 1505 rpriv->rep->vport); 1506 if (!IS_ERR(dl_port)) { 1507 SET_NETDEV_DEVLINK_PORT(netdev, dl_port); 1508 mlx5e_rep_vnic_reporter_create(priv, dl_port); 1509 } 1510 1511 err = register_netdev(netdev); 1512 if (err) { 1513 netdev_warn(netdev, 1514 "Failed to register representor netdev for vport %d\n", 1515 rep->vport); 1516 goto err_detach_netdev; 1517 } 1518 1519 return 0; 1520 1521 err_detach_netdev: 1522 mlx5e_rep_vnic_reporter_destroy(priv); 1523 mlx5e_detach_netdev(netdev_priv(netdev)); 1524 err_cleanup_profile: 1525 priv->profile->cleanup(priv); 1526 1527 err_destroy_netdev: 1528 mlx5e_destroy_netdev(netdev_priv(netdev)); 1529 return err; 1530 } 1531 1532 static int 1533 mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 1534 { 1535 struct mlx5e_rep_priv *rpriv; 1536 int err; 1537 1538 rpriv = kvzalloc(sizeof(*rpriv), GFP_KERNEL); 1539 if (!rpriv) 1540 return -ENOMEM; 1541 1542 /* rpriv->rep to be looked up when profile->init() is called */ 1543 rpriv->rep = rep; 1544 rep->rep_data[REP_ETH].priv = rpriv; 1545 INIT_LIST_HEAD(&rpriv->vport_sqs_list); 1546 1547 if (rep->vport == MLX5_VPORT_UPLINK) 1548 err = mlx5e_vport_uplink_rep_load(dev, rep); 1549 else 1550 err = mlx5e_vport_vf_rep_load(dev, rep); 1551 1552 if (err) 1553 kvfree(rpriv); 1554 1555 return err; 1556 } 1557 1558 static void 1559 mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) 1560 { 1561 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 1562 struct net_device *netdev = rpriv->netdev; 1563 struct mlx5e_priv *priv = netdev_priv(netdev); 1564 void *ppriv = priv->ppriv; 1565 1566 if (rep->vport == MLX5_VPORT_UPLINK) { 1567 mlx5e_vport_uplink_rep_unload(rpriv); 1568 goto free_ppriv; 1569 } 1570 1571 unregister_netdev(netdev); 1572 mlx5e_rep_vnic_reporter_destroy(priv); 1573 mlx5e_detach_netdev(priv); 1574 priv->profile->cleanup(priv); 1575 mlx5e_destroy_netdev(priv); 1576 free_ppriv: 1577 kvfree(ppriv); /* mlx5e_rep_priv */ 1578 } 1579 1580 static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep) 1581 { 1582 struct mlx5e_rep_priv *rpriv; 1583 1584 rpriv = mlx5e_rep_to_rep_priv(rep); 1585 1586 return rpriv->netdev; 1587 } 1588 1589 static void mlx5e_vport_rep_event_unpair(struct mlx5_eswitch_rep *rep, 1590 struct mlx5_eswitch *peer_esw) 1591 { 1592 u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 1593 struct mlx5e_rep_priv *rpriv; 1594 struct mlx5e_rep_sq *rep_sq; 1595 1596 WARN_ON_ONCE(!peer_esw); 1597 rpriv = mlx5e_rep_to_rep_priv(rep); 1598 list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) { 1599 struct mlx5e_rep_sq_peer *sq_peer = xa_load(&rep_sq->sq_peer, i); 1600 1601 if (!sq_peer || sq_peer->peer != peer_esw) 1602 continue; 1603 1604 mlx5_eswitch_del_send_to_vport_rule(sq_peer->rule); 1605 xa_erase(&rep_sq->sq_peer, i); 1606 kfree(sq_peer); 1607 } 1608 } 1609 1610 static int mlx5e_vport_rep_event_pair(struct mlx5_eswitch *esw, 1611 struct mlx5_eswitch_rep *rep, 1612 struct mlx5_eswitch *peer_esw) 1613 { 1614 u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 1615 struct mlx5_flow_handle *flow_rule; 1616 struct mlx5e_rep_sq_peer *sq_peer; 1617 struct mlx5e_rep_priv *rpriv; 1618 struct mlx5e_rep_sq *rep_sq; 1619 int err; 1620 1621 rpriv = mlx5e_rep_to_rep_priv(rep); 1622 list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) { 1623 sq_peer = xa_load(&rep_sq->sq_peer, i); 1624 1625 if (sq_peer && sq_peer->peer) 1626 continue; 1627 1628 flow_rule = mlx5_eswitch_add_send_to_vport_rule(peer_esw, esw, rep, 1629 rep_sq->sqn); 1630 if (IS_ERR(flow_rule)) { 1631 err = PTR_ERR(flow_rule); 1632 goto err_out; 1633 } 1634 1635 if (sq_peer) { 1636 sq_peer->rule = flow_rule; 1637 sq_peer->peer = peer_esw; 1638 continue; 1639 } 1640 sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL); 1641 if (!sq_peer) { 1642 err = -ENOMEM; 1643 goto err_sq_alloc; 1644 } 1645 err = xa_insert(&rep_sq->sq_peer, i, sq_peer, GFP_KERNEL); 1646 if (err) 1647 goto err_xa; 1648 sq_peer->rule = flow_rule; 1649 sq_peer->peer = peer_esw; 1650 } 1651 1652 return 0; 1653 err_xa: 1654 kfree(sq_peer); 1655 err_sq_alloc: 1656 mlx5_eswitch_del_send_to_vport_rule(flow_rule); 1657 err_out: 1658 mlx5e_vport_rep_event_unpair(rep, peer_esw); 1659 return err; 1660 } 1661 1662 static int mlx5e_vport_rep_event(struct mlx5_eswitch *esw, 1663 struct mlx5_eswitch_rep *rep, 1664 enum mlx5_switchdev_event event, 1665 void *data) 1666 { 1667 int err = 0; 1668 1669 if (event == MLX5_SWITCHDEV_EVENT_PAIR) 1670 err = mlx5e_vport_rep_event_pair(esw, rep, data); 1671 else if (event == MLX5_SWITCHDEV_EVENT_UNPAIR) 1672 mlx5e_vport_rep_event_unpair(rep, data); 1673 1674 return err; 1675 } 1676 1677 static const struct mlx5_eswitch_rep_ops rep_ops = { 1678 .load = mlx5e_vport_rep_load, 1679 .unload = mlx5e_vport_rep_unload, 1680 .get_proto_dev = mlx5e_vport_rep_get_proto_dev, 1681 .event = mlx5e_vport_rep_event, 1682 }; 1683 1684 static int mlx5e_rep_probe(struct auxiliary_device *adev, 1685 const struct auxiliary_device_id *id) 1686 { 1687 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 1688 struct mlx5_core_dev *mdev = edev->mdev; 1689 struct mlx5_eswitch *esw; 1690 1691 esw = mdev->priv.eswitch; 1692 mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_ETH); 1693 return 0; 1694 } 1695 1696 static void mlx5e_rep_remove(struct auxiliary_device *adev) 1697 { 1698 struct mlx5_adev *vdev = container_of(adev, struct mlx5_adev, adev); 1699 struct mlx5_core_dev *mdev = vdev->mdev; 1700 struct mlx5_eswitch *esw; 1701 1702 esw = mdev->priv.eswitch; 1703 mlx5_eswitch_unregister_vport_reps(esw, REP_ETH); 1704 } 1705 1706 static const struct auxiliary_device_id mlx5e_rep_id_table[] = { 1707 { .name = MLX5_ADEV_NAME ".eth-rep", }, 1708 {}, 1709 }; 1710 1711 MODULE_DEVICE_TABLE(auxiliary, mlx5e_rep_id_table); 1712 1713 static struct auxiliary_driver mlx5e_rep_driver = { 1714 .name = "eth-rep", 1715 .probe = mlx5e_rep_probe, 1716 .remove = mlx5e_rep_remove, 1717 .id_table = mlx5e_rep_id_table, 1718 }; 1719 1720 int mlx5e_rep_init(void) 1721 { 1722 return auxiliary_driver_register(&mlx5e_rep_driver); 1723 } 1724 1725 void mlx5e_rep_cleanup(void) 1726 { 1727 auxiliary_driver_unregister(&mlx5e_rep_driver); 1728 } 1729