1de8650a8SEran Ben Elisha /* SPDX-License-Identifier: GPL-2.0 */ 2de8650a8SEran Ben Elisha /* Copyright (c) 2019 Mellanox Technologies. */ 3de8650a8SEran Ben Elisha 4de8650a8SEran Ben Elisha #include <net/devlink.h> 5de8650a8SEran Ben Elisha #include "reporter.h" 6de8650a8SEran Ben Elisha #include "lib/eq.h" 7de8650a8SEran Ben Elisha 8de8650a8SEran Ben Elisha #define MLX5E_TX_REPORTER_PER_SQ_MAX_LEN 256 9de8650a8SEran Ben Elisha 10de8650a8SEran Ben Elisha struct mlx5e_tx_err_ctx { 11de8650a8SEran Ben Elisha int (*recover)(struct mlx5e_txqsq *sq); 12de8650a8SEran Ben Elisha struct mlx5e_txqsq *sq; 13de8650a8SEran Ben Elisha }; 14de8650a8SEran Ben Elisha 15de8650a8SEran Ben Elisha static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq) 16de8650a8SEran Ben Elisha { 17de8650a8SEran Ben Elisha unsigned long exp_time = jiffies + msecs_to_jiffies(2000); 18de8650a8SEran Ben Elisha 19de8650a8SEran Ben Elisha while (time_before(jiffies, exp_time)) { 20de8650a8SEran Ben Elisha if (sq->cc == sq->pc) 21de8650a8SEran Ben Elisha return 0; 22de8650a8SEran Ben Elisha 23de8650a8SEran Ben Elisha msleep(20); 24de8650a8SEran Ben Elisha } 25de8650a8SEran Ben Elisha 26de8650a8SEran Ben Elisha netdev_err(sq->channel->netdev, 27de8650a8SEran Ben Elisha "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n", 28de8650a8SEran Ben Elisha sq->sqn, sq->cc, sq->pc); 29de8650a8SEran Ben Elisha 30de8650a8SEran Ben Elisha return -ETIMEDOUT; 31de8650a8SEran Ben Elisha } 32de8650a8SEran Ben Elisha 33de8650a8SEran Ben Elisha static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq) 34de8650a8SEran Ben Elisha { 35de8650a8SEran Ben Elisha WARN_ONCE(sq->cc != sq->pc, 36de8650a8SEran Ben Elisha "SQ 0x%x: cc (0x%x) != pc (0x%x)\n", 37de8650a8SEran Ben Elisha sq->sqn, sq->cc, sq->pc); 38de8650a8SEran Ben Elisha sq->cc = 0; 39de8650a8SEran Ben Elisha sq->dma_fifo_cc = 0; 40de8650a8SEran Ben Elisha sq->pc = 0; 41de8650a8SEran Ben Elisha } 42de8650a8SEran Ben Elisha 43de8650a8SEran Ben Elisha static int mlx5e_sq_to_ready(struct mlx5e_txqsq *sq, int curr_state) 44de8650a8SEran Ben Elisha { 45de8650a8SEran Ben Elisha struct mlx5_core_dev *mdev = sq->channel->mdev; 46de8650a8SEran Ben Elisha struct net_device *dev = sq->channel->netdev; 47de8650a8SEran Ben Elisha struct mlx5e_modify_sq_param msp = {0}; 48de8650a8SEran Ben Elisha int err; 49de8650a8SEran Ben Elisha 50de8650a8SEran Ben Elisha msp.curr_state = curr_state; 51de8650a8SEran Ben Elisha msp.next_state = MLX5_SQC_STATE_RST; 52de8650a8SEran Ben Elisha 53de8650a8SEran Ben Elisha err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 54de8650a8SEran Ben Elisha if (err) { 55de8650a8SEran Ben Elisha netdev_err(dev, "Failed to move sq 0x%x to reset\n", sq->sqn); 56de8650a8SEran Ben Elisha return err; 57de8650a8SEran Ben Elisha } 58de8650a8SEran Ben Elisha 59de8650a8SEran Ben Elisha memset(&msp, 0, sizeof(msp)); 60de8650a8SEran Ben Elisha msp.curr_state = MLX5_SQC_STATE_RST; 61de8650a8SEran Ben Elisha msp.next_state = MLX5_SQC_STATE_RDY; 62de8650a8SEran Ben Elisha 63de8650a8SEran Ben Elisha err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 64de8650a8SEran Ben Elisha if (err) { 65de8650a8SEran Ben Elisha netdev_err(dev, "Failed to move sq 0x%x to ready\n", sq->sqn); 66de8650a8SEran Ben Elisha return err; 67de8650a8SEran Ben Elisha } 68de8650a8SEran Ben Elisha 69de8650a8SEran Ben Elisha return 0; 70de8650a8SEran Ben Elisha } 71de8650a8SEran Ben Elisha 72de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_err_cqe_recover(struct mlx5e_txqsq *sq) 73de8650a8SEran Ben Elisha { 74de8650a8SEran Ben Elisha struct mlx5_core_dev *mdev = sq->channel->mdev; 75de8650a8SEran Ben Elisha struct net_device *dev = sq->channel->netdev; 76de8650a8SEran Ben Elisha u8 state; 77de8650a8SEran Ben Elisha int err; 78de8650a8SEran Ben Elisha 79de8650a8SEran Ben Elisha if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) 80de8650a8SEran Ben Elisha return 0; 81de8650a8SEran Ben Elisha 82de8650a8SEran Ben Elisha err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); 83de8650a8SEran Ben Elisha if (err) { 84de8650a8SEran Ben Elisha netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", 85de8650a8SEran Ben Elisha sq->sqn, err); 86276d197eSAya Levin goto out; 87de8650a8SEran Ben Elisha } 88de8650a8SEran Ben Elisha 89d9a2fcf5SAya Levin if (state != MLX5_SQC_STATE_ERR) 90276d197eSAya Levin goto out; 91de8650a8SEran Ben Elisha 92de8650a8SEran Ben Elisha mlx5e_tx_disable_queue(sq->txq); 93de8650a8SEran Ben Elisha 94de8650a8SEran Ben Elisha err = mlx5e_wait_for_sq_flush(sq); 95de8650a8SEran Ben Elisha if (err) 96276d197eSAya Levin goto out; 97de8650a8SEran Ben Elisha 98de8650a8SEran Ben Elisha /* At this point, no new packets will arrive from the stack as TXQ is 99de8650a8SEran Ben Elisha * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all 100de8650a8SEran Ben Elisha * pending WQEs. SQ can safely reset the SQ. 101de8650a8SEran Ben Elisha */ 102de8650a8SEran Ben Elisha 103de8650a8SEran Ben Elisha err = mlx5e_sq_to_ready(sq, state); 104de8650a8SEran Ben Elisha if (err) 105276d197eSAya Levin goto out; 106de8650a8SEran Ben Elisha 107de8650a8SEran Ben Elisha mlx5e_reset_txqsq_cc_pc(sq); 108de8650a8SEran Ben Elisha sq->stats->recover++; 109276d197eSAya Levin clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 110de8650a8SEran Ben Elisha mlx5e_activate_txqsq(sq); 111de8650a8SEran Ben Elisha 112de8650a8SEran Ben Elisha return 0; 113276d197eSAya Levin out: 114276d197eSAya Levin clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 115276d197eSAya Levin return err; 116de8650a8SEran Ben Elisha } 117de8650a8SEran Ben Elisha 1182c493ae0SEran Ben Elisha static int mlx5_tx_health_report(struct devlink_health_reporter *tx_reporter, 1192c493ae0SEran Ben Elisha char *err_str, 1202c493ae0SEran Ben Elisha struct mlx5e_tx_err_ctx *err_ctx) 1212c493ae0SEran Ben Elisha { 1222c493ae0SEran Ben Elisha if (IS_ERR_OR_NULL(tx_reporter)) { 1232c493ae0SEran Ben Elisha netdev_err(err_ctx->sq->channel->netdev, err_str); 1242c493ae0SEran Ben Elisha return err_ctx->recover(err_ctx->sq); 1252c493ae0SEran Ben Elisha } 1262c493ae0SEran Ben Elisha 1272c493ae0SEran Ben Elisha return devlink_health_report(tx_reporter, err_str, err_ctx); 1282c493ae0SEran Ben Elisha } 1292c493ae0SEran Ben Elisha 130de8650a8SEran Ben Elisha void mlx5e_tx_reporter_err_cqe(struct mlx5e_txqsq *sq) 131de8650a8SEran Ben Elisha { 132de8650a8SEran Ben Elisha char err_str[MLX5E_TX_REPORTER_PER_SQ_MAX_LEN]; 133de8650a8SEran Ben Elisha struct mlx5e_tx_err_ctx err_ctx = {0}; 134de8650a8SEran Ben Elisha 135de8650a8SEran Ben Elisha err_ctx.sq = sq; 136de8650a8SEran Ben Elisha err_ctx.recover = mlx5e_tx_reporter_err_cqe_recover; 137de8650a8SEran Ben Elisha sprintf(err_str, "ERR CQE on SQ: 0x%x", sq->sqn); 138de8650a8SEran Ben Elisha 1392c493ae0SEran Ben Elisha mlx5_tx_health_report(sq->channel->priv->tx_reporter, err_str, 140de8650a8SEran Ben Elisha &err_ctx); 141de8650a8SEran Ben Elisha } 142de8650a8SEran Ben Elisha 1437d91126bSEran Ben Elisha static int mlx5e_tx_reporter_timeout_recover(struct mlx5e_txqsq *sq) 1447d91126bSEran Ben Elisha { 1457d91126bSEran Ben Elisha struct mlx5_eq_comp *eq = sq->cq.mcq.eq; 1467d91126bSEran Ben Elisha u32 eqe_count; 1477d91126bSEran Ben Elisha 1487d91126bSEran Ben Elisha netdev_err(sq->channel->netdev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n", 1497d91126bSEran Ben Elisha eq->core.eqn, eq->core.cons_index, eq->core.irqn); 1507d91126bSEran Ben Elisha 1517d91126bSEran Ben Elisha eqe_count = mlx5_eq_poll_irq_disabled(eq); 1527d91126bSEran Ben Elisha if (!eqe_count) { 1537d91126bSEran Ben Elisha clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 15439825350SAya Levin return -EIO; 1557d91126bSEran Ben Elisha } 1567d91126bSEran Ben Elisha 1577d91126bSEran Ben Elisha netdev_err(sq->channel->netdev, "Recover %d eqes on EQ 0x%x\n", 1587d91126bSEran Ben Elisha eqe_count, eq->core.eqn); 1597d91126bSEran Ben Elisha sq->channel->stats->eq_rearm++; 16039825350SAya Levin return 0; 1617d91126bSEran Ben Elisha } 1627d91126bSEran Ben Elisha 1637d91126bSEran Ben Elisha int mlx5e_tx_reporter_timeout(struct mlx5e_txqsq *sq) 1647d91126bSEran Ben Elisha { 1657d91126bSEran Ben Elisha char err_str[MLX5E_TX_REPORTER_PER_SQ_MAX_LEN]; 1667d91126bSEran Ben Elisha struct mlx5e_tx_err_ctx err_ctx; 1677d91126bSEran Ben Elisha 1687d91126bSEran Ben Elisha err_ctx.sq = sq; 1697d91126bSEran Ben Elisha err_ctx.recover = mlx5e_tx_reporter_timeout_recover; 1707d91126bSEran Ben Elisha sprintf(err_str, 1717d91126bSEran Ben Elisha "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u\n", 1727d91126bSEran Ben Elisha sq->channel->ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc, 1737d91126bSEran Ben Elisha jiffies_to_usecs(jiffies - sq->txq->trans_start)); 1747d91126bSEran Ben Elisha 1752c493ae0SEran Ben Elisha return mlx5_tx_health_report(sq->channel->priv->tx_reporter, err_str, 1767d91126bSEran Ben Elisha &err_ctx); 1777d91126bSEran Ben Elisha } 1787d91126bSEran Ben Elisha 179de8650a8SEran Ben Elisha /* state lock cannot be grabbed within this function. 180de8650a8SEran Ben Elisha * It can cause a dead lock or a read-after-free. 181de8650a8SEran Ben Elisha */ 1826bdbc1cbSEran Ben Elisha static int mlx5e_tx_reporter_recover_from_ctx(struct mlx5e_tx_err_ctx *err_ctx) 183de8650a8SEran Ben Elisha { 184de8650a8SEran Ben Elisha return err_ctx->recover(err_ctx->sq); 185de8650a8SEran Ben Elisha } 186de8650a8SEran Ben Elisha 187de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_recover_all(struct mlx5e_priv *priv) 188de8650a8SEran Ben Elisha { 189192fba79SEran Ben Elisha int err = 0; 190de8650a8SEran Ben Elisha 191de8650a8SEran Ben Elisha rtnl_lock(); 192de8650a8SEran Ben Elisha mutex_lock(&priv->state_lock); 193192fba79SEran Ben Elisha 194192fba79SEran Ben Elisha if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 195192fba79SEran Ben Elisha goto out; 196192fba79SEran Ben Elisha 197484c1adaSEran Ben Elisha err = mlx5e_safe_reopen_channels(priv); 198192fba79SEran Ben Elisha 199192fba79SEran Ben Elisha out: 200de8650a8SEran Ben Elisha mutex_unlock(&priv->state_lock); 201de8650a8SEran Ben Elisha rtnl_unlock(); 202de8650a8SEran Ben Elisha 203de8650a8SEran Ben Elisha return err; 204de8650a8SEran Ben Elisha } 205de8650a8SEran Ben Elisha 206de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_recover(struct devlink_health_reporter *reporter, 207de8650a8SEran Ben Elisha void *context) 208de8650a8SEran Ben Elisha { 209de8650a8SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 210de8650a8SEran Ben Elisha struct mlx5e_tx_err_ctx *err_ctx = context; 211de8650a8SEran Ben Elisha 212de8650a8SEran Ben Elisha return err_ctx ? mlx5e_tx_reporter_recover_from_ctx(err_ctx) : 213de8650a8SEran Ben Elisha mlx5e_tx_reporter_recover_all(priv); 214de8650a8SEran Ben Elisha } 215de8650a8SEran Ben Elisha 216de8650a8SEran Ben Elisha static int 217de8650a8SEran Ben Elisha mlx5e_tx_reporter_build_diagnose_output(struct devlink_fmsg *fmsg, 218de8650a8SEran Ben Elisha u32 sqn, u8 state, bool stopped) 219de8650a8SEran Ben Elisha { 220de8650a8SEran Ben Elisha int err; 221de8650a8SEran Ben Elisha 222de8650a8SEran Ben Elisha err = devlink_fmsg_obj_nest_start(fmsg); 223de8650a8SEran Ben Elisha if (err) 224de8650a8SEran Ben Elisha return err; 225de8650a8SEran Ben Elisha 226de8650a8SEran Ben Elisha err = devlink_fmsg_u32_pair_put(fmsg, "sqn", sqn); 227de8650a8SEran Ben Elisha if (err) 228de8650a8SEran Ben Elisha return err; 229de8650a8SEran Ben Elisha 230de8650a8SEran Ben Elisha err = devlink_fmsg_u8_pair_put(fmsg, "HW state", state); 231de8650a8SEran Ben Elisha if (err) 232de8650a8SEran Ben Elisha return err; 233de8650a8SEran Ben Elisha 234de8650a8SEran Ben Elisha err = devlink_fmsg_bool_pair_put(fmsg, "stopped", stopped); 235de8650a8SEran Ben Elisha if (err) 236de8650a8SEran Ben Elisha return err; 237de8650a8SEran Ben Elisha 238de8650a8SEran Ben Elisha err = devlink_fmsg_obj_nest_end(fmsg); 239de8650a8SEran Ben Elisha if (err) 240de8650a8SEran Ben Elisha return err; 241de8650a8SEran Ben Elisha 242de8650a8SEran Ben Elisha return 0; 243de8650a8SEran Ben Elisha } 244de8650a8SEran Ben Elisha 245de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter, 246de8650a8SEran Ben Elisha struct devlink_fmsg *fmsg) 247de8650a8SEran Ben Elisha { 248de8650a8SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 249de8650a8SEran Ben Elisha int i, err = 0; 250de8650a8SEran Ben Elisha 251de8650a8SEran Ben Elisha mutex_lock(&priv->state_lock); 252de8650a8SEran Ben Elisha 253de8650a8SEran Ben Elisha if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 254de8650a8SEran Ben Elisha goto unlock; 255de8650a8SEran Ben Elisha 256de8650a8SEran Ben Elisha err = devlink_fmsg_arr_pair_nest_start(fmsg, "SQs"); 257de8650a8SEran Ben Elisha if (err) 258de8650a8SEran Ben Elisha goto unlock; 259de8650a8SEran Ben Elisha 260de8650a8SEran Ben Elisha for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; 261de8650a8SEran Ben Elisha i++) { 262de8650a8SEran Ben Elisha struct mlx5e_txqsq *sq = priv->txq2sq[i]; 263de8650a8SEran Ben Elisha u8 state; 264de8650a8SEran Ben Elisha 265de8650a8SEran Ben Elisha err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state); 266de8650a8SEran Ben Elisha if (err) 26799d31cbdSAya Levin goto unlock; 268de8650a8SEran Ben Elisha 269de8650a8SEran Ben Elisha err = mlx5e_tx_reporter_build_diagnose_output(fmsg, sq->sqn, 270de8650a8SEran Ben Elisha state, 271de8650a8SEran Ben Elisha netif_xmit_stopped(sq->txq)); 272de8650a8SEran Ben Elisha if (err) 27399d31cbdSAya Levin goto unlock; 274de8650a8SEran Ben Elisha } 275de8650a8SEran Ben Elisha err = devlink_fmsg_arr_pair_nest_end(fmsg); 276de8650a8SEran Ben Elisha if (err) 277de8650a8SEran Ben Elisha goto unlock; 278de8650a8SEran Ben Elisha 279de8650a8SEran Ben Elisha unlock: 280de8650a8SEran Ben Elisha mutex_unlock(&priv->state_lock); 281de8650a8SEran Ben Elisha return err; 282de8650a8SEran Ben Elisha } 283de8650a8SEran Ben Elisha 284de8650a8SEran Ben Elisha static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = { 285de8650a8SEran Ben Elisha .name = "tx", 286de8650a8SEran Ben Elisha .recover = mlx5e_tx_reporter_recover, 287de8650a8SEran Ben Elisha .diagnose = mlx5e_tx_reporter_diagnose, 288de8650a8SEran Ben Elisha }; 289de8650a8SEran Ben Elisha 290de8650a8SEran Ben Elisha #define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500 291de8650a8SEran Ben Elisha 292de8650a8SEran Ben Elisha int mlx5e_tx_reporter_create(struct mlx5e_priv *priv) 293de8650a8SEran Ben Elisha { 294de8650a8SEran Ben Elisha struct mlx5_core_dev *mdev = priv->mdev; 295de8650a8SEran Ben Elisha struct devlink *devlink = priv_to_devlink(mdev); 296de8650a8SEran Ben Elisha 297de8650a8SEran Ben Elisha priv->tx_reporter = 298de8650a8SEran Ben Elisha devlink_health_reporter_create(devlink, &mlx5_tx_reporter_ops, 299de8650a8SEran Ben Elisha MLX5_REPORTER_TX_GRACEFUL_PERIOD, 300de8650a8SEran Ben Elisha true, priv); 301772ac5e2SEran Ben Elisha if (IS_ERR(priv->tx_reporter)) 302de8650a8SEran Ben Elisha netdev_warn(priv->netdev, 303de8650a8SEran Ben Elisha "Failed to create tx reporter, err = %ld\n", 304de8650a8SEran Ben Elisha PTR_ERR(priv->tx_reporter)); 3052e5b0534SEran Ben Elisha return IS_ERR_OR_NULL(priv->tx_reporter); 306de8650a8SEran Ben Elisha } 307de8650a8SEran Ben Elisha 308de8650a8SEran Ben Elisha void mlx5e_tx_reporter_destroy(struct mlx5e_priv *priv) 309de8650a8SEran Ben Elisha { 310de8650a8SEran Ben Elisha if (IS_ERR_OR_NULL(priv->tx_reporter)) 311de8650a8SEran Ben Elisha return; 312de8650a8SEran Ben Elisha 313de8650a8SEran Ben Elisha devlink_health_reporter_destroy(priv->tx_reporter); 314de8650a8SEran Ben Elisha } 315