1de8650a8SEran Ben Elisha /* SPDX-License-Identifier: GPL-2.0 */ 2de8650a8SEran Ben Elisha /* Copyright (c) 2019 Mellanox Technologies. */ 3de8650a8SEran Ben Elisha 44edc17fdSAya Levin #include "health.h" 5de8650a8SEran Ben Elisha 6de8650a8SEran Ben Elisha static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq) 7de8650a8SEran Ben Elisha { 8de8650a8SEran Ben Elisha unsigned long exp_time = jiffies + msecs_to_jiffies(2000); 9de8650a8SEran Ben Elisha 10de8650a8SEran Ben Elisha while (time_before(jiffies, exp_time)) { 11de8650a8SEran Ben Elisha if (sq->cc == sq->pc) 12de8650a8SEran Ben Elisha return 0; 13de8650a8SEran Ben Elisha 14de8650a8SEran Ben Elisha msleep(20); 15de8650a8SEran Ben Elisha } 16de8650a8SEran Ben Elisha 17de8650a8SEran Ben Elisha netdev_err(sq->channel->netdev, 18de8650a8SEran Ben Elisha "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n", 19de8650a8SEran Ben Elisha sq->sqn, sq->cc, sq->pc); 20de8650a8SEran Ben Elisha 21de8650a8SEran Ben Elisha return -ETIMEDOUT; 22de8650a8SEran Ben Elisha } 23de8650a8SEran Ben Elisha 24de8650a8SEran Ben Elisha static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq) 25de8650a8SEran Ben Elisha { 26de8650a8SEran Ben Elisha WARN_ONCE(sq->cc != sq->pc, 27de8650a8SEran Ben Elisha "SQ 0x%x: cc (0x%x) != pc (0x%x)\n", 28de8650a8SEran Ben Elisha sq->sqn, sq->cc, sq->pc); 29de8650a8SEran Ben Elisha sq->cc = 0; 30de8650a8SEran Ben Elisha sq->dma_fifo_cc = 0; 31de8650a8SEran Ben Elisha sq->pc = 0; 32de8650a8SEran Ben Elisha } 33de8650a8SEran Ben Elisha 34c50de4afSAya Levin static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) 35de8650a8SEran Ben Elisha { 36c50de4afSAya Levin struct mlx5_core_dev *mdev; 37c50de4afSAya Levin struct net_device *dev; 38c50de4afSAya Levin struct mlx5e_txqsq *sq; 39de8650a8SEran Ben Elisha u8 state; 40de8650a8SEran Ben Elisha int err; 41de8650a8SEran Ben Elisha 42c50de4afSAya Levin sq = ctx; 43c50de4afSAya Levin mdev = sq->channel->mdev; 44c50de4afSAya Levin dev = sq->channel->netdev; 45c50de4afSAya Levin 46c50de4afSAya Levin if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) 47c50de4afSAya Levin return 0; 48c50de4afSAya Levin 49de8650a8SEran Ben Elisha err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); 50de8650a8SEran Ben Elisha if (err) { 51de8650a8SEran Ben Elisha netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", 52de8650a8SEran Ben Elisha sq->sqn, err); 53276d197eSAya Levin goto out; 54de8650a8SEran Ben Elisha } 55de8650a8SEran Ben Elisha 56d9a2fcf5SAya Levin if (state != MLX5_SQC_STATE_ERR) 57276d197eSAya Levin goto out; 58de8650a8SEran Ben Elisha 59de8650a8SEran Ben Elisha mlx5e_tx_disable_queue(sq->txq); 60de8650a8SEran Ben Elisha 61de8650a8SEran Ben Elisha err = mlx5e_wait_for_sq_flush(sq); 62de8650a8SEran Ben Elisha if (err) 63276d197eSAya Levin goto out; 64de8650a8SEran Ben Elisha 65de8650a8SEran Ben Elisha /* At this point, no new packets will arrive from the stack as TXQ is 66de8650a8SEran Ben Elisha * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all 67de8650a8SEran Ben Elisha * pending WQEs. SQ can safely reset the SQ. 68de8650a8SEran Ben Elisha */ 69de8650a8SEran Ben Elisha 70c50de4afSAya Levin err = mlx5e_health_sq_to_ready(sq->channel, sq->sqn); 71de8650a8SEran Ben Elisha if (err) 72276d197eSAya Levin goto out; 73de8650a8SEran Ben Elisha 74de8650a8SEran Ben Elisha mlx5e_reset_txqsq_cc_pc(sq); 75de8650a8SEran Ben Elisha sq->stats->recover++; 76276d197eSAya Levin clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 77de8650a8SEran Ben Elisha mlx5e_activate_txqsq(sq); 78de8650a8SEran Ben Elisha 79de8650a8SEran Ben Elisha return 0; 80276d197eSAya Levin out: 81276d197eSAya Levin clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 82276d197eSAya Levin return err; 83de8650a8SEran Ben Elisha } 84de8650a8SEran Ben Elisha 85c50de4afSAya Levin static int mlx5e_tx_reporter_timeout_recover(void *ctx) 867d91126bSEran Ben Elisha { 87c50de4afSAya Levin struct mlx5_eq_comp *eq; 88c50de4afSAya Levin struct mlx5e_txqsq *sq; 89c50de4afSAya Levin int err; 907d91126bSEran Ben Elisha 91c50de4afSAya Levin sq = ctx; 92c50de4afSAya Levin eq = sq->cq.mcq.eq; 93c50de4afSAya Levin err = mlx5e_health_channel_eq_recover(eq, sq->channel); 94c50de4afSAya Levin if (err) 957d91126bSEran Ben Elisha clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 967d91126bSEran Ben Elisha 97c50de4afSAya Levin return err; 987d91126bSEran Ben Elisha } 997d91126bSEran Ben Elisha 100de8650a8SEran Ben Elisha /* state lock cannot be grabbed within this function. 101de8650a8SEran Ben Elisha * It can cause a dead lock or a read-after-free. 102de8650a8SEran Ben Elisha */ 103c50de4afSAya Levin static int mlx5e_tx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx) 104de8650a8SEran Ben Elisha { 105c50de4afSAya Levin return err_ctx->recover(err_ctx->ctx); 106de8650a8SEran Ben Elisha } 107de8650a8SEran Ben Elisha 108de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_recover(struct devlink_health_reporter *reporter, 109e7a98105SJiri Pirko void *context, 110e7a98105SJiri Pirko struct netlink_ext_ack *extack) 111de8650a8SEran Ben Elisha { 112de8650a8SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 113c50de4afSAya Levin struct mlx5e_err_ctx *err_ctx = context; 114de8650a8SEran Ben Elisha 115de8650a8SEran Ben Elisha return err_ctx ? mlx5e_tx_reporter_recover_from_ctx(err_ctx) : 116c50de4afSAya Levin mlx5e_health_recover_channels(priv); 117de8650a8SEran Ben Elisha } 118de8650a8SEran Ben Elisha 119de8650a8SEran Ben Elisha static int 120de8650a8SEran Ben Elisha mlx5e_tx_reporter_build_diagnose_output(struct devlink_fmsg *fmsg, 1212d708887SAya Levin struct mlx5e_txqsq *sq, int tc) 122de8650a8SEran Ben Elisha { 123dd921fd2SAya Levin struct mlx5e_priv *priv = sq->channel->priv; 124dd921fd2SAya Levin bool stopped = netif_xmit_stopped(sq->txq); 125dd921fd2SAya Levin u8 state; 126de8650a8SEran Ben Elisha int err; 127de8650a8SEran Ben Elisha 128dd921fd2SAya Levin err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state); 129dd921fd2SAya Levin if (err) 130dd921fd2SAya Levin return err; 131dd921fd2SAya Levin 132de8650a8SEran Ben Elisha err = devlink_fmsg_obj_nest_start(fmsg); 133de8650a8SEran Ben Elisha if (err) 134de8650a8SEran Ben Elisha return err; 135de8650a8SEran Ben Elisha 1362d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "channel ix", sq->ch_ix); 1372d708887SAya Levin if (err) 1382d708887SAya Levin return err; 1392d708887SAya Levin 1402d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "tc", tc); 1412d708887SAya Levin if (err) 1422d708887SAya Levin return err; 1432d708887SAya Levin 1442d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "txq ix", sq->txq_ix); 1452d708887SAya Levin if (err) 1462d708887SAya Levin return err; 1472d708887SAya Levin 148dd921fd2SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "sqn", sq->sqn); 149de8650a8SEran Ben Elisha if (err) 150de8650a8SEran Ben Elisha return err; 151de8650a8SEran Ben Elisha 152de8650a8SEran Ben Elisha err = devlink_fmsg_u8_pair_put(fmsg, "HW state", state); 153de8650a8SEran Ben Elisha if (err) 154de8650a8SEran Ben Elisha return err; 155de8650a8SEran Ben Elisha 156de8650a8SEran Ben Elisha err = devlink_fmsg_bool_pair_put(fmsg, "stopped", stopped); 157de8650a8SEran Ben Elisha if (err) 158de8650a8SEran Ben Elisha return err; 159de8650a8SEran Ben Elisha 1602d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "cc", sq->cc); 1612d708887SAya Levin if (err) 1622d708887SAya Levin return err; 1632d708887SAya Levin 1642d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "pc", sq->pc); 1652d708887SAya Levin if (err) 1662d708887SAya Levin return err; 1672d708887SAya Levin 1682bf09e60SAya Levin err = mlx5e_reporter_cq_diagnose(&sq->cq, fmsg); 1692bf09e60SAya Levin if (err) 1702bf09e60SAya Levin return err; 1712bf09e60SAya Levin 172de8650a8SEran Ben Elisha err = devlink_fmsg_obj_nest_end(fmsg); 173de8650a8SEran Ben Elisha if (err) 174de8650a8SEran Ben Elisha return err; 175de8650a8SEran Ben Elisha 176de8650a8SEran Ben Elisha return 0; 177de8650a8SEran Ben Elisha } 178de8650a8SEran Ben Elisha 179de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter, 180e7a98105SJiri Pirko struct devlink_fmsg *fmsg, 181e7a98105SJiri Pirko struct netlink_ext_ack *extack) 182de8650a8SEran Ben Elisha { 183de8650a8SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 1842d708887SAya Levin struct mlx5e_txqsq *generic_sq = priv->txq2sq[0]; 1852d708887SAya Levin u32 sq_stride, sq_sz; 1862d708887SAya Levin 1872d708887SAya Levin int i, tc, err = 0; 188de8650a8SEran Ben Elisha 189de8650a8SEran Ben Elisha mutex_lock(&priv->state_lock); 190de8650a8SEran Ben Elisha 191de8650a8SEran Ben Elisha if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 192de8650a8SEran Ben Elisha goto unlock; 193de8650a8SEran Ben Elisha 1942d708887SAya Levin sq_sz = mlx5_wq_cyc_get_size(&generic_sq->wq); 1952d708887SAya Levin sq_stride = MLX5_SEND_WQE_BB; 1962d708887SAya Levin 1972d708887SAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "Common Config"); 1982d708887SAya Levin if (err) 1992d708887SAya Levin goto unlock; 2002d708887SAya Levin 2012d708887SAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "SQ"); 2022d708887SAya Levin if (err) 2032d708887SAya Levin goto unlock; 2042d708887SAya Levin 2052d708887SAya Levin err = devlink_fmsg_u64_pair_put(fmsg, "stride size", sq_stride); 2062d708887SAya Levin if (err) 2072d708887SAya Levin goto unlock; 2082d708887SAya Levin 2092d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "size", sq_sz); 2102d708887SAya Levin if (err) 2112d708887SAya Levin goto unlock; 2122d708887SAya Levin 2132bf09e60SAya Levin err = mlx5e_reporter_cq_common_diagnose(&generic_sq->cq, fmsg); 2142bf09e60SAya Levin if (err) 2152bf09e60SAya Levin goto unlock; 2162bf09e60SAya Levin 2172d708887SAya Levin err = mlx5e_reporter_named_obj_nest_end(fmsg); 2182d708887SAya Levin if (err) 2192d708887SAya Levin goto unlock; 2202d708887SAya Levin 2212d708887SAya Levin err = mlx5e_reporter_named_obj_nest_end(fmsg); 2222d708887SAya Levin if (err) 2232d708887SAya Levin goto unlock; 2242d708887SAya Levin 225de8650a8SEran Ben Elisha err = devlink_fmsg_arr_pair_nest_start(fmsg, "SQs"); 226de8650a8SEran Ben Elisha if (err) 227de8650a8SEran Ben Elisha goto unlock; 228de8650a8SEran Ben Elisha 2292d708887SAya Levin for (i = 0; i < priv->channels.num; i++) { 2302d708887SAya Levin struct mlx5e_channel *c = priv->channels.c[i]; 231de8650a8SEran Ben Elisha 2322d708887SAya Levin for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 2332d708887SAya Levin struct mlx5e_txqsq *sq = &c->sq[tc]; 2342d708887SAya Levin 2352d708887SAya Levin err = mlx5e_tx_reporter_build_diagnose_output(fmsg, sq, tc); 236de8650a8SEran Ben Elisha if (err) 23799d31cbdSAya Levin goto unlock; 238de8650a8SEran Ben Elisha } 2392d708887SAya Levin } 240de8650a8SEran Ben Elisha err = devlink_fmsg_arr_pair_nest_end(fmsg); 241de8650a8SEran Ben Elisha if (err) 242de8650a8SEran Ben Elisha goto unlock; 243de8650a8SEran Ben Elisha 244de8650a8SEran Ben Elisha unlock: 245de8650a8SEran Ben Elisha mutex_unlock(&priv->state_lock); 246de8650a8SEran Ben Elisha return err; 247de8650a8SEran Ben Elisha } 248de8650a8SEran Ben Elisha 2495f29458bSAya Levin static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 2505f29458bSAya Levin void *ctx) 2515f29458bSAya Levin { 2525f29458bSAya Levin struct mlx5_rsc_key key = {}; 2535f29458bSAya Levin struct mlx5e_txqsq *sq = ctx; 2545f29458bSAya Levin int err; 2555f29458bSAya Levin 2565f29458bSAya Levin if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 2575f29458bSAya Levin return 0; 2585f29458bSAya Levin 2595f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "SX Slice"); 2605f29458bSAya Levin if (err) 2615f29458bSAya Levin return err; 2625f29458bSAya Levin 2635f29458bSAya Levin key.size = PAGE_SIZE; 2645f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 2655f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 2665f29458bSAya Levin if (err) 2675f29458bSAya Levin return err; 2685f29458bSAya Levin 2695f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_end(fmsg); 2705f29458bSAya Levin if (err) 2715f29458bSAya Levin return err; 2725f29458bSAya Levin 2735f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "SQ"); 2745f29458bSAya Levin if (err) 2755f29458bSAya Levin return err; 2765f29458bSAya Levin 2775f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "QPC"); 2785f29458bSAya Levin if (err) 2795f29458bSAya Levin return err; 2805f29458bSAya Levin 2815f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 2825f29458bSAya Levin key.index1 = sq->sqn; 2835f29458bSAya Levin key.num_of_obj1 = 1; 2845f29458bSAya Levin 2855f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 2865f29458bSAya Levin if (err) 2875f29458bSAya Levin return err; 2885f29458bSAya Levin 2895f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_end(fmsg); 2905f29458bSAya Levin if (err) 2915f29458bSAya Levin return err; 2925f29458bSAya Levin 2935f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "send_buff"); 2945f29458bSAya Levin if (err) 2955f29458bSAya Levin return err; 2965f29458bSAya Levin 2975f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_SND_BUFF; 2985f29458bSAya Levin key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 2995f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 3005f29458bSAya Levin if (err) 3015f29458bSAya Levin return err; 3025f29458bSAya Levin 3035f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_end(fmsg); 3045f29458bSAya Levin if (err) 3055f29458bSAya Levin return err; 3065f29458bSAya Levin 3075f29458bSAya Levin return mlx5e_reporter_named_obj_nest_end(fmsg); 3085f29458bSAya Levin } 3095f29458bSAya Levin 3105f29458bSAya Levin static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv, 3115f29458bSAya Levin struct devlink_fmsg *fmsg) 3125f29458bSAya Levin { 3135f29458bSAya Levin struct mlx5_rsc_key key = {}; 3145f29458bSAya Levin int i, tc, err; 3155f29458bSAya Levin 3165f29458bSAya Levin if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3175f29458bSAya Levin return 0; 3185f29458bSAya Levin 3195f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_start(fmsg, "SX Slice"); 3205f29458bSAya Levin if (err) 3215f29458bSAya Levin return err; 3225f29458bSAya Levin 3235f29458bSAya Levin key.size = PAGE_SIZE; 3245f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 3255f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 3265f29458bSAya Levin if (err) 3275f29458bSAya Levin return err; 3285f29458bSAya Levin 3295f29458bSAya Levin err = mlx5e_reporter_named_obj_nest_end(fmsg); 3305f29458bSAya Levin if (err) 3315f29458bSAya Levin return err; 3325f29458bSAya Levin 3335f29458bSAya Levin err = devlink_fmsg_arr_pair_nest_start(fmsg, "SQs"); 3345f29458bSAya Levin if (err) 3355f29458bSAya Levin return err; 3365f29458bSAya Levin 3375f29458bSAya Levin for (i = 0; i < priv->channels.num; i++) { 3385f29458bSAya Levin struct mlx5e_channel *c = priv->channels.c[i]; 3395f29458bSAya Levin 3405f29458bSAya Levin for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 3415f29458bSAya Levin struct mlx5e_txqsq *sq = &c->sq[tc]; 3425f29458bSAya Levin 3435f29458bSAya Levin err = mlx5e_health_queue_dump(priv, fmsg, sq->sqn, "SQ"); 3445f29458bSAya Levin if (err) 3455f29458bSAya Levin return err; 3465f29458bSAya Levin } 3475f29458bSAya Levin } 3485f29458bSAya Levin return devlink_fmsg_arr_pair_nest_end(fmsg); 3495f29458bSAya Levin } 3505f29458bSAya Levin 3515f29458bSAya Levin static int mlx5e_tx_reporter_dump_from_ctx(struct mlx5e_priv *priv, 3525f29458bSAya Levin struct mlx5e_err_ctx *err_ctx, 3535f29458bSAya Levin struct devlink_fmsg *fmsg) 3545f29458bSAya Levin { 3555f29458bSAya Levin return err_ctx->dump(priv, fmsg, err_ctx->ctx); 3565f29458bSAya Levin } 3575f29458bSAya Levin 3585f29458bSAya Levin static int mlx5e_tx_reporter_dump(struct devlink_health_reporter *reporter, 3595f29458bSAya Levin struct devlink_fmsg *fmsg, void *context, 3605f29458bSAya Levin struct netlink_ext_ack *extack) 3615f29458bSAya Levin { 3625f29458bSAya Levin struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 3635f29458bSAya Levin struct mlx5e_err_ctx *err_ctx = context; 3645f29458bSAya Levin 3655f29458bSAya Levin return err_ctx ? mlx5e_tx_reporter_dump_from_ctx(priv, err_ctx, fmsg) : 3665f29458bSAya Levin mlx5e_tx_reporter_dump_all_sqs(priv, fmsg); 3675f29458bSAya Levin } 3685f29458bSAya Levin 3690a56be3cSAya Levin void mlx5e_reporter_tx_err_cqe(struct mlx5e_txqsq *sq) 3700a56be3cSAya Levin { 3710a56be3cSAya Levin struct mlx5e_priv *priv = sq->channel->priv; 3720a56be3cSAya Levin char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 3730a56be3cSAya Levin struct mlx5e_err_ctx err_ctx = {}; 3740a56be3cSAya Levin 3750a56be3cSAya Levin err_ctx.ctx = sq; 3760a56be3cSAya Levin err_ctx.recover = mlx5e_tx_reporter_err_cqe_recover; 3775f29458bSAya Levin err_ctx.dump = mlx5e_tx_reporter_dump_sq; 378b21aef7eSJoe Perches snprintf(err_str, sizeof(err_str), "ERR CQE on SQ: 0x%x", sq->sqn); 3790a56be3cSAya Levin 3800a56be3cSAya Levin mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); 3810a56be3cSAya Levin } 3820a56be3cSAya Levin 3830a56be3cSAya Levin int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq) 3840a56be3cSAya Levin { 3850a56be3cSAya Levin struct mlx5e_priv *priv = sq->channel->priv; 3860a56be3cSAya Levin char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 3870a56be3cSAya Levin struct mlx5e_err_ctx err_ctx = {}; 3880a56be3cSAya Levin 3890a56be3cSAya Levin err_ctx.ctx = sq; 3900a56be3cSAya Levin err_ctx.recover = mlx5e_tx_reporter_timeout_recover; 3915f29458bSAya Levin err_ctx.dump = mlx5e_tx_reporter_dump_sq; 392b21aef7eSJoe Perches snprintf(err_str, sizeof(err_str), 393b21aef7eSJoe Perches "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u", 3940a56be3cSAya Levin sq->channel->ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc, 3950a56be3cSAya Levin jiffies_to_usecs(jiffies - sq->txq->trans_start)); 3960a56be3cSAya Levin 3970a56be3cSAya Levin return mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); 3980a56be3cSAya Levin } 3990a56be3cSAya Levin 400de8650a8SEran Ben Elisha static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = { 401de8650a8SEran Ben Elisha .name = "tx", 402de8650a8SEran Ben Elisha .recover = mlx5e_tx_reporter_recover, 403de8650a8SEran Ben Elisha .diagnose = mlx5e_tx_reporter_diagnose, 4045f29458bSAya Levin .dump = mlx5e_tx_reporter_dump, 405de8650a8SEran Ben Elisha }; 406de8650a8SEran Ben Elisha 407de8650a8SEran Ben Elisha #define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500 408de8650a8SEran Ben Elisha 40906293ae4SAya Levin int mlx5e_reporter_tx_create(struct mlx5e_priv *priv) 410de8650a8SEran Ben Elisha { 411baf6dfdbSAya Levin struct devlink_health_reporter *reporter; 412de8650a8SEran Ben Elisha struct mlx5_core_dev *mdev = priv->mdev; 413c50de4afSAya Levin struct devlink *devlink; 414de8650a8SEran Ben Elisha 415c50de4afSAya Levin devlink = priv_to_devlink(mdev); 416baf6dfdbSAya Levin reporter = 417de8650a8SEran Ben Elisha devlink_health_reporter_create(devlink, &mlx5_tx_reporter_ops, 418de8650a8SEran Ben Elisha MLX5_REPORTER_TX_GRACEFUL_PERIOD, 419de8650a8SEran Ben Elisha true, priv); 420baf6dfdbSAya Levin if (IS_ERR(reporter)) { 421de8650a8SEran Ben Elisha netdev_warn(priv->netdev, 422de8650a8SEran Ben Elisha "Failed to create tx reporter, err = %ld\n", 423baf6dfdbSAya Levin PTR_ERR(reporter)); 424baf6dfdbSAya Levin return PTR_ERR(reporter); 4257f7cc235SAya Levin } 426baf6dfdbSAya Levin priv->tx_reporter = reporter; 4277f7cc235SAya Levin return 0; 428de8650a8SEran Ben Elisha } 429de8650a8SEran Ben Elisha 43006293ae4SAya Levin void mlx5e_reporter_tx_destroy(struct mlx5e_priv *priv) 431de8650a8SEran Ben Elisha { 432baf6dfdbSAya Levin if (!priv->tx_reporter) 433de8650a8SEran Ben Elisha return; 434de8650a8SEran Ben Elisha 435de8650a8SEran Ben Elisha devlink_health_reporter_destroy(priv->tx_reporter); 436de8650a8SEran Ben Elisha } 437