1de8650a8SEran Ben Elisha /* SPDX-License-Identifier: GPL-2.0 */ 2de8650a8SEran Ben Elisha /* Copyright (c) 2019 Mellanox Technologies. */ 3de8650a8SEran Ben Elisha 44edc17fdSAya Levin #include "health.h" 5145e5637SEran Ben Elisha #include "en/ptp.h" 6c27971d0SRoi Dayan #include "en/devlink.h" 7de8650a8SEran Ben Elisha 8de8650a8SEran Ben Elisha static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq) 9de8650a8SEran Ben Elisha { 10e74e28aeSAya Levin unsigned long exp_time = jiffies + 11e74e28aeSAya Levin msecs_to_jiffies(MLX5E_REPORTER_FLUSH_TIMEOUT_MSEC); 12de8650a8SEran Ben Elisha 13de8650a8SEran Ben Elisha while (time_before(jiffies, exp_time)) { 14de8650a8SEran Ben Elisha if (sq->cc == sq->pc) 15de8650a8SEran Ben Elisha return 0; 16de8650a8SEran Ben Elisha 17de8650a8SEran Ben Elisha msleep(20); 18de8650a8SEran Ben Elisha } 19de8650a8SEran Ben Elisha 204ad40d8eSEran Ben Elisha netdev_err(sq->netdev, 21de8650a8SEran Ben Elisha "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n", 22de8650a8SEran Ben Elisha sq->sqn, sq->cc, sq->pc); 23de8650a8SEran Ben Elisha 24de8650a8SEran Ben Elisha return -ETIMEDOUT; 25de8650a8SEran Ben Elisha } 26de8650a8SEran Ben Elisha 27de8650a8SEran Ben Elisha static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq) 28de8650a8SEran Ben Elisha { 29de8650a8SEran Ben Elisha WARN_ONCE(sq->cc != sq->pc, 30de8650a8SEran Ben Elisha "SQ 0x%x: cc (0x%x) != pc (0x%x)\n", 31de8650a8SEran Ben Elisha sq->sqn, sq->cc, sq->pc); 32de8650a8SEran Ben Elisha sq->cc = 0; 33de8650a8SEran Ben Elisha sq->dma_fifo_cc = 0; 34de8650a8SEran Ben Elisha sq->pc = 0; 35de8650a8SEran Ben Elisha } 36de8650a8SEran Ben Elisha 37c50de4afSAya Levin static int mlx5e_tx_reporter_err_cqe_recover(void *ctx) 38de8650a8SEran Ben Elisha { 39c50de4afSAya Levin struct mlx5_core_dev *mdev; 40c50de4afSAya Levin struct net_device *dev; 41c50de4afSAya Levin struct mlx5e_txqsq *sq; 42de8650a8SEran Ben Elisha u8 state; 43de8650a8SEran Ben Elisha int err; 44de8650a8SEran Ben Elisha 45c50de4afSAya Levin sq = ctx; 464ad40d8eSEran Ben Elisha mdev = sq->mdev; 474ad40d8eSEran Ben Elisha dev = sq->netdev; 48c50de4afSAya Levin 49c50de4afSAya Levin if (!test_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state)) 50c50de4afSAya Levin return 0; 51c50de4afSAya Levin 52de8650a8SEran Ben Elisha err = mlx5_core_query_sq_state(mdev, sq->sqn, &state); 53de8650a8SEran Ben Elisha if (err) { 54de8650a8SEran Ben Elisha netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n", 55de8650a8SEran Ben Elisha sq->sqn, err); 56276d197eSAya Levin goto out; 57de8650a8SEran Ben Elisha } 58de8650a8SEran Ben Elisha 59d9a2fcf5SAya Levin if (state != MLX5_SQC_STATE_ERR) 60276d197eSAya Levin goto out; 61de8650a8SEran Ben Elisha 62de8650a8SEran Ben Elisha mlx5e_tx_disable_queue(sq->txq); 63de8650a8SEran Ben Elisha 64de8650a8SEran Ben Elisha err = mlx5e_wait_for_sq_flush(sq); 65de8650a8SEran Ben Elisha if (err) 66276d197eSAya Levin goto out; 67de8650a8SEran Ben Elisha 68de8650a8SEran Ben Elisha /* At this point, no new packets will arrive from the stack as TXQ is 69de8650a8SEran Ben Elisha * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all 70de8650a8SEran Ben Elisha * pending WQEs. SQ can safely reset the SQ. 71de8650a8SEran Ben Elisha */ 72de8650a8SEran Ben Elisha 734ad40d8eSEran Ben Elisha err = mlx5e_health_sq_to_ready(mdev, dev, sq->sqn); 74de8650a8SEran Ben Elisha if (err) 75276d197eSAya Levin goto out; 76de8650a8SEran Ben Elisha 77de8650a8SEran Ben Elisha mlx5e_reset_txqsq_cc_pc(sq); 78de8650a8SEran Ben Elisha sq->stats->recover++; 79276d197eSAya Levin clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 80de8650a8SEran Ben Elisha mlx5e_activate_txqsq(sq); 81de8650a8SEran Ben Elisha 82de8650a8SEran Ben Elisha return 0; 83276d197eSAya Levin out: 84276d197eSAya Levin clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 85276d197eSAya Levin return err; 86de8650a8SEran Ben Elisha } 87de8650a8SEran Ben Elisha 88e6205564SAya Levin struct mlx5e_tx_timeout_ctx { 89e6205564SAya Levin struct mlx5e_txqsq *sq; 90e6205564SAya Levin signed int status; 91e6205564SAya Levin }; 92e6205564SAya Levin 93c50de4afSAya Levin static int mlx5e_tx_reporter_timeout_recover(void *ctx) 947d91126bSEran Ben Elisha { 95e6205564SAya Levin struct mlx5e_tx_timeout_ctx *to_ctx; 96e6205564SAya Levin struct mlx5e_priv *priv; 97c50de4afSAya Levin struct mlx5_eq_comp *eq; 98c50de4afSAya Levin struct mlx5e_txqsq *sq; 99c50de4afSAya Levin int err; 1007d91126bSEran Ben Elisha 101e6205564SAya Levin to_ctx = ctx; 102e6205564SAya Levin sq = to_ctx->sq; 103c50de4afSAya Levin eq = sq->cq.mcq.eq; 1044ad40d8eSEran Ben Elisha priv = sq->priv; 1054ad40d8eSEran Ben Elisha err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats); 106e6205564SAya Levin if (!err) { 107e6205564SAya Levin to_ctx->status = 0; /* this sq recovered */ 108e6205564SAya Levin return err; 109e6205564SAya Levin } 110e6205564SAya Levin 111e6205564SAya Levin err = mlx5e_safe_reopen_channels(priv); 112e6205564SAya Levin if (!err) { 113e6205564SAya Levin to_ctx->status = 1; /* all channels recovered */ 114e6205564SAya Levin return err; 115e6205564SAya Levin } 116e6205564SAya Levin 117e6205564SAya Levin to_ctx->status = err; 1187d91126bSEran Ben Elisha clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 119e6205564SAya Levin netdev_err(priv->netdev, 120e6205564SAya Levin "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", 121e6205564SAya Levin err); 1227d91126bSEran Ben Elisha 123c50de4afSAya Levin return err; 1247d91126bSEran Ben Elisha } 1257d91126bSEran Ben Elisha 126de8650a8SEran Ben Elisha /* state lock cannot be grabbed within this function. 127de8650a8SEran Ben Elisha * It can cause a dead lock or a read-after-free. 128de8650a8SEran Ben Elisha */ 129c50de4afSAya Levin static int mlx5e_tx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx) 130de8650a8SEran Ben Elisha { 131c50de4afSAya Levin return err_ctx->recover(err_ctx->ctx); 132de8650a8SEran Ben Elisha } 133de8650a8SEran Ben Elisha 134de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_recover(struct devlink_health_reporter *reporter, 135e7a98105SJiri Pirko void *context, 136e7a98105SJiri Pirko struct netlink_ext_ack *extack) 137de8650a8SEran Ben Elisha { 138de8650a8SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 139c50de4afSAya Levin struct mlx5e_err_ctx *err_ctx = context; 140de8650a8SEran Ben Elisha 141de8650a8SEran Ben Elisha return err_ctx ? mlx5e_tx_reporter_recover_from_ctx(err_ctx) : 142c50de4afSAya Levin mlx5e_health_recover_channels(priv); 143de8650a8SEran Ben Elisha } 144de8650a8SEran Ben Elisha 145de8650a8SEran Ben Elisha static int 146145e5637SEran Ben Elisha mlx5e_tx_reporter_build_diagnose_output_sq_common(struct devlink_fmsg *fmsg, 1472d708887SAya Levin struct mlx5e_txqsq *sq, int tc) 148de8650a8SEran Ben Elisha { 149dd921fd2SAya Levin bool stopped = netif_xmit_stopped(sq->txq); 1504ad40d8eSEran Ben Elisha struct mlx5e_priv *priv = sq->priv; 151dd921fd2SAya Levin u8 state; 152de8650a8SEran Ben Elisha int err; 153de8650a8SEran Ben Elisha 154dd921fd2SAya Levin err = mlx5_core_query_sq_state(priv->mdev, sq->sqn, &state); 155dd921fd2SAya Levin if (err) 156dd921fd2SAya Levin return err; 157dd921fd2SAya Levin 1582d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "tc", tc); 1592d708887SAya Levin if (err) 1602d708887SAya Levin return err; 1612d708887SAya Levin 1622d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "txq ix", sq->txq_ix); 1632d708887SAya Levin if (err) 1642d708887SAya Levin return err; 1652d708887SAya Levin 166dd921fd2SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "sqn", sq->sqn); 167de8650a8SEran Ben Elisha if (err) 168de8650a8SEran Ben Elisha return err; 169de8650a8SEran Ben Elisha 170de8650a8SEran Ben Elisha err = devlink_fmsg_u8_pair_put(fmsg, "HW state", state); 171de8650a8SEran Ben Elisha if (err) 172de8650a8SEran Ben Elisha return err; 173de8650a8SEran Ben Elisha 174de8650a8SEran Ben Elisha err = devlink_fmsg_bool_pair_put(fmsg, "stopped", stopped); 175de8650a8SEran Ben Elisha if (err) 176de8650a8SEran Ben Elisha return err; 177de8650a8SEran Ben Elisha 1782d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "cc", sq->cc); 1792d708887SAya Levin if (err) 1802d708887SAya Levin return err; 1812d708887SAya Levin 1822d708887SAya Levin err = devlink_fmsg_u32_pair_put(fmsg, "pc", sq->pc); 1832d708887SAya Levin if (err) 1842d708887SAya Levin return err; 1852d708887SAya Levin 186d5cbedd7SAya Levin err = mlx5e_health_cq_diag_fmsg(&sq->cq, fmsg); 1872bf09e60SAya Levin if (err) 1882bf09e60SAya Levin return err; 1892bf09e60SAya Levin 190145e5637SEran Ben Elisha return mlx5e_health_eq_diag_fmsg(sq->cq.mcq.eq, fmsg); 191145e5637SEran Ben Elisha } 192145e5637SEran Ben Elisha 193145e5637SEran Ben Elisha static int 194145e5637SEran Ben Elisha mlx5e_tx_reporter_build_diagnose_output(struct devlink_fmsg *fmsg, 195145e5637SEran Ben Elisha struct mlx5e_txqsq *sq, int tc) 196145e5637SEran Ben Elisha { 197145e5637SEran Ben Elisha int err; 198145e5637SEran Ben Elisha 199145e5637SEran Ben Elisha err = devlink_fmsg_obj_nest_start(fmsg); 200145e5637SEran Ben Elisha if (err) 201145e5637SEran Ben Elisha return err; 202145e5637SEran Ben Elisha 203145e5637SEran Ben Elisha err = devlink_fmsg_u32_pair_put(fmsg, "channel ix", sq->ch_ix); 204145e5637SEran Ben Elisha if (err) 205145e5637SEran Ben Elisha return err; 206145e5637SEran Ben Elisha 207145e5637SEran Ben Elisha err = mlx5e_tx_reporter_build_diagnose_output_sq_common(fmsg, sq, tc); 20856837c2aSAya Levin if (err) 20956837c2aSAya Levin return err; 21056837c2aSAya Levin 211de8650a8SEran Ben Elisha err = devlink_fmsg_obj_nest_end(fmsg); 212de8650a8SEran Ben Elisha if (err) 213de8650a8SEran Ben Elisha return err; 214de8650a8SEran Ben Elisha 215de8650a8SEran Ben Elisha return 0; 216de8650a8SEran Ben Elisha } 217de8650a8SEran Ben Elisha 218145e5637SEran Ben Elisha static int 219145e5637SEran Ben Elisha mlx5e_tx_reporter_build_diagnose_output_ptpsq(struct devlink_fmsg *fmsg, 220145e5637SEran Ben Elisha struct mlx5e_ptpsq *ptpsq, int tc) 221145e5637SEran Ben Elisha { 222145e5637SEran Ben Elisha int err; 223145e5637SEran Ben Elisha 224145e5637SEran Ben Elisha err = devlink_fmsg_obj_nest_start(fmsg); 225145e5637SEran Ben Elisha if (err) 226145e5637SEran Ben Elisha return err; 227145e5637SEran Ben Elisha 228145e5637SEran Ben Elisha err = devlink_fmsg_string_pair_put(fmsg, "channel", "ptp"); 229145e5637SEran Ben Elisha if (err) 230145e5637SEran Ben Elisha return err; 231145e5637SEran Ben Elisha 2321880bc4eSEran Ben Elisha err = mlx5e_tx_reporter_build_diagnose_output_sq_common(fmsg, &ptpsq->txqsq, tc); 2331880bc4eSEran Ben Elisha if (err) 2341880bc4eSEran Ben Elisha return err; 2351880bc4eSEran Ben Elisha 2361880bc4eSEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Port TS"); 2371880bc4eSEran Ben Elisha if (err) 2381880bc4eSEran Ben Elisha return err; 2391880bc4eSEran Ben Elisha 2401880bc4eSEran Ben Elisha err = mlx5e_health_cq_diag_fmsg(&ptpsq->ts_cq, fmsg); 2411880bc4eSEran Ben Elisha if (err) 2421880bc4eSEran Ben Elisha return err; 2431880bc4eSEran Ben Elisha 2441880bc4eSEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 245145e5637SEran Ben Elisha if (err) 246145e5637SEran Ben Elisha return err; 247145e5637SEran Ben Elisha 248145e5637SEran Ben Elisha err = devlink_fmsg_obj_nest_end(fmsg); 249145e5637SEran Ben Elisha if (err) 250145e5637SEran Ben Elisha return err; 251145e5637SEran Ben Elisha 252145e5637SEran Ben Elisha return 0; 253145e5637SEran Ben Elisha } 254145e5637SEran Ben Elisha 255145e5637SEran Ben Elisha static int 256145e5637SEran Ben Elisha mlx5e_tx_reporter_diagnose_generic_txqsq(struct devlink_fmsg *fmsg, 257145e5637SEran Ben Elisha struct mlx5e_txqsq *txqsq) 258145e5637SEran Ben Elisha { 259145e5637SEran Ben Elisha u32 sq_stride, sq_sz; 260145e5637SEran Ben Elisha int err; 261145e5637SEran Ben Elisha 262145e5637SEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SQ"); 263145e5637SEran Ben Elisha if (err) 264145e5637SEran Ben Elisha return err; 265145e5637SEran Ben Elisha 266145e5637SEran Ben Elisha sq_sz = mlx5_wq_cyc_get_size(&txqsq->wq); 267145e5637SEran Ben Elisha sq_stride = MLX5_SEND_WQE_BB; 268145e5637SEran Ben Elisha 269145e5637SEran Ben Elisha err = devlink_fmsg_u64_pair_put(fmsg, "stride size", sq_stride); 270145e5637SEran Ben Elisha if (err) 271145e5637SEran Ben Elisha return err; 272145e5637SEran Ben Elisha 273145e5637SEran Ben Elisha err = devlink_fmsg_u32_pair_put(fmsg, "size", sq_sz); 274145e5637SEran Ben Elisha if (err) 275145e5637SEran Ben Elisha return err; 276145e5637SEran Ben Elisha 277145e5637SEran Ben Elisha err = mlx5e_health_cq_common_diag_fmsg(&txqsq->cq, fmsg); 278145e5637SEran Ben Elisha if (err) 279145e5637SEran Ben Elisha return err; 280145e5637SEran Ben Elisha 281145e5637SEran Ben Elisha return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 282145e5637SEran Ben Elisha } 283145e5637SEran Ben Elisha 284145e5637SEran Ben Elisha static int 2851880bc4eSEran Ben Elisha mlx5e_tx_reporter_diagnose_generic_tx_port_ts(struct devlink_fmsg *fmsg, 2861880bc4eSEran Ben Elisha struct mlx5e_ptpsq *ptpsq) 2871880bc4eSEran Ben Elisha { 2881880bc4eSEran Ben Elisha int err; 2891880bc4eSEran Ben Elisha 2901880bc4eSEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Port TS"); 2911880bc4eSEran Ben Elisha if (err) 2921880bc4eSEran Ben Elisha return err; 2931880bc4eSEran Ben Elisha 2941880bc4eSEran Ben Elisha err = mlx5e_health_cq_common_diag_fmsg(&ptpsq->ts_cq, fmsg); 2951880bc4eSEran Ben Elisha if (err) 2961880bc4eSEran Ben Elisha return err; 2971880bc4eSEran Ben Elisha 2981880bc4eSEran Ben Elisha return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 2991880bc4eSEran Ben Elisha } 3001880bc4eSEran Ben Elisha 3011880bc4eSEran Ben Elisha static int 302145e5637SEran Ben Elisha mlx5e_tx_reporter_diagnose_common_config(struct devlink_health_reporter *reporter, 303145e5637SEran Ben Elisha struct devlink_fmsg *fmsg) 304145e5637SEran Ben Elisha { 305145e5637SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 306145e5637SEran Ben Elisha struct mlx5e_txqsq *generic_sq = priv->txq2sq[0]; 307145e5637SEran Ben Elisha struct mlx5e_ptpsq *generic_ptpsq; 308145e5637SEran Ben Elisha int err; 309145e5637SEran Ben Elisha 310145e5637SEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common Config"); 311145e5637SEran Ben Elisha if (err) 312145e5637SEran Ben Elisha return err; 313145e5637SEran Ben Elisha 314145e5637SEran Ben Elisha err = mlx5e_tx_reporter_diagnose_generic_txqsq(fmsg, generic_sq); 315145e5637SEran Ben Elisha if (err) 316145e5637SEran Ben Elisha return err; 317145e5637SEran Ben Elisha 318145e5637SEran Ben Elisha generic_ptpsq = priv->channels.port_ptp ? 319145e5637SEran Ben Elisha &priv->channels.port_ptp->ptpsq[0] : 320145e5637SEran Ben Elisha NULL; 321145e5637SEran Ben Elisha if (!generic_ptpsq) 322145e5637SEran Ben Elisha goto out; 323145e5637SEran Ben Elisha 324145e5637SEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "PTP"); 325145e5637SEran Ben Elisha if (err) 326145e5637SEran Ben Elisha return err; 327145e5637SEran Ben Elisha 328145e5637SEran Ben Elisha err = mlx5e_tx_reporter_diagnose_generic_txqsq(fmsg, &generic_ptpsq->txqsq); 329145e5637SEran Ben Elisha if (err) 330145e5637SEran Ben Elisha return err; 331145e5637SEran Ben Elisha 3321880bc4eSEran Ben Elisha err = mlx5e_tx_reporter_diagnose_generic_tx_port_ts(fmsg, generic_ptpsq); 3331880bc4eSEran Ben Elisha if (err) 3341880bc4eSEran Ben Elisha return err; 3351880bc4eSEran Ben Elisha 336145e5637SEran Ben Elisha err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 337145e5637SEran Ben Elisha if (err) 338145e5637SEran Ben Elisha return err; 339145e5637SEran Ben Elisha 340145e5637SEran Ben Elisha out: 341145e5637SEran Ben Elisha return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 342145e5637SEran Ben Elisha } 343145e5637SEran Ben Elisha 344de8650a8SEran Ben Elisha static int mlx5e_tx_reporter_diagnose(struct devlink_health_reporter *reporter, 345e7a98105SJiri Pirko struct devlink_fmsg *fmsg, 346e7a98105SJiri Pirko struct netlink_ext_ack *extack) 347de8650a8SEran Ben Elisha { 348de8650a8SEran Ben Elisha struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 349145e5637SEran Ben Elisha struct mlx5e_port_ptp *ptp_ch = priv->channels.port_ptp; 3502d708887SAya Levin 3512d708887SAya Levin int i, tc, err = 0; 352de8650a8SEran Ben Elisha 353de8650a8SEran Ben Elisha mutex_lock(&priv->state_lock); 354de8650a8SEran Ben Elisha 355de8650a8SEran Ben Elisha if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 356de8650a8SEran Ben Elisha goto unlock; 357de8650a8SEran Ben Elisha 358145e5637SEran Ben Elisha err = mlx5e_tx_reporter_diagnose_common_config(reporter, fmsg); 3592d708887SAya Levin if (err) 3602d708887SAya Levin goto unlock; 3612d708887SAya Levin 362de8650a8SEran Ben Elisha err = devlink_fmsg_arr_pair_nest_start(fmsg, "SQs"); 363de8650a8SEran Ben Elisha if (err) 364de8650a8SEran Ben Elisha goto unlock; 365de8650a8SEran Ben Elisha 3662d708887SAya Levin for (i = 0; i < priv->channels.num; i++) { 3672d708887SAya Levin struct mlx5e_channel *c = priv->channels.c[i]; 368de8650a8SEran Ben Elisha 3692d708887SAya Levin for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 3702d708887SAya Levin struct mlx5e_txqsq *sq = &c->sq[tc]; 3712d708887SAya Levin 3722d708887SAya Levin err = mlx5e_tx_reporter_build_diagnose_output(fmsg, sq, tc); 373de8650a8SEran Ben Elisha if (err) 37499d31cbdSAya Levin goto unlock; 375de8650a8SEran Ben Elisha } 3762d708887SAya Levin } 377145e5637SEran Ben Elisha 378145e5637SEran Ben Elisha if (!ptp_ch) 379145e5637SEran Ben Elisha goto close_sqs_nest; 380145e5637SEran Ben Elisha 381145e5637SEran Ben Elisha for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 382145e5637SEran Ben Elisha err = mlx5e_tx_reporter_build_diagnose_output_ptpsq(fmsg, 383145e5637SEran Ben Elisha &ptp_ch->ptpsq[tc], 384145e5637SEran Ben Elisha tc); 385145e5637SEran Ben Elisha if (err) 386145e5637SEran Ben Elisha goto unlock; 387145e5637SEran Ben Elisha } 388145e5637SEran Ben Elisha 389145e5637SEran Ben Elisha close_sqs_nest: 390de8650a8SEran Ben Elisha err = devlink_fmsg_arr_pair_nest_end(fmsg); 391de8650a8SEran Ben Elisha if (err) 392de8650a8SEran Ben Elisha goto unlock; 393de8650a8SEran Ben Elisha 394de8650a8SEran Ben Elisha unlock: 395de8650a8SEran Ben Elisha mutex_unlock(&priv->state_lock); 396de8650a8SEran Ben Elisha return err; 397de8650a8SEran Ben Elisha } 398de8650a8SEran Ben Elisha 3995f29458bSAya Levin static int mlx5e_tx_reporter_dump_sq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 4005f29458bSAya Levin void *ctx) 4015f29458bSAya Levin { 4025f29458bSAya Levin struct mlx5_rsc_key key = {}; 4035f29458bSAya Levin struct mlx5e_txqsq *sq = ctx; 4045f29458bSAya Levin int err; 4055f29458bSAya Levin 4065f29458bSAya Levin if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4075f29458bSAya Levin return 0; 4085f29458bSAya Levin 409d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); 4105f29458bSAya Levin if (err) 4115f29458bSAya Levin return err; 4125f29458bSAya Levin 4135f29458bSAya Levin key.size = PAGE_SIZE; 4145f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 4155f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 4165f29458bSAya Levin if (err) 4175f29458bSAya Levin return err; 4185f29458bSAya Levin 419d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 4205f29458bSAya Levin if (err) 4215f29458bSAya Levin return err; 4225f29458bSAya Levin 423d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SQ"); 4245f29458bSAya Levin if (err) 4255f29458bSAya Levin return err; 4265f29458bSAya Levin 427d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 4285f29458bSAya Levin if (err) 4295f29458bSAya Levin return err; 4305f29458bSAya Levin 4315f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 4325f29458bSAya Levin key.index1 = sq->sqn; 4335f29458bSAya Levin key.num_of_obj1 = 1; 4345f29458bSAya Levin 4355f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 4365f29458bSAya Levin if (err) 4375f29458bSAya Levin return err; 4385f29458bSAya Levin 439d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 4405f29458bSAya Levin if (err) 4415f29458bSAya Levin return err; 4425f29458bSAya Levin 443d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff"); 4445f29458bSAya Levin if (err) 4455f29458bSAya Levin return err; 4465f29458bSAya Levin 4475f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_SND_BUFF; 4485f29458bSAya Levin key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 4495f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 4505f29458bSAya Levin if (err) 4515f29458bSAya Levin return err; 4525f29458bSAya Levin 453d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 4545f29458bSAya Levin if (err) 4555f29458bSAya Levin return err; 4565f29458bSAya Levin 457d5cbedd7SAya Levin return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 4585f29458bSAya Levin } 4595f29458bSAya Levin 4605f29458bSAya Levin static int mlx5e_tx_reporter_dump_all_sqs(struct mlx5e_priv *priv, 4615f29458bSAya Levin struct devlink_fmsg *fmsg) 4625f29458bSAya Levin { 463145e5637SEran Ben Elisha struct mlx5e_port_ptp *ptp_ch = priv->channels.port_ptp; 4645f29458bSAya Levin struct mlx5_rsc_key key = {}; 4655f29458bSAya Levin int i, tc, err; 4665f29458bSAya Levin 4675f29458bSAya Levin if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4685f29458bSAya Levin return 0; 4695f29458bSAya Levin 470d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); 4715f29458bSAya Levin if (err) 4725f29458bSAya Levin return err; 4735f29458bSAya Levin 4745f29458bSAya Levin key.size = PAGE_SIZE; 4755f29458bSAya Levin key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 4765f29458bSAya Levin err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 4775f29458bSAya Levin if (err) 4785f29458bSAya Levin return err; 4795f29458bSAya Levin 480d5cbedd7SAya Levin err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 4815f29458bSAya Levin if (err) 4825f29458bSAya Levin return err; 4835f29458bSAya Levin 4845f29458bSAya Levin err = devlink_fmsg_arr_pair_nest_start(fmsg, "SQs"); 4855f29458bSAya Levin if (err) 4865f29458bSAya Levin return err; 4875f29458bSAya Levin 4885f29458bSAya Levin for (i = 0; i < priv->channels.num; i++) { 4895f29458bSAya Levin struct mlx5e_channel *c = priv->channels.c[i]; 4905f29458bSAya Levin 4915f29458bSAya Levin for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 4925f29458bSAya Levin struct mlx5e_txqsq *sq = &c->sq[tc]; 4935f29458bSAya Levin 4945f29458bSAya Levin err = mlx5e_health_queue_dump(priv, fmsg, sq->sqn, "SQ"); 4955f29458bSAya Levin if (err) 4965f29458bSAya Levin return err; 4975f29458bSAya Levin } 4985f29458bSAya Levin } 499145e5637SEran Ben Elisha 500145e5637SEran Ben Elisha if (ptp_ch) { 501145e5637SEran Ben Elisha for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 502145e5637SEran Ben Elisha struct mlx5e_txqsq *sq = &ptp_ch->ptpsq[tc].txqsq; 503145e5637SEran Ben Elisha 504145e5637SEran Ben Elisha err = mlx5e_health_queue_dump(priv, fmsg, sq->sqn, "PTP SQ"); 505145e5637SEran Ben Elisha if (err) 506145e5637SEran Ben Elisha return err; 507145e5637SEran Ben Elisha } 508145e5637SEran Ben Elisha } 509145e5637SEran Ben Elisha 5105f29458bSAya Levin return devlink_fmsg_arr_pair_nest_end(fmsg); 5115f29458bSAya Levin } 5125f29458bSAya Levin 5135f29458bSAya Levin static int mlx5e_tx_reporter_dump_from_ctx(struct mlx5e_priv *priv, 5145f29458bSAya Levin struct mlx5e_err_ctx *err_ctx, 5155f29458bSAya Levin struct devlink_fmsg *fmsg) 5165f29458bSAya Levin { 5175f29458bSAya Levin return err_ctx->dump(priv, fmsg, err_ctx->ctx); 5185f29458bSAya Levin } 5195f29458bSAya Levin 5205f29458bSAya Levin static int mlx5e_tx_reporter_dump(struct devlink_health_reporter *reporter, 5215f29458bSAya Levin struct devlink_fmsg *fmsg, void *context, 5225f29458bSAya Levin struct netlink_ext_ack *extack) 5235f29458bSAya Levin { 5245f29458bSAya Levin struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 5255f29458bSAya Levin struct mlx5e_err_ctx *err_ctx = context; 5265f29458bSAya Levin 5275f29458bSAya Levin return err_ctx ? mlx5e_tx_reporter_dump_from_ctx(priv, err_ctx, fmsg) : 5285f29458bSAya Levin mlx5e_tx_reporter_dump_all_sqs(priv, fmsg); 5295f29458bSAya Levin } 5305f29458bSAya Levin 5310a56be3cSAya Levin void mlx5e_reporter_tx_err_cqe(struct mlx5e_txqsq *sq) 5320a56be3cSAya Levin { 5330a56be3cSAya Levin char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 5344ad40d8eSEran Ben Elisha struct mlx5e_priv *priv = sq->priv; 5350a56be3cSAya Levin struct mlx5e_err_ctx err_ctx = {}; 5360a56be3cSAya Levin 5370a56be3cSAya Levin err_ctx.ctx = sq; 5380a56be3cSAya Levin err_ctx.recover = mlx5e_tx_reporter_err_cqe_recover; 5395f29458bSAya Levin err_ctx.dump = mlx5e_tx_reporter_dump_sq; 540b21aef7eSJoe Perches snprintf(err_str, sizeof(err_str), "ERR CQE on SQ: 0x%x", sq->sqn); 5410a56be3cSAya Levin 5420a56be3cSAya Levin mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); 5430a56be3cSAya Levin } 5440a56be3cSAya Levin 5450a56be3cSAya Levin int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq) 5460a56be3cSAya Levin { 5470a56be3cSAya Levin char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 548e6205564SAya Levin struct mlx5e_tx_timeout_ctx to_ctx = {}; 5494ad40d8eSEran Ben Elisha struct mlx5e_priv *priv = sq->priv; 5500a56be3cSAya Levin struct mlx5e_err_ctx err_ctx = {}; 5510a56be3cSAya Levin 552e6205564SAya Levin to_ctx.sq = sq; 553e6205564SAya Levin err_ctx.ctx = &to_ctx; 5540a56be3cSAya Levin err_ctx.recover = mlx5e_tx_reporter_timeout_recover; 5555f29458bSAya Levin err_ctx.dump = mlx5e_tx_reporter_dump_sq; 556b21aef7eSJoe Perches snprintf(err_str, sizeof(err_str), 557b21aef7eSJoe Perches "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u", 5584ad40d8eSEran Ben Elisha sq->ch_ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc, 5590a56be3cSAya Levin jiffies_to_usecs(jiffies - sq->txq->trans_start)); 5600a56be3cSAya Levin 561e6205564SAya Levin mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); 562e6205564SAya Levin return to_ctx.status; 5630a56be3cSAya Levin } 5640a56be3cSAya Levin 565de8650a8SEran Ben Elisha static const struct devlink_health_reporter_ops mlx5_tx_reporter_ops = { 566de8650a8SEran Ben Elisha .name = "tx", 567de8650a8SEran Ben Elisha .recover = mlx5e_tx_reporter_recover, 568de8650a8SEran Ben Elisha .diagnose = mlx5e_tx_reporter_diagnose, 5695f29458bSAya Levin .dump = mlx5e_tx_reporter_dump, 570de8650a8SEran Ben Elisha }; 571de8650a8SEran Ben Elisha 572de8650a8SEran Ben Elisha #define MLX5_REPORTER_TX_GRACEFUL_PERIOD 500 573de8650a8SEran Ben Elisha 574b3ea4c4fSEran Ben Elisha void mlx5e_reporter_tx_create(struct mlx5e_priv *priv) 575de8650a8SEran Ben Elisha { 576c27971d0SRoi Dayan struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); 577baf6dfdbSAya Levin struct devlink_health_reporter *reporter; 578de8650a8SEran Ben Elisha 579c27971d0SRoi Dayan reporter = devlink_port_health_reporter_create(dl_port, &mlx5_tx_reporter_ops, 580b7e93bb6SVladyslav Tarasiuk MLX5_REPORTER_TX_GRACEFUL_PERIOD, priv); 581baf6dfdbSAya Levin if (IS_ERR(reporter)) { 582de8650a8SEran Ben Elisha netdev_warn(priv->netdev, 583de8650a8SEran Ben Elisha "Failed to create tx reporter, err = %ld\n", 584baf6dfdbSAya Levin PTR_ERR(reporter)); 585b3ea4c4fSEran Ben Elisha return; 5867f7cc235SAya Levin } 587baf6dfdbSAya Levin priv->tx_reporter = reporter; 588de8650a8SEran Ben Elisha } 589de8650a8SEran Ben Elisha 59006293ae4SAya Levin void mlx5e_reporter_tx_destroy(struct mlx5e_priv *priv) 591de8650a8SEran Ben Elisha { 592baf6dfdbSAya Levin if (!priv->tx_reporter) 593de8650a8SEran Ben Elisha return; 594de8650a8SEran Ben Elisha 595b7e93bb6SVladyslav Tarasiuk devlink_port_health_reporter_destroy(priv->tx_reporter); 596*7a9fb35eSRoi Dayan priv->tx_reporter = NULL; 597de8650a8SEran Ben Elisha } 598