1c50de4afSAya Levin // SPDX-License-Identifier: GPL-2.0 2c50de4afSAya Levin // Copyright (c) 2019 Mellanox Technologies. 3c50de4afSAya Levin 4c50de4afSAya Levin #include "health.h" 5c50de4afSAya Levin #include "lib/eq.h" 6c50de4afSAya Levin 72d708887SAya Levin int mlx5e_reporter_named_obj_nest_start(struct devlink_fmsg *fmsg, char *name) 82d708887SAya Levin { 92d708887SAya Levin int err; 102d708887SAya Levin 112d708887SAya Levin err = devlink_fmsg_pair_nest_start(fmsg, name); 122d708887SAya Levin if (err) 132d708887SAya Levin return err; 142d708887SAya Levin 152d708887SAya Levin err = devlink_fmsg_obj_nest_start(fmsg); 162d708887SAya Levin if (err) 172d708887SAya Levin return err; 182d708887SAya Levin 192d708887SAya Levin return 0; 202d708887SAya Levin } 212d708887SAya Levin 222d708887SAya Levin int mlx5e_reporter_named_obj_nest_end(struct devlink_fmsg *fmsg) 232d708887SAya Levin { 242d708887SAya Levin int err; 252d708887SAya Levin 262d708887SAya Levin err = devlink_fmsg_obj_nest_end(fmsg); 272d708887SAya Levin if (err) 282d708887SAya Levin return err; 292d708887SAya Levin 302d708887SAya Levin err = devlink_fmsg_pair_nest_end(fmsg); 312d708887SAya Levin if (err) 322d708887SAya Levin return err; 332d708887SAya Levin 342d708887SAya Levin return 0; 352d708887SAya Levin } 362d708887SAya Levin 37c50de4afSAya Levin int mlx5e_health_sq_to_ready(struct mlx5e_channel *channel, u32 sqn) 38c50de4afSAya Levin { 39c50de4afSAya Levin struct mlx5_core_dev *mdev = channel->mdev; 40c50de4afSAya Levin struct net_device *dev = channel->netdev; 41c50de4afSAya Levin struct mlx5e_modify_sq_param msp = {}; 42c50de4afSAya Levin int err; 43c50de4afSAya Levin 44c50de4afSAya Levin msp.curr_state = MLX5_SQC_STATE_ERR; 45c50de4afSAya Levin msp.next_state = MLX5_SQC_STATE_RST; 46c50de4afSAya Levin 47c50de4afSAya Levin err = mlx5e_modify_sq(mdev, sqn, &msp); 48c50de4afSAya Levin if (err) { 49c50de4afSAya Levin netdev_err(dev, "Failed to move sq 0x%x to reset\n", sqn); 50c50de4afSAya Levin return err; 51c50de4afSAya Levin } 52c50de4afSAya Levin 53c50de4afSAya Levin memset(&msp, 0, sizeof(msp)); 54c50de4afSAya Levin msp.curr_state = MLX5_SQC_STATE_RST; 55c50de4afSAya Levin msp.next_state = MLX5_SQC_STATE_RDY; 56c50de4afSAya Levin 57c50de4afSAya Levin err = mlx5e_modify_sq(mdev, sqn, &msp); 58c50de4afSAya Levin if (err) { 59c50de4afSAya Levin netdev_err(dev, "Failed to move sq 0x%x to ready\n", sqn); 60c50de4afSAya Levin return err; 61c50de4afSAya Levin } 62c50de4afSAya Levin 63c50de4afSAya Levin return 0; 64c50de4afSAya Levin } 65c50de4afSAya Levin 66c50de4afSAya Levin int mlx5e_health_recover_channels(struct mlx5e_priv *priv) 67c50de4afSAya Levin { 68c50de4afSAya Levin int err = 0; 69c50de4afSAya Levin 70c50de4afSAya Levin rtnl_lock(); 71c50de4afSAya Levin mutex_lock(&priv->state_lock); 72c50de4afSAya Levin 73c50de4afSAya Levin if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 74c50de4afSAya Levin goto out; 75c50de4afSAya Levin 76c50de4afSAya Levin err = mlx5e_safe_reopen_channels(priv); 77c50de4afSAya Levin 78c50de4afSAya Levin out: 79c50de4afSAya Levin mutex_unlock(&priv->state_lock); 80c50de4afSAya Levin rtnl_unlock(); 81c50de4afSAya Levin 82c50de4afSAya Levin return err; 83c50de4afSAya Levin } 84c50de4afSAya Levin 85c50de4afSAya Levin int mlx5e_health_channel_eq_recover(struct mlx5_eq_comp *eq, struct mlx5e_channel *channel) 86c50de4afSAya Levin { 87c50de4afSAya Levin u32 eqe_count; 88c50de4afSAya Levin 89c50de4afSAya Levin netdev_err(channel->netdev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n", 90c50de4afSAya Levin eq->core.eqn, eq->core.cons_index, eq->core.irqn); 91c50de4afSAya Levin 92c50de4afSAya Levin eqe_count = mlx5_eq_poll_irq_disabled(eq); 93c50de4afSAya Levin if (!eqe_count) 94c50de4afSAya Levin return -EIO; 95c50de4afSAya Levin 96c50de4afSAya Levin netdev_err(channel->netdev, "Recovered %d eqes on EQ 0x%x\n", 97c50de4afSAya Levin eqe_count, eq->core.eqn); 98c50de4afSAya Levin 99c50de4afSAya Levin channel->stats->eq_rearm++; 100c50de4afSAya Levin return 0; 101c50de4afSAya Levin } 102c50de4afSAya Levin 103c50de4afSAya Levin int mlx5e_health_report(struct mlx5e_priv *priv, 104c50de4afSAya Levin struct devlink_health_reporter *reporter, char *err_str, 105c50de4afSAya Levin struct mlx5e_err_ctx *err_ctx) 106c50de4afSAya Levin { 107c50de4afSAya Levin if (!reporter) { 108c50de4afSAya Levin netdev_err(priv->netdev, err_str); 109c50de4afSAya Levin return err_ctx->recover(&err_ctx->ctx); 110c50de4afSAya Levin } 111c50de4afSAya Levin return devlink_health_report(reporter, err_str, err_ctx); 112c50de4afSAya Levin } 113