1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include "health.h" 5 #include "params.h" 6 #include "txrx.h" 7 #include "devlink.h" 8 #include "ptp.h" 9 10 static int mlx5e_query_rq_state(struct mlx5_core_dev *dev, u32 rqn, u8 *state) 11 { 12 int outlen = MLX5_ST_SZ_BYTES(query_rq_out); 13 void *out; 14 void *rqc; 15 int err; 16 17 out = kvzalloc(outlen, GFP_KERNEL); 18 if (!out) 19 return -ENOMEM; 20 21 err = mlx5_core_query_rq(dev, rqn, out); 22 if (err) 23 goto out; 24 25 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context); 26 *state = MLX5_GET(rqc, rqc, state); 27 28 out: 29 kvfree(out); 30 return err; 31 } 32 33 static int mlx5e_wait_for_icosq_flush(struct mlx5e_icosq *icosq) 34 { 35 unsigned long exp_time = jiffies + 36 msecs_to_jiffies(MLX5E_REPORTER_FLUSH_TIMEOUT_MSEC); 37 38 while (time_before(jiffies, exp_time)) { 39 if (icosq->cc == icosq->pc) 40 return 0; 41 42 msleep(20); 43 } 44 45 netdev_err(icosq->channel->netdev, 46 "Wait for ICOSQ 0x%x flush timeout (cc = 0x%x, pc = 0x%x)\n", 47 icosq->sqn, icosq->cc, icosq->pc); 48 49 return -ETIMEDOUT; 50 } 51 52 static void mlx5e_reset_icosq_cc_pc(struct mlx5e_icosq *icosq) 53 { 54 WARN_ONCE(icosq->cc != icosq->pc, "ICOSQ 0x%x: cc (0x%x) != pc (0x%x)\n", 55 icosq->sqn, icosq->cc, icosq->pc); 56 icosq->cc = 0; 57 icosq->pc = 0; 58 } 59 60 static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx) 61 { 62 struct mlx5_core_dev *mdev; 63 struct mlx5e_icosq *icosq; 64 struct net_device *dev; 65 struct mlx5e_rq *rq; 66 u8 state; 67 int err; 68 69 icosq = ctx; 70 rq = &icosq->channel->rq; 71 mdev = icosq->channel->mdev; 72 dev = icosq->channel->netdev; 73 err = mlx5_core_query_sq_state(mdev, icosq->sqn, &state); 74 if (err) { 75 netdev_err(dev, "Failed to query ICOSQ 0x%x state. err = %d\n", 76 icosq->sqn, err); 77 goto out; 78 } 79 80 if (state != MLX5_SQC_STATE_ERR) 81 goto out; 82 83 mlx5e_deactivate_rq(rq); 84 err = mlx5e_wait_for_icosq_flush(icosq); 85 if (err) 86 goto out; 87 88 mlx5e_deactivate_icosq(icosq); 89 90 /* At this point, both the rq and the icosq are disabled */ 91 92 err = mlx5e_health_sq_to_ready(mdev, dev, icosq->sqn); 93 if (err) 94 goto out; 95 96 mlx5e_reset_icosq_cc_pc(icosq); 97 mlx5e_free_rx_in_progress_descs(rq); 98 clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state); 99 mlx5e_activate_icosq(icosq); 100 mlx5e_activate_rq(rq); 101 102 rq->stats->recover++; 103 return 0; 104 out: 105 clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state); 106 return err; 107 } 108 109 static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state) 110 { 111 struct net_device *dev = rq->netdev; 112 int err; 113 114 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 115 if (err) { 116 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 117 return err; 118 } 119 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 120 if (err) { 121 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 122 return err; 123 } 124 125 return 0; 126 } 127 128 static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) 129 { 130 struct mlx5e_rq *rq = ctx; 131 int err; 132 133 mlx5e_deactivate_rq(rq); 134 mlx5e_free_rx_descs(rq); 135 136 err = mlx5e_rq_to_ready(rq, MLX5_RQC_STATE_ERR); 137 if (err) 138 goto out; 139 140 clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); 141 mlx5e_activate_rq(rq); 142 rq->stats->recover++; 143 return 0; 144 out: 145 clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); 146 return err; 147 } 148 149 static int mlx5e_rx_reporter_timeout_recover(void *ctx) 150 { 151 struct mlx5_eq_comp *eq; 152 struct mlx5e_rq *rq; 153 int err; 154 155 rq = ctx; 156 eq = rq->cq.mcq.eq; 157 158 err = mlx5e_health_channel_eq_recover(rq->netdev, eq, rq->cq.ch_stats); 159 if (err && rq->icosq) 160 clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); 161 162 return err; 163 } 164 165 static int mlx5e_rx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx) 166 { 167 return err_ctx->recover(err_ctx->ctx); 168 } 169 170 static int mlx5e_rx_reporter_recover(struct devlink_health_reporter *reporter, 171 void *context, 172 struct netlink_ext_ack *extack) 173 { 174 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 175 struct mlx5e_err_ctx *err_ctx = context; 176 177 return err_ctx ? mlx5e_rx_reporter_recover_from_ctx(err_ctx) : 178 mlx5e_health_recover_channels(priv); 179 } 180 181 static int mlx5e_reporter_icosq_diagnose(struct mlx5e_icosq *icosq, u8 hw_state, 182 struct devlink_fmsg *fmsg) 183 { 184 int err; 185 186 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); 187 if (err) 188 return err; 189 190 err = devlink_fmsg_u32_pair_put(fmsg, "sqn", icosq->sqn); 191 if (err) 192 return err; 193 194 err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); 195 if (err) 196 return err; 197 198 err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cc); 199 if (err) 200 return err; 201 202 err = devlink_fmsg_u32_pair_put(fmsg, "pc", icosq->pc); 203 if (err) 204 return err; 205 206 err = devlink_fmsg_u32_pair_put(fmsg, "WQE size", 207 mlx5_wq_cyc_get_size(&icosq->wq)); 208 if (err) 209 return err; 210 211 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "CQ"); 212 if (err) 213 return err; 214 215 err = devlink_fmsg_u32_pair_put(fmsg, "cqn", icosq->cq.mcq.cqn); 216 if (err) 217 return err; 218 219 err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cq.wq.cc); 220 if (err) 221 return err; 222 223 err = devlink_fmsg_u32_pair_put(fmsg, "size", mlx5_cqwq_get_size(&icosq->cq.wq)); 224 if (err) 225 return err; 226 227 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 228 if (err) 229 return err; 230 231 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 232 } 233 234 static int 235 mlx5e_rx_reporter_build_diagnose_output_rq_common(struct mlx5e_rq *rq, 236 struct devlink_fmsg *fmsg) 237 { 238 u16 wqe_counter; 239 int wqes_sz; 240 u8 hw_state; 241 u16 wq_head; 242 int err; 243 244 err = mlx5e_query_rq_state(rq->mdev, rq->rqn, &hw_state); 245 if (err) 246 return err; 247 248 wqes_sz = mlx5e_rqwq_get_cur_sz(rq); 249 wq_head = mlx5e_rqwq_get_head(rq); 250 wqe_counter = mlx5e_rqwq_get_wqe_counter(rq); 251 252 err = devlink_fmsg_u32_pair_put(fmsg, "rqn", rq->rqn); 253 if (err) 254 return err; 255 256 err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); 257 if (err) 258 return err; 259 260 err = devlink_fmsg_u8_pair_put(fmsg, "SW state", rq->state); 261 if (err) 262 return err; 263 264 err = devlink_fmsg_u32_pair_put(fmsg, "WQE counter", wqe_counter); 265 if (err) 266 return err; 267 268 err = devlink_fmsg_u32_pair_put(fmsg, "posted WQEs", wqes_sz); 269 if (err) 270 return err; 271 272 err = devlink_fmsg_u32_pair_put(fmsg, "cc", wq_head); 273 if (err) 274 return err; 275 276 err = mlx5e_health_cq_diag_fmsg(&rq->cq, fmsg); 277 if (err) 278 return err; 279 280 err = mlx5e_health_eq_diag_fmsg(rq->cq.mcq.eq, fmsg); 281 if (err) 282 return err; 283 284 if (rq->icosq) { 285 struct mlx5e_icosq *icosq = rq->icosq; 286 u8 icosq_hw_state; 287 288 err = mlx5_core_query_sq_state(rq->mdev, icosq->sqn, &icosq_hw_state); 289 if (err) 290 return err; 291 292 err = mlx5e_reporter_icosq_diagnose(icosq, icosq_hw_state, fmsg); 293 if (err) 294 return err; 295 } 296 297 return 0; 298 } 299 300 static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, 301 struct devlink_fmsg *fmsg) 302 { 303 int err; 304 305 err = devlink_fmsg_obj_nest_start(fmsg); 306 if (err) 307 return err; 308 309 err = devlink_fmsg_u32_pair_put(fmsg, "channel ix", rq->ix); 310 if (err) 311 return err; 312 313 err = mlx5e_rx_reporter_build_diagnose_output_rq_common(rq, fmsg); 314 if (err) 315 return err; 316 317 return devlink_fmsg_obj_nest_end(fmsg); 318 } 319 320 static int mlx5e_rx_reporter_diagnose_generic_rq(struct mlx5e_rq *rq, 321 struct devlink_fmsg *fmsg) 322 { 323 struct mlx5e_priv *priv = rq->priv; 324 struct mlx5e_params *params; 325 u32 rq_stride, rq_sz; 326 bool real_time; 327 int err; 328 329 params = &priv->channels.params; 330 rq_sz = mlx5e_rqwq_get_size(rq); 331 real_time = mlx5_is_real_time_rq(priv->mdev); 332 rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL)); 333 334 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); 335 if (err) 336 return err; 337 338 err = devlink_fmsg_u8_pair_put(fmsg, "type", params->rq_wq_type); 339 if (err) 340 return err; 341 342 err = devlink_fmsg_u64_pair_put(fmsg, "stride size", rq_stride); 343 if (err) 344 return err; 345 346 err = devlink_fmsg_u32_pair_put(fmsg, "size", rq_sz); 347 if (err) 348 return err; 349 350 err = devlink_fmsg_string_pair_put(fmsg, "ts_format", real_time ? "RT" : "FRC"); 351 if (err) 352 return err; 353 354 err = mlx5e_health_cq_common_diag_fmsg(&rq->cq, fmsg); 355 if (err) 356 return err; 357 358 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 359 } 360 361 static int 362 mlx5e_rx_reporter_diagnose_common_ptp_config(struct mlx5e_priv *priv, struct mlx5e_ptp *ptp_ch, 363 struct devlink_fmsg *fmsg) 364 { 365 int err; 366 367 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "PTP"); 368 if (err) 369 return err; 370 371 err = devlink_fmsg_u32_pair_put(fmsg, "filter_type", priv->tstamp.rx_filter); 372 if (err) 373 return err; 374 375 err = mlx5e_rx_reporter_diagnose_generic_rq(&ptp_ch->rq, fmsg); 376 if (err) 377 return err; 378 379 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 380 } 381 382 static int 383 mlx5e_rx_reporter_diagnose_common_config(struct devlink_health_reporter *reporter, 384 struct devlink_fmsg *fmsg) 385 { 386 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 387 struct mlx5e_rq *generic_rq = &priv->channels.c[0]->rq; 388 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 389 int err; 390 391 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common config"); 392 if (err) 393 return err; 394 395 err = mlx5e_rx_reporter_diagnose_generic_rq(generic_rq, fmsg); 396 if (err) 397 return err; 398 399 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 400 err = mlx5e_rx_reporter_diagnose_common_ptp_config(priv, ptp_ch, fmsg); 401 if (err) 402 return err; 403 } 404 405 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 406 } 407 408 static int mlx5e_rx_reporter_build_diagnose_output_ptp_rq(struct mlx5e_rq *rq, 409 struct devlink_fmsg *fmsg) 410 { 411 int err; 412 413 err = devlink_fmsg_obj_nest_start(fmsg); 414 if (err) 415 return err; 416 417 err = devlink_fmsg_string_pair_put(fmsg, "channel", "ptp"); 418 if (err) 419 return err; 420 421 err = mlx5e_rx_reporter_build_diagnose_output_rq_common(rq, fmsg); 422 if (err) 423 return err; 424 425 err = devlink_fmsg_obj_nest_end(fmsg); 426 if (err) 427 return err; 428 429 return 0; 430 } 431 432 static int mlx5e_rx_reporter_diagnose(struct devlink_health_reporter *reporter, 433 struct devlink_fmsg *fmsg, 434 struct netlink_ext_ack *extack) 435 { 436 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 437 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 438 int i, err = 0; 439 440 mutex_lock(&priv->state_lock); 441 442 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 443 goto unlock; 444 445 err = mlx5e_rx_reporter_diagnose_common_config(reporter, fmsg); 446 if (err) 447 goto unlock; 448 449 err = devlink_fmsg_arr_pair_nest_start(fmsg, "RQs"); 450 if (err) 451 goto unlock; 452 453 for (i = 0; i < priv->channels.num; i++) { 454 struct mlx5e_rq *rq = &priv->channels.c[i]->rq; 455 456 err = mlx5e_rx_reporter_build_diagnose_output(rq, fmsg); 457 if (err) 458 goto unlock; 459 } 460 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 461 err = mlx5e_rx_reporter_build_diagnose_output_ptp_rq(&ptp_ch->rq, fmsg); 462 if (err) 463 goto unlock; 464 } 465 err = devlink_fmsg_arr_pair_nest_end(fmsg); 466 unlock: 467 mutex_unlock(&priv->state_lock); 468 return err; 469 } 470 471 static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 472 void *ctx) 473 { 474 struct mlx5e_txqsq *icosq = ctx; 475 struct mlx5_rsc_key key = {}; 476 int err; 477 478 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 479 return 0; 480 481 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); 482 if (err) 483 return err; 484 485 key.size = PAGE_SIZE; 486 key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 487 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 488 if (err) 489 return err; 490 491 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 492 if (err) 493 return err; 494 495 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); 496 if (err) 497 return err; 498 499 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 500 if (err) 501 return err; 502 503 key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 504 key.index1 = icosq->sqn; 505 key.num_of_obj1 = 1; 506 507 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 508 if (err) 509 return err; 510 511 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 512 if (err) 513 return err; 514 515 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff"); 516 if (err) 517 return err; 518 519 key.rsc = MLX5_SGMT_TYPE_SND_BUFF; 520 key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 521 522 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 523 if (err) 524 return err; 525 526 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 527 if (err) 528 return err; 529 530 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 531 } 532 533 static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 534 void *ctx) 535 { 536 struct mlx5_rsc_key key = {}; 537 struct mlx5e_rq *rq = ctx; 538 int err; 539 540 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 541 return 0; 542 543 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); 544 if (err) 545 return err; 546 547 key.size = PAGE_SIZE; 548 key.rsc = MLX5_SGMT_TYPE_RX_SLICE_ALL; 549 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 550 if (err) 551 return err; 552 553 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 554 if (err) 555 return err; 556 557 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); 558 if (err) 559 return err; 560 561 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 562 if (err) 563 return err; 564 565 key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 566 key.index1 = rq->rqn; 567 key.num_of_obj1 = 1; 568 569 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 570 if (err) 571 return err; 572 573 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 574 if (err) 575 return err; 576 577 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "receive_buff"); 578 if (err) 579 return err; 580 581 key.rsc = MLX5_SGMT_TYPE_RCV_BUFF; 582 key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 583 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 584 if (err) 585 return err; 586 587 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 588 if (err) 589 return err; 590 591 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 592 } 593 594 static int mlx5e_rx_reporter_dump_all_rqs(struct mlx5e_priv *priv, 595 struct devlink_fmsg *fmsg) 596 { 597 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 598 struct mlx5_rsc_key key = {}; 599 int i, err; 600 601 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 602 return 0; 603 604 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); 605 if (err) 606 return err; 607 608 key.size = PAGE_SIZE; 609 key.rsc = MLX5_SGMT_TYPE_RX_SLICE_ALL; 610 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 611 if (err) 612 return err; 613 614 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 615 if (err) 616 return err; 617 618 err = devlink_fmsg_arr_pair_nest_start(fmsg, "RQs"); 619 if (err) 620 return err; 621 622 for (i = 0; i < priv->channels.num; i++) { 623 struct mlx5e_rq *rq = &priv->channels.c[i]->rq; 624 625 err = mlx5e_health_queue_dump(priv, fmsg, rq->rqn, "RQ"); 626 if (err) 627 return err; 628 } 629 630 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 631 err = mlx5e_health_queue_dump(priv, fmsg, ptp_ch->rq.rqn, "PTP RQ"); 632 if (err) 633 return err; 634 } 635 636 return devlink_fmsg_arr_pair_nest_end(fmsg); 637 } 638 639 static int mlx5e_rx_reporter_dump_from_ctx(struct mlx5e_priv *priv, 640 struct mlx5e_err_ctx *err_ctx, 641 struct devlink_fmsg *fmsg) 642 { 643 return err_ctx->dump(priv, fmsg, err_ctx->ctx); 644 } 645 646 static int mlx5e_rx_reporter_dump(struct devlink_health_reporter *reporter, 647 struct devlink_fmsg *fmsg, void *context, 648 struct netlink_ext_ack *extack) 649 { 650 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 651 struct mlx5e_err_ctx *err_ctx = context; 652 653 return err_ctx ? mlx5e_rx_reporter_dump_from_ctx(priv, err_ctx, fmsg) : 654 mlx5e_rx_reporter_dump_all_rqs(priv, fmsg); 655 } 656 657 void mlx5e_reporter_rx_timeout(struct mlx5e_rq *rq) 658 { 659 char icosq_str[MLX5E_REPORTER_PER_Q_MAX_LEN] = {}; 660 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 661 struct mlx5e_icosq *icosq = rq->icosq; 662 struct mlx5e_priv *priv = rq->priv; 663 struct mlx5e_err_ctx err_ctx = {}; 664 665 err_ctx.ctx = rq; 666 err_ctx.recover = mlx5e_rx_reporter_timeout_recover; 667 err_ctx.dump = mlx5e_rx_reporter_dump_rq; 668 669 if (icosq) 670 snprintf(icosq_str, sizeof(icosq_str), "ICOSQ: 0x%x, ", icosq->sqn); 671 snprintf(err_str, sizeof(err_str), 672 "RX timeout on channel: %d, %sRQ: 0x%x, CQ: 0x%x", 673 rq->ix, icosq_str, rq->rqn, rq->cq.mcq.cqn); 674 675 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 676 } 677 678 void mlx5e_reporter_rq_cqe_err(struct mlx5e_rq *rq) 679 { 680 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 681 struct mlx5e_priv *priv = rq->priv; 682 struct mlx5e_err_ctx err_ctx = {}; 683 684 err_ctx.ctx = rq; 685 err_ctx.recover = mlx5e_rx_reporter_err_rq_cqe_recover; 686 err_ctx.dump = mlx5e_rx_reporter_dump_rq; 687 snprintf(err_str, sizeof(err_str), "ERR CQE on RQ: 0x%x", rq->rqn); 688 689 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 690 } 691 692 void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq) 693 { 694 struct mlx5e_priv *priv = icosq->channel->priv; 695 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 696 struct mlx5e_err_ctx err_ctx = {}; 697 698 err_ctx.ctx = icosq; 699 err_ctx.recover = mlx5e_rx_reporter_err_icosq_cqe_recover; 700 err_ctx.dump = mlx5e_rx_reporter_dump_icosq; 701 snprintf(err_str, sizeof(err_str), "ERR CQE on ICOSQ: 0x%x", icosq->sqn); 702 703 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 704 } 705 706 static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = { 707 .name = "rx", 708 .recover = mlx5e_rx_reporter_recover, 709 .diagnose = mlx5e_rx_reporter_diagnose, 710 .dump = mlx5e_rx_reporter_dump, 711 }; 712 713 #define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500 714 715 void mlx5e_reporter_rx_create(struct mlx5e_priv *priv) 716 { 717 struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); 718 struct devlink_health_reporter *reporter; 719 720 reporter = devlink_port_health_reporter_create(dl_port, &mlx5_rx_reporter_ops, 721 MLX5E_REPORTER_RX_GRACEFUL_PERIOD, priv); 722 if (IS_ERR(reporter)) { 723 netdev_warn(priv->netdev, "Failed to create rx reporter, err = %ld\n", 724 PTR_ERR(reporter)); 725 return; 726 } 727 priv->rx_reporter = reporter; 728 } 729 730 void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv) 731 { 732 if (!priv->rx_reporter) 733 return; 734 735 devlink_port_health_reporter_destroy(priv->rx_reporter); 736 priv->rx_reporter = NULL; 737 } 738