1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include "health.h" 5 #include "params.h" 6 #include "txrx.h" 7 #include "devlink.h" 8 #include "ptp.h" 9 #include "lib/tout.h" 10 11 static int mlx5e_query_rq_state(struct mlx5_core_dev *dev, u32 rqn, u8 *state) 12 { 13 int outlen = MLX5_ST_SZ_BYTES(query_rq_out); 14 void *out; 15 void *rqc; 16 int err; 17 18 out = kvzalloc(outlen, GFP_KERNEL); 19 if (!out) 20 return -ENOMEM; 21 22 err = mlx5_core_query_rq(dev, rqn, out); 23 if (err) 24 goto out; 25 26 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context); 27 *state = MLX5_GET(rqc, rqc, state); 28 29 out: 30 kvfree(out); 31 return err; 32 } 33 34 static int mlx5e_wait_for_icosq_flush(struct mlx5e_icosq *icosq) 35 { 36 struct mlx5_core_dev *dev = icosq->channel->mdev; 37 unsigned long exp_time; 38 39 exp_time = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, FLUSH_ON_ERROR)); 40 41 while (time_before(jiffies, exp_time)) { 42 if (icosq->cc == icosq->pc) 43 return 0; 44 45 msleep(20); 46 } 47 48 netdev_err(icosq->channel->netdev, 49 "Wait for ICOSQ 0x%x flush timeout (cc = 0x%x, pc = 0x%x)\n", 50 icosq->sqn, icosq->cc, icosq->pc); 51 52 return -ETIMEDOUT; 53 } 54 55 static void mlx5e_reset_icosq_cc_pc(struct mlx5e_icosq *icosq) 56 { 57 WARN_ONCE(icosq->cc != icosq->pc, "ICOSQ 0x%x: cc (0x%x) != pc (0x%x)\n", 58 icosq->sqn, icosq->cc, icosq->pc); 59 icosq->cc = 0; 60 icosq->pc = 0; 61 } 62 63 static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx) 64 { 65 struct mlx5_core_dev *mdev; 66 struct mlx5e_icosq *icosq; 67 struct net_device *dev; 68 struct mlx5e_rq *rq; 69 u8 state; 70 int err; 71 72 icosq = ctx; 73 rq = &icosq->channel->rq; 74 mdev = icosq->channel->mdev; 75 dev = icosq->channel->netdev; 76 err = mlx5_core_query_sq_state(mdev, icosq->sqn, &state); 77 if (err) { 78 netdev_err(dev, "Failed to query ICOSQ 0x%x state. err = %d\n", 79 icosq->sqn, err); 80 goto out; 81 } 82 83 if (state != MLX5_SQC_STATE_ERR) 84 goto out; 85 86 mlx5e_deactivate_rq(rq); 87 err = mlx5e_wait_for_icosq_flush(icosq); 88 if (err) 89 goto out; 90 91 mlx5e_deactivate_icosq(icosq); 92 93 /* At this point, both the rq and the icosq are disabled */ 94 95 err = mlx5e_health_sq_to_ready(mdev, dev, icosq->sqn); 96 if (err) 97 goto out; 98 99 mlx5e_reset_icosq_cc_pc(icosq); 100 mlx5e_free_rx_in_progress_descs(rq); 101 clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state); 102 mlx5e_activate_icosq(icosq); 103 mlx5e_activate_rq(rq); 104 105 rq->stats->recover++; 106 return 0; 107 out: 108 clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state); 109 return err; 110 } 111 112 static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state) 113 { 114 struct net_device *dev = rq->netdev; 115 int err; 116 117 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 118 if (err) { 119 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 120 return err; 121 } 122 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 123 if (err) { 124 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 125 return err; 126 } 127 128 return 0; 129 } 130 131 static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) 132 { 133 struct mlx5e_rq *rq = ctx; 134 int err; 135 136 mlx5e_deactivate_rq(rq); 137 mlx5e_free_rx_descs(rq); 138 139 err = mlx5e_rq_to_ready(rq, MLX5_RQC_STATE_ERR); 140 if (err) 141 goto out; 142 143 clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); 144 mlx5e_activate_rq(rq); 145 rq->stats->recover++; 146 return 0; 147 out: 148 clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); 149 return err; 150 } 151 152 static int mlx5e_rx_reporter_timeout_recover(void *ctx) 153 { 154 struct mlx5_eq_comp *eq; 155 struct mlx5e_rq *rq; 156 int err; 157 158 rq = ctx; 159 eq = rq->cq.mcq.eq; 160 161 err = mlx5e_health_channel_eq_recover(rq->netdev, eq, rq->cq.ch_stats); 162 if (err && rq->icosq) 163 clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); 164 165 return err; 166 } 167 168 static int mlx5e_rx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx) 169 { 170 return err_ctx->recover(err_ctx->ctx); 171 } 172 173 static int mlx5e_rx_reporter_recover(struct devlink_health_reporter *reporter, 174 void *context, 175 struct netlink_ext_ack *extack) 176 { 177 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 178 struct mlx5e_err_ctx *err_ctx = context; 179 180 return err_ctx ? mlx5e_rx_reporter_recover_from_ctx(err_ctx) : 181 mlx5e_health_recover_channels(priv); 182 } 183 184 static int mlx5e_reporter_icosq_diagnose(struct mlx5e_icosq *icosq, u8 hw_state, 185 struct devlink_fmsg *fmsg) 186 { 187 int err; 188 189 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); 190 if (err) 191 return err; 192 193 err = devlink_fmsg_u32_pair_put(fmsg, "sqn", icosq->sqn); 194 if (err) 195 return err; 196 197 err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); 198 if (err) 199 return err; 200 201 err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cc); 202 if (err) 203 return err; 204 205 err = devlink_fmsg_u32_pair_put(fmsg, "pc", icosq->pc); 206 if (err) 207 return err; 208 209 err = devlink_fmsg_u32_pair_put(fmsg, "WQE size", 210 mlx5_wq_cyc_get_size(&icosq->wq)); 211 if (err) 212 return err; 213 214 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "CQ"); 215 if (err) 216 return err; 217 218 err = devlink_fmsg_u32_pair_put(fmsg, "cqn", icosq->cq.mcq.cqn); 219 if (err) 220 return err; 221 222 err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cq.wq.cc); 223 if (err) 224 return err; 225 226 err = devlink_fmsg_u32_pair_put(fmsg, "size", mlx5_cqwq_get_size(&icosq->cq.wq)); 227 if (err) 228 return err; 229 230 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 231 if (err) 232 return err; 233 234 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 235 } 236 237 static int 238 mlx5e_rx_reporter_build_diagnose_output_rq_common(struct mlx5e_rq *rq, 239 struct devlink_fmsg *fmsg) 240 { 241 u16 wqe_counter; 242 int wqes_sz; 243 u8 hw_state; 244 u16 wq_head; 245 int err; 246 247 err = mlx5e_query_rq_state(rq->mdev, rq->rqn, &hw_state); 248 if (err) 249 return err; 250 251 wqes_sz = mlx5e_rqwq_get_cur_sz(rq); 252 wq_head = mlx5e_rqwq_get_head(rq); 253 wqe_counter = mlx5e_rqwq_get_wqe_counter(rq); 254 255 err = devlink_fmsg_u32_pair_put(fmsg, "rqn", rq->rqn); 256 if (err) 257 return err; 258 259 err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); 260 if (err) 261 return err; 262 263 err = devlink_fmsg_u8_pair_put(fmsg, "SW state", rq->state); 264 if (err) 265 return err; 266 267 err = devlink_fmsg_u32_pair_put(fmsg, "WQE counter", wqe_counter); 268 if (err) 269 return err; 270 271 err = devlink_fmsg_u32_pair_put(fmsg, "posted WQEs", wqes_sz); 272 if (err) 273 return err; 274 275 err = devlink_fmsg_u32_pair_put(fmsg, "cc", wq_head); 276 if (err) 277 return err; 278 279 err = mlx5e_health_cq_diag_fmsg(&rq->cq, fmsg); 280 if (err) 281 return err; 282 283 err = mlx5e_health_eq_diag_fmsg(rq->cq.mcq.eq, fmsg); 284 if (err) 285 return err; 286 287 if (rq->icosq) { 288 struct mlx5e_icosq *icosq = rq->icosq; 289 u8 icosq_hw_state; 290 291 err = mlx5_core_query_sq_state(rq->mdev, icosq->sqn, &icosq_hw_state); 292 if (err) 293 return err; 294 295 err = mlx5e_reporter_icosq_diagnose(icosq, icosq_hw_state, fmsg); 296 if (err) 297 return err; 298 } 299 300 return 0; 301 } 302 303 static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, 304 struct devlink_fmsg *fmsg) 305 { 306 int err; 307 308 err = devlink_fmsg_obj_nest_start(fmsg); 309 if (err) 310 return err; 311 312 err = devlink_fmsg_u32_pair_put(fmsg, "channel ix", rq->ix); 313 if (err) 314 return err; 315 316 err = mlx5e_rx_reporter_build_diagnose_output_rq_common(rq, fmsg); 317 if (err) 318 return err; 319 320 return devlink_fmsg_obj_nest_end(fmsg); 321 } 322 323 static int mlx5e_rx_reporter_diagnose_generic_rq(struct mlx5e_rq *rq, 324 struct devlink_fmsg *fmsg) 325 { 326 struct mlx5e_priv *priv = rq->priv; 327 struct mlx5e_params *params; 328 u32 rq_stride, rq_sz; 329 bool real_time; 330 int err; 331 332 params = &priv->channels.params; 333 rq_sz = mlx5e_rqwq_get_size(rq); 334 real_time = mlx5_is_real_time_rq(priv->mdev); 335 rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL)); 336 337 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); 338 if (err) 339 return err; 340 341 err = devlink_fmsg_u8_pair_put(fmsg, "type", params->rq_wq_type); 342 if (err) 343 return err; 344 345 err = devlink_fmsg_u64_pair_put(fmsg, "stride size", rq_stride); 346 if (err) 347 return err; 348 349 err = devlink_fmsg_u32_pair_put(fmsg, "size", rq_sz); 350 if (err) 351 return err; 352 353 err = devlink_fmsg_string_pair_put(fmsg, "ts_format", real_time ? "RT" : "FRC"); 354 if (err) 355 return err; 356 357 err = mlx5e_health_cq_common_diag_fmsg(&rq->cq, fmsg); 358 if (err) 359 return err; 360 361 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 362 } 363 364 static int 365 mlx5e_rx_reporter_diagnose_common_ptp_config(struct mlx5e_priv *priv, struct mlx5e_ptp *ptp_ch, 366 struct devlink_fmsg *fmsg) 367 { 368 int err; 369 370 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "PTP"); 371 if (err) 372 return err; 373 374 err = devlink_fmsg_u32_pair_put(fmsg, "filter_type", priv->tstamp.rx_filter); 375 if (err) 376 return err; 377 378 err = mlx5e_rx_reporter_diagnose_generic_rq(&ptp_ch->rq, fmsg); 379 if (err) 380 return err; 381 382 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 383 } 384 385 static int 386 mlx5e_rx_reporter_diagnose_common_config(struct devlink_health_reporter *reporter, 387 struct devlink_fmsg *fmsg) 388 { 389 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 390 struct mlx5e_rq *generic_rq = &priv->channels.c[0]->rq; 391 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 392 int err; 393 394 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common config"); 395 if (err) 396 return err; 397 398 err = mlx5e_rx_reporter_diagnose_generic_rq(generic_rq, fmsg); 399 if (err) 400 return err; 401 402 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 403 err = mlx5e_rx_reporter_diagnose_common_ptp_config(priv, ptp_ch, fmsg); 404 if (err) 405 return err; 406 } 407 408 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 409 } 410 411 static int mlx5e_rx_reporter_build_diagnose_output_ptp_rq(struct mlx5e_rq *rq, 412 struct devlink_fmsg *fmsg) 413 { 414 int err; 415 416 err = devlink_fmsg_obj_nest_start(fmsg); 417 if (err) 418 return err; 419 420 err = devlink_fmsg_string_pair_put(fmsg, "channel", "ptp"); 421 if (err) 422 return err; 423 424 err = mlx5e_rx_reporter_build_diagnose_output_rq_common(rq, fmsg); 425 if (err) 426 return err; 427 428 err = devlink_fmsg_obj_nest_end(fmsg); 429 if (err) 430 return err; 431 432 return 0; 433 } 434 435 static int mlx5e_rx_reporter_diagnose(struct devlink_health_reporter *reporter, 436 struct devlink_fmsg *fmsg, 437 struct netlink_ext_ack *extack) 438 { 439 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 440 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 441 int i, err = 0; 442 443 mutex_lock(&priv->state_lock); 444 445 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 446 goto unlock; 447 448 err = mlx5e_rx_reporter_diagnose_common_config(reporter, fmsg); 449 if (err) 450 goto unlock; 451 452 err = devlink_fmsg_arr_pair_nest_start(fmsg, "RQs"); 453 if (err) 454 goto unlock; 455 456 for (i = 0; i < priv->channels.num; i++) { 457 struct mlx5e_rq *rq = &priv->channels.c[i]->rq; 458 459 err = mlx5e_rx_reporter_build_diagnose_output(rq, fmsg); 460 if (err) 461 goto unlock; 462 } 463 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 464 err = mlx5e_rx_reporter_build_diagnose_output_ptp_rq(&ptp_ch->rq, fmsg); 465 if (err) 466 goto unlock; 467 } 468 err = devlink_fmsg_arr_pair_nest_end(fmsg); 469 unlock: 470 mutex_unlock(&priv->state_lock); 471 return err; 472 } 473 474 static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 475 void *ctx) 476 { 477 struct mlx5e_txqsq *icosq = ctx; 478 struct mlx5_rsc_key key = {}; 479 int err; 480 481 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 482 return 0; 483 484 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); 485 if (err) 486 return err; 487 488 key.size = PAGE_SIZE; 489 key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 490 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 491 if (err) 492 return err; 493 494 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 495 if (err) 496 return err; 497 498 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); 499 if (err) 500 return err; 501 502 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 503 if (err) 504 return err; 505 506 key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 507 key.index1 = icosq->sqn; 508 key.num_of_obj1 = 1; 509 510 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 511 if (err) 512 return err; 513 514 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 515 if (err) 516 return err; 517 518 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff"); 519 if (err) 520 return err; 521 522 key.rsc = MLX5_SGMT_TYPE_SND_BUFF; 523 key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 524 525 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 526 if (err) 527 return err; 528 529 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 530 if (err) 531 return err; 532 533 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 534 } 535 536 static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 537 void *ctx) 538 { 539 struct mlx5_rsc_key key = {}; 540 struct mlx5e_rq *rq = ctx; 541 int err; 542 543 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 544 return 0; 545 546 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); 547 if (err) 548 return err; 549 550 key.size = PAGE_SIZE; 551 key.rsc = MLX5_SGMT_TYPE_RX_SLICE_ALL; 552 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 553 if (err) 554 return err; 555 556 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 557 if (err) 558 return err; 559 560 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); 561 if (err) 562 return err; 563 564 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 565 if (err) 566 return err; 567 568 key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 569 key.index1 = rq->rqn; 570 key.num_of_obj1 = 1; 571 572 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 573 if (err) 574 return err; 575 576 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 577 if (err) 578 return err; 579 580 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "receive_buff"); 581 if (err) 582 return err; 583 584 key.rsc = MLX5_SGMT_TYPE_RCV_BUFF; 585 key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 586 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 587 if (err) 588 return err; 589 590 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 591 if (err) 592 return err; 593 594 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 595 } 596 597 static int mlx5e_rx_reporter_dump_all_rqs(struct mlx5e_priv *priv, 598 struct devlink_fmsg *fmsg) 599 { 600 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 601 struct mlx5_rsc_key key = {}; 602 int i, err; 603 604 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 605 return 0; 606 607 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); 608 if (err) 609 return err; 610 611 key.size = PAGE_SIZE; 612 key.rsc = MLX5_SGMT_TYPE_RX_SLICE_ALL; 613 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 614 if (err) 615 return err; 616 617 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 618 if (err) 619 return err; 620 621 err = devlink_fmsg_arr_pair_nest_start(fmsg, "RQs"); 622 if (err) 623 return err; 624 625 for (i = 0; i < priv->channels.num; i++) { 626 struct mlx5e_rq *rq = &priv->channels.c[i]->rq; 627 628 err = mlx5e_health_queue_dump(priv, fmsg, rq->rqn, "RQ"); 629 if (err) 630 return err; 631 } 632 633 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 634 err = mlx5e_health_queue_dump(priv, fmsg, ptp_ch->rq.rqn, "PTP RQ"); 635 if (err) 636 return err; 637 } 638 639 return devlink_fmsg_arr_pair_nest_end(fmsg); 640 } 641 642 static int mlx5e_rx_reporter_dump_from_ctx(struct mlx5e_priv *priv, 643 struct mlx5e_err_ctx *err_ctx, 644 struct devlink_fmsg *fmsg) 645 { 646 return err_ctx->dump(priv, fmsg, err_ctx->ctx); 647 } 648 649 static int mlx5e_rx_reporter_dump(struct devlink_health_reporter *reporter, 650 struct devlink_fmsg *fmsg, void *context, 651 struct netlink_ext_ack *extack) 652 { 653 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 654 struct mlx5e_err_ctx *err_ctx = context; 655 656 return err_ctx ? mlx5e_rx_reporter_dump_from_ctx(priv, err_ctx, fmsg) : 657 mlx5e_rx_reporter_dump_all_rqs(priv, fmsg); 658 } 659 660 void mlx5e_reporter_rx_timeout(struct mlx5e_rq *rq) 661 { 662 char icosq_str[MLX5E_REPORTER_PER_Q_MAX_LEN] = {}; 663 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 664 struct mlx5e_icosq *icosq = rq->icosq; 665 struct mlx5e_priv *priv = rq->priv; 666 struct mlx5e_err_ctx err_ctx = {}; 667 668 err_ctx.ctx = rq; 669 err_ctx.recover = mlx5e_rx_reporter_timeout_recover; 670 err_ctx.dump = mlx5e_rx_reporter_dump_rq; 671 672 if (icosq) 673 snprintf(icosq_str, sizeof(icosq_str), "ICOSQ: 0x%x, ", icosq->sqn); 674 snprintf(err_str, sizeof(err_str), 675 "RX timeout on channel: %d, %sRQ: 0x%x, CQ: 0x%x", 676 rq->ix, icosq_str, rq->rqn, rq->cq.mcq.cqn); 677 678 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 679 } 680 681 void mlx5e_reporter_rq_cqe_err(struct mlx5e_rq *rq) 682 { 683 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 684 struct mlx5e_priv *priv = rq->priv; 685 struct mlx5e_err_ctx err_ctx = {}; 686 687 err_ctx.ctx = rq; 688 err_ctx.recover = mlx5e_rx_reporter_err_rq_cqe_recover; 689 err_ctx.dump = mlx5e_rx_reporter_dump_rq; 690 snprintf(err_str, sizeof(err_str), "ERR CQE on RQ: 0x%x", rq->rqn); 691 692 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 693 } 694 695 void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq) 696 { 697 struct mlx5e_priv *priv = icosq->channel->priv; 698 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 699 struct mlx5e_err_ctx err_ctx = {}; 700 701 err_ctx.ctx = icosq; 702 err_ctx.recover = mlx5e_rx_reporter_err_icosq_cqe_recover; 703 err_ctx.dump = mlx5e_rx_reporter_dump_icosq; 704 snprintf(err_str, sizeof(err_str), "ERR CQE on ICOSQ: 0x%x", icosq->sqn); 705 706 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 707 } 708 709 static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = { 710 .name = "rx", 711 .recover = mlx5e_rx_reporter_recover, 712 .diagnose = mlx5e_rx_reporter_diagnose, 713 .dump = mlx5e_rx_reporter_dump, 714 }; 715 716 #define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500 717 718 void mlx5e_reporter_rx_create(struct mlx5e_priv *priv) 719 { 720 struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); 721 struct devlink_health_reporter *reporter; 722 723 reporter = devlink_port_health_reporter_create(dl_port, &mlx5_rx_reporter_ops, 724 MLX5E_REPORTER_RX_GRACEFUL_PERIOD, priv); 725 if (IS_ERR(reporter)) { 726 netdev_warn(priv->netdev, "Failed to create rx reporter, err = %ld\n", 727 PTR_ERR(reporter)); 728 return; 729 } 730 priv->rx_reporter = reporter; 731 } 732 733 void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv) 734 { 735 if (!priv->rx_reporter) 736 return; 737 738 devlink_port_health_reporter_destroy(priv->rx_reporter); 739 priv->rx_reporter = NULL; 740 } 741