1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include "health.h" 5 #include "params.h" 6 #include "txrx.h" 7 #include "devlink.h" 8 #include "ptp.h" 9 #include "lib/tout.h" 10 11 static int mlx5e_query_rq_state(struct mlx5_core_dev *dev, u32 rqn, u8 *state) 12 { 13 int outlen = MLX5_ST_SZ_BYTES(query_rq_out); 14 void *out; 15 void *rqc; 16 int err; 17 18 out = kvzalloc(outlen, GFP_KERNEL); 19 if (!out) 20 return -ENOMEM; 21 22 err = mlx5_core_query_rq(dev, rqn, out); 23 if (err) 24 goto out; 25 26 rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context); 27 *state = MLX5_GET(rqc, rqc, state); 28 29 out: 30 kvfree(out); 31 return err; 32 } 33 34 static int mlx5e_wait_for_icosq_flush(struct mlx5e_icosq *icosq) 35 { 36 struct mlx5_core_dev *dev = icosq->channel->mdev; 37 unsigned long exp_time; 38 39 exp_time = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, FLUSH_ON_ERROR)); 40 41 while (time_before(jiffies, exp_time)) { 42 if (icosq->cc == icosq->pc) 43 return 0; 44 45 msleep(20); 46 } 47 48 netdev_err(icosq->channel->netdev, 49 "Wait for ICOSQ 0x%x flush timeout (cc = 0x%x, pc = 0x%x)\n", 50 icosq->sqn, icosq->cc, icosq->pc); 51 52 return -ETIMEDOUT; 53 } 54 55 static void mlx5e_reset_icosq_cc_pc(struct mlx5e_icosq *icosq) 56 { 57 WARN_ONCE(icosq->cc != icosq->pc, "ICOSQ 0x%x: cc (0x%x) != pc (0x%x)\n", 58 icosq->sqn, icosq->cc, icosq->pc); 59 icosq->cc = 0; 60 icosq->pc = 0; 61 } 62 63 static int mlx5e_rx_reporter_err_icosq_cqe_recover(void *ctx) 64 { 65 struct mlx5e_rq *xskrq = NULL; 66 struct mlx5_core_dev *mdev; 67 struct mlx5e_icosq *icosq; 68 struct net_device *dev; 69 struct mlx5e_rq *rq; 70 u8 state; 71 int err; 72 73 icosq = ctx; 74 75 mutex_lock(&icosq->channel->icosq_recovery_lock); 76 77 /* mlx5e_close_rq cancels this work before RQ and ICOSQ are killed. */ 78 rq = &icosq->channel->rq; 79 if (test_bit(MLX5E_RQ_STATE_ENABLED, &icosq->channel->xskrq.state)) 80 xskrq = &icosq->channel->xskrq; 81 mdev = icosq->channel->mdev; 82 dev = icosq->channel->netdev; 83 err = mlx5_core_query_sq_state(mdev, icosq->sqn, &state); 84 if (err) { 85 netdev_err(dev, "Failed to query ICOSQ 0x%x state. err = %d\n", 86 icosq->sqn, err); 87 goto out; 88 } 89 90 if (state != MLX5_SQC_STATE_ERR) 91 goto out; 92 93 mlx5e_deactivate_rq(rq); 94 if (xskrq) 95 mlx5e_deactivate_rq(xskrq); 96 97 err = mlx5e_wait_for_icosq_flush(icosq); 98 if (err) 99 goto out; 100 101 mlx5e_deactivate_icosq(icosq); 102 103 /* At this point, both the rq and the icosq are disabled */ 104 105 err = mlx5e_health_sq_to_ready(mdev, dev, icosq->sqn); 106 if (err) 107 goto out; 108 109 mlx5e_reset_icosq_cc_pc(icosq); 110 111 mlx5e_free_rx_in_progress_descs(rq); 112 if (xskrq) 113 mlx5e_free_rx_in_progress_descs(xskrq); 114 115 clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state); 116 mlx5e_activate_icosq(icosq); 117 118 mlx5e_activate_rq(rq); 119 rq->stats->recover++; 120 121 if (xskrq) { 122 mlx5e_activate_rq(xskrq); 123 xskrq->stats->recover++; 124 } 125 126 mlx5e_trigger_napi_icosq(icosq->channel); 127 128 mutex_unlock(&icosq->channel->icosq_recovery_lock); 129 130 return 0; 131 out: 132 clear_bit(MLX5E_SQ_STATE_RECOVERING, &icosq->state); 133 mutex_unlock(&icosq->channel->icosq_recovery_lock); 134 return err; 135 } 136 137 static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx) 138 { 139 struct mlx5e_rq *rq = ctx; 140 int err; 141 142 mlx5e_deactivate_rq(rq); 143 err = mlx5e_flush_rq(rq, MLX5_RQC_STATE_ERR); 144 clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state); 145 if (err) 146 return err; 147 148 mlx5e_activate_rq(rq); 149 rq->stats->recover++; 150 if (rq->channel) 151 mlx5e_trigger_napi_icosq(rq->channel); 152 else 153 mlx5e_trigger_napi_sched(rq->cq.napi); 154 return 0; 155 } 156 157 static int mlx5e_rx_reporter_timeout_recover(void *ctx) 158 { 159 struct mlx5_eq_comp *eq; 160 struct mlx5e_rq *rq; 161 int err; 162 163 rq = ctx; 164 eq = rq->cq.mcq.eq; 165 166 err = mlx5e_health_channel_eq_recover(rq->netdev, eq, rq->cq.ch_stats); 167 if (err && rq->icosq) 168 clear_bit(MLX5E_SQ_STATE_ENABLED, &rq->icosq->state); 169 170 return err; 171 } 172 173 static int mlx5e_rx_reporter_recover_from_ctx(struct mlx5e_err_ctx *err_ctx) 174 { 175 return err_ctx->recover(err_ctx->ctx); 176 } 177 178 static int mlx5e_rx_reporter_recover(struct devlink_health_reporter *reporter, 179 void *context, 180 struct netlink_ext_ack *extack) 181 { 182 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 183 struct mlx5e_err_ctx *err_ctx = context; 184 185 return err_ctx ? mlx5e_rx_reporter_recover_from_ctx(err_ctx) : 186 mlx5e_health_recover_channels(priv); 187 } 188 189 static int mlx5e_reporter_icosq_diagnose(struct mlx5e_icosq *icosq, u8 hw_state, 190 struct devlink_fmsg *fmsg) 191 { 192 int err; 193 194 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); 195 if (err) 196 return err; 197 198 err = devlink_fmsg_u32_pair_put(fmsg, "sqn", icosq->sqn); 199 if (err) 200 return err; 201 202 err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); 203 if (err) 204 return err; 205 206 err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cc); 207 if (err) 208 return err; 209 210 err = devlink_fmsg_u32_pair_put(fmsg, "pc", icosq->pc); 211 if (err) 212 return err; 213 214 err = devlink_fmsg_u32_pair_put(fmsg, "WQE size", 215 mlx5_wq_cyc_get_size(&icosq->wq)); 216 if (err) 217 return err; 218 219 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "CQ"); 220 if (err) 221 return err; 222 223 err = devlink_fmsg_u32_pair_put(fmsg, "cqn", icosq->cq.mcq.cqn); 224 if (err) 225 return err; 226 227 err = devlink_fmsg_u32_pair_put(fmsg, "cc", icosq->cq.wq.cc); 228 if (err) 229 return err; 230 231 err = devlink_fmsg_u32_pair_put(fmsg, "size", mlx5_cqwq_get_size(&icosq->cq.wq)); 232 if (err) 233 return err; 234 235 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 236 if (err) 237 return err; 238 239 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 240 } 241 242 static int 243 mlx5e_rx_reporter_build_diagnose_output_rq_common(struct mlx5e_rq *rq, 244 struct devlink_fmsg *fmsg) 245 { 246 u16 wqe_counter; 247 int wqes_sz; 248 u8 hw_state; 249 u16 wq_head; 250 int err; 251 252 err = mlx5e_query_rq_state(rq->mdev, rq->rqn, &hw_state); 253 if (err) 254 return err; 255 256 wqes_sz = mlx5e_rqwq_get_cur_sz(rq); 257 wq_head = mlx5e_rqwq_get_head(rq); 258 wqe_counter = mlx5e_rqwq_get_wqe_counter(rq); 259 260 err = devlink_fmsg_u32_pair_put(fmsg, "rqn", rq->rqn); 261 if (err) 262 return err; 263 264 err = devlink_fmsg_u8_pair_put(fmsg, "HW state", hw_state); 265 if (err) 266 return err; 267 268 err = devlink_fmsg_u8_pair_put(fmsg, "SW state", rq->state); 269 if (err) 270 return err; 271 272 err = devlink_fmsg_u32_pair_put(fmsg, "WQE counter", wqe_counter); 273 if (err) 274 return err; 275 276 err = devlink_fmsg_u32_pair_put(fmsg, "posted WQEs", wqes_sz); 277 if (err) 278 return err; 279 280 err = devlink_fmsg_u32_pair_put(fmsg, "cc", wq_head); 281 if (err) 282 return err; 283 284 err = mlx5e_health_cq_diag_fmsg(&rq->cq, fmsg); 285 if (err) 286 return err; 287 288 err = mlx5e_health_eq_diag_fmsg(rq->cq.mcq.eq, fmsg); 289 if (err) 290 return err; 291 292 if (rq->icosq) { 293 struct mlx5e_icosq *icosq = rq->icosq; 294 u8 icosq_hw_state; 295 296 err = mlx5_core_query_sq_state(rq->mdev, icosq->sqn, &icosq_hw_state); 297 if (err) 298 return err; 299 300 err = mlx5e_reporter_icosq_diagnose(icosq, icosq_hw_state, fmsg); 301 if (err) 302 return err; 303 } 304 305 return 0; 306 } 307 308 static int mlx5e_rx_reporter_build_diagnose_output(struct mlx5e_rq *rq, 309 struct devlink_fmsg *fmsg) 310 { 311 int err; 312 313 err = devlink_fmsg_obj_nest_start(fmsg); 314 if (err) 315 return err; 316 317 err = devlink_fmsg_u32_pair_put(fmsg, "channel ix", rq->ix); 318 if (err) 319 return err; 320 321 err = mlx5e_rx_reporter_build_diagnose_output_rq_common(rq, fmsg); 322 if (err) 323 return err; 324 325 return devlink_fmsg_obj_nest_end(fmsg); 326 } 327 328 static int mlx5e_rx_reporter_diagnose_generic_rq(struct mlx5e_rq *rq, 329 struct devlink_fmsg *fmsg) 330 { 331 struct mlx5e_priv *priv = rq->priv; 332 struct mlx5e_params *params; 333 u32 rq_stride, rq_sz; 334 bool real_time; 335 int err; 336 337 params = &priv->channels.params; 338 rq_sz = mlx5e_rqwq_get_size(rq); 339 real_time = mlx5_is_real_time_rq(priv->mdev); 340 rq_stride = BIT(mlx5e_mpwqe_get_log_stride_size(priv->mdev, params, NULL)); 341 342 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); 343 if (err) 344 return err; 345 346 err = devlink_fmsg_u8_pair_put(fmsg, "type", params->rq_wq_type); 347 if (err) 348 return err; 349 350 err = devlink_fmsg_u64_pair_put(fmsg, "stride size", rq_stride); 351 if (err) 352 return err; 353 354 err = devlink_fmsg_u32_pair_put(fmsg, "size", rq_sz); 355 if (err) 356 return err; 357 358 err = devlink_fmsg_string_pair_put(fmsg, "ts_format", real_time ? "RT" : "FRC"); 359 if (err) 360 return err; 361 362 err = mlx5e_health_cq_common_diag_fmsg(&rq->cq, fmsg); 363 if (err) 364 return err; 365 366 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 367 } 368 369 static int 370 mlx5e_rx_reporter_diagnose_common_ptp_config(struct mlx5e_priv *priv, struct mlx5e_ptp *ptp_ch, 371 struct devlink_fmsg *fmsg) 372 { 373 int err; 374 375 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "PTP"); 376 if (err) 377 return err; 378 379 err = devlink_fmsg_u32_pair_put(fmsg, "filter_type", priv->tstamp.rx_filter); 380 if (err) 381 return err; 382 383 err = mlx5e_rx_reporter_diagnose_generic_rq(&ptp_ch->rq, fmsg); 384 if (err) 385 return err; 386 387 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 388 } 389 390 static int 391 mlx5e_rx_reporter_diagnose_common_config(struct devlink_health_reporter *reporter, 392 struct devlink_fmsg *fmsg) 393 { 394 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 395 struct mlx5e_rq *generic_rq = &priv->channels.c[0]->rq; 396 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 397 int err; 398 399 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "Common config"); 400 if (err) 401 return err; 402 403 err = mlx5e_rx_reporter_diagnose_generic_rq(generic_rq, fmsg); 404 if (err) 405 return err; 406 407 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 408 err = mlx5e_rx_reporter_diagnose_common_ptp_config(priv, ptp_ch, fmsg); 409 if (err) 410 return err; 411 } 412 413 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 414 } 415 416 static int mlx5e_rx_reporter_build_diagnose_output_ptp_rq(struct mlx5e_rq *rq, 417 struct devlink_fmsg *fmsg) 418 { 419 int err; 420 421 err = devlink_fmsg_obj_nest_start(fmsg); 422 if (err) 423 return err; 424 425 err = devlink_fmsg_string_pair_put(fmsg, "channel", "ptp"); 426 if (err) 427 return err; 428 429 err = mlx5e_rx_reporter_build_diagnose_output_rq_common(rq, fmsg); 430 if (err) 431 return err; 432 433 err = devlink_fmsg_obj_nest_end(fmsg); 434 if (err) 435 return err; 436 437 return 0; 438 } 439 440 static int mlx5e_rx_reporter_diagnose(struct devlink_health_reporter *reporter, 441 struct devlink_fmsg *fmsg, 442 struct netlink_ext_ack *extack) 443 { 444 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 445 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 446 int i, err = 0; 447 448 mutex_lock(&priv->state_lock); 449 450 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 451 goto unlock; 452 453 err = mlx5e_rx_reporter_diagnose_common_config(reporter, fmsg); 454 if (err) 455 goto unlock; 456 457 err = devlink_fmsg_arr_pair_nest_start(fmsg, "RQs"); 458 if (err) 459 goto unlock; 460 461 for (i = 0; i < priv->channels.num; i++) { 462 struct mlx5e_rq *rq = &priv->channels.c[i]->rq; 463 464 err = mlx5e_rx_reporter_build_diagnose_output(rq, fmsg); 465 if (err) 466 goto unlock; 467 } 468 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 469 err = mlx5e_rx_reporter_build_diagnose_output_ptp_rq(&ptp_ch->rq, fmsg); 470 if (err) 471 goto unlock; 472 } 473 err = devlink_fmsg_arr_pair_nest_end(fmsg); 474 unlock: 475 mutex_unlock(&priv->state_lock); 476 return err; 477 } 478 479 static int mlx5e_rx_reporter_dump_icosq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 480 void *ctx) 481 { 482 struct mlx5e_txqsq *icosq = ctx; 483 struct mlx5_rsc_key key = {}; 484 int err; 485 486 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 487 return 0; 488 489 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "SX Slice"); 490 if (err) 491 return err; 492 493 key.size = PAGE_SIZE; 494 key.rsc = MLX5_SGMT_TYPE_SX_SLICE_ALL; 495 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 496 if (err) 497 return err; 498 499 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 500 if (err) 501 return err; 502 503 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "ICOSQ"); 504 if (err) 505 return err; 506 507 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 508 if (err) 509 return err; 510 511 key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 512 key.index1 = icosq->sqn; 513 key.num_of_obj1 = 1; 514 515 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 516 if (err) 517 return err; 518 519 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 520 if (err) 521 return err; 522 523 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "send_buff"); 524 if (err) 525 return err; 526 527 key.rsc = MLX5_SGMT_TYPE_SND_BUFF; 528 key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 529 530 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 531 if (err) 532 return err; 533 534 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 535 if (err) 536 return err; 537 538 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 539 } 540 541 static int mlx5e_rx_reporter_dump_rq(struct mlx5e_priv *priv, struct devlink_fmsg *fmsg, 542 void *ctx) 543 { 544 struct mlx5_rsc_key key = {}; 545 struct mlx5e_rq *rq = ctx; 546 int err; 547 548 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 549 return 0; 550 551 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); 552 if (err) 553 return err; 554 555 key.size = PAGE_SIZE; 556 key.rsc = MLX5_SGMT_TYPE_RX_SLICE_ALL; 557 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 558 if (err) 559 return err; 560 561 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 562 if (err) 563 return err; 564 565 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RQ"); 566 if (err) 567 return err; 568 569 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "QPC"); 570 if (err) 571 return err; 572 573 key.rsc = MLX5_SGMT_TYPE_FULL_QPC; 574 key.index1 = rq->rqn; 575 key.num_of_obj1 = 1; 576 577 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 578 if (err) 579 return err; 580 581 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 582 if (err) 583 return err; 584 585 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "receive_buff"); 586 if (err) 587 return err; 588 589 key.rsc = MLX5_SGMT_TYPE_RCV_BUFF; 590 key.num_of_obj2 = MLX5_RSC_DUMP_ALL; 591 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 592 if (err) 593 return err; 594 595 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 596 if (err) 597 return err; 598 599 return mlx5e_health_fmsg_named_obj_nest_end(fmsg); 600 } 601 602 static int mlx5e_rx_reporter_dump_all_rqs(struct mlx5e_priv *priv, 603 struct devlink_fmsg *fmsg) 604 { 605 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 606 struct mlx5_rsc_key key = {}; 607 int i, err; 608 609 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 610 return 0; 611 612 err = mlx5e_health_fmsg_named_obj_nest_start(fmsg, "RX Slice"); 613 if (err) 614 return err; 615 616 key.size = PAGE_SIZE; 617 key.rsc = MLX5_SGMT_TYPE_RX_SLICE_ALL; 618 err = mlx5e_health_rsc_fmsg_dump(priv, &key, fmsg); 619 if (err) 620 return err; 621 622 err = mlx5e_health_fmsg_named_obj_nest_end(fmsg); 623 if (err) 624 return err; 625 626 err = devlink_fmsg_arr_pair_nest_start(fmsg, "RQs"); 627 if (err) 628 return err; 629 630 for (i = 0; i < priv->channels.num; i++) { 631 struct mlx5e_rq *rq = &priv->channels.c[i]->rq; 632 633 err = mlx5e_health_queue_dump(priv, fmsg, rq->rqn, "RQ"); 634 if (err) 635 return err; 636 } 637 638 if (ptp_ch && test_bit(MLX5E_PTP_STATE_RX, ptp_ch->state)) { 639 err = mlx5e_health_queue_dump(priv, fmsg, ptp_ch->rq.rqn, "PTP RQ"); 640 if (err) 641 return err; 642 } 643 644 return devlink_fmsg_arr_pair_nest_end(fmsg); 645 } 646 647 static int mlx5e_rx_reporter_dump_from_ctx(struct mlx5e_priv *priv, 648 struct mlx5e_err_ctx *err_ctx, 649 struct devlink_fmsg *fmsg) 650 { 651 return err_ctx->dump(priv, fmsg, err_ctx->ctx); 652 } 653 654 static int mlx5e_rx_reporter_dump(struct devlink_health_reporter *reporter, 655 struct devlink_fmsg *fmsg, void *context, 656 struct netlink_ext_ack *extack) 657 { 658 struct mlx5e_priv *priv = devlink_health_reporter_priv(reporter); 659 struct mlx5e_err_ctx *err_ctx = context; 660 661 return err_ctx ? mlx5e_rx_reporter_dump_from_ctx(priv, err_ctx, fmsg) : 662 mlx5e_rx_reporter_dump_all_rqs(priv, fmsg); 663 } 664 665 void mlx5e_reporter_rx_timeout(struct mlx5e_rq *rq) 666 { 667 char icosq_str[MLX5E_REPORTER_PER_Q_MAX_LEN] = {}; 668 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 669 struct mlx5e_icosq *icosq = rq->icosq; 670 struct mlx5e_priv *priv = rq->priv; 671 struct mlx5e_err_ctx err_ctx = {}; 672 673 err_ctx.ctx = rq; 674 err_ctx.recover = mlx5e_rx_reporter_timeout_recover; 675 err_ctx.dump = mlx5e_rx_reporter_dump_rq; 676 677 if (icosq) 678 snprintf(icosq_str, sizeof(icosq_str), "ICOSQ: 0x%x, ", icosq->sqn); 679 snprintf(err_str, sizeof(err_str), 680 "RX timeout on channel: %d, %sRQ: 0x%x, CQ: 0x%x", 681 rq->ix, icosq_str, rq->rqn, rq->cq.mcq.cqn); 682 683 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 684 } 685 686 void mlx5e_reporter_rq_cqe_err(struct mlx5e_rq *rq) 687 { 688 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 689 struct mlx5e_priv *priv = rq->priv; 690 struct mlx5e_err_ctx err_ctx = {}; 691 692 err_ctx.ctx = rq; 693 err_ctx.recover = mlx5e_rx_reporter_err_rq_cqe_recover; 694 err_ctx.dump = mlx5e_rx_reporter_dump_rq; 695 snprintf(err_str, sizeof(err_str), "ERR CQE on RQ: 0x%x", rq->rqn); 696 697 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 698 } 699 700 void mlx5e_reporter_icosq_cqe_err(struct mlx5e_icosq *icosq) 701 { 702 struct mlx5e_priv *priv = icosq->channel->priv; 703 char err_str[MLX5E_REPORTER_PER_Q_MAX_LEN]; 704 struct mlx5e_err_ctx err_ctx = {}; 705 706 err_ctx.ctx = icosq; 707 err_ctx.recover = mlx5e_rx_reporter_err_icosq_cqe_recover; 708 err_ctx.dump = mlx5e_rx_reporter_dump_icosq; 709 snprintf(err_str, sizeof(err_str), "ERR CQE on ICOSQ: 0x%x", icosq->sqn); 710 711 mlx5e_health_report(priv, priv->rx_reporter, err_str, &err_ctx); 712 } 713 714 void mlx5e_reporter_icosq_suspend_recovery(struct mlx5e_channel *c) 715 { 716 mutex_lock(&c->icosq_recovery_lock); 717 } 718 719 void mlx5e_reporter_icosq_resume_recovery(struct mlx5e_channel *c) 720 { 721 mutex_unlock(&c->icosq_recovery_lock); 722 } 723 724 static const struct devlink_health_reporter_ops mlx5_rx_reporter_ops = { 725 .name = "rx", 726 .recover = mlx5e_rx_reporter_recover, 727 .diagnose = mlx5e_rx_reporter_diagnose, 728 .dump = mlx5e_rx_reporter_dump, 729 }; 730 731 #define MLX5E_REPORTER_RX_GRACEFUL_PERIOD 500 732 733 void mlx5e_reporter_rx_create(struct mlx5e_priv *priv) 734 { 735 struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); 736 struct devlink_health_reporter *reporter; 737 738 reporter = devlink_port_health_reporter_create(dl_port, &mlx5_rx_reporter_ops, 739 MLX5E_REPORTER_RX_GRACEFUL_PERIOD, priv); 740 if (IS_ERR(reporter)) { 741 netdev_warn(priv->netdev, "Failed to create rx reporter, err = %ld\n", 742 PTR_ERR(reporter)); 743 return; 744 } 745 priv->rx_reporter = reporter; 746 } 747 748 void mlx5e_reporter_rx_destroy(struct mlx5e_priv *priv) 749 { 750 if (!priv->rx_reporter) 751 return; 752 753 devlink_port_health_reporter_destroy(priv->rx_reporter); 754 priv->rx_reporter = NULL; 755 } 756