1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/kernel.h> 34 #include <linux/module.h> 35 #include <linux/random.h> 36 #include <linux/vmalloc.h> 37 #include <linux/hardirq.h> 38 #include <linux/mlx5/driver.h> 39 #include "mlx5_core.h" 40 #include "lib/eq.h" 41 #include "lib/mlx5.h" 42 #include "lib/pci_vsc.h" 43 #include "diag/fw_tracer.h" 44 45 enum { 46 MLX5_HEALTH_POLL_INTERVAL = 2 * HZ, 47 MAX_MISSES = 3, 48 }; 49 50 enum { 51 MLX5_HEALTH_SYNDR_FW_ERR = 0x1, 52 MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7, 53 MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8, 54 MLX5_HEALTH_SYNDR_CRC_ERR = 0x9, 55 MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa, 56 MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb, 57 MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc, 58 MLX5_HEALTH_SYNDR_EQ_ERR = 0xd, 59 MLX5_HEALTH_SYNDR_EQ_INV = 0xe, 60 MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf, 61 MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 62 }; 63 64 enum { 65 MLX5_DROP_NEW_HEALTH_WORK, 66 }; 67 68 enum { 69 MLX5_SENSOR_NO_ERR = 0, 70 MLX5_SENSOR_PCI_COMM_ERR = 1, 71 MLX5_SENSOR_PCI_ERR = 2, 72 MLX5_SENSOR_NIC_DISABLED = 3, 73 MLX5_SENSOR_NIC_SW_RESET = 4, 74 MLX5_SENSOR_FW_SYND_RFR = 5, 75 }; 76 77 u8 mlx5_get_nic_state(struct mlx5_core_dev *dev) 78 { 79 return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7; 80 } 81 82 void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state) 83 { 84 u32 cur_cmdq_addr_l_sz; 85 86 cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz); 87 iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) | 88 state << MLX5_NIC_IFC_OFFSET, 89 &dev->iseg->cmdq_addr_l_sz); 90 } 91 92 static bool sensor_pci_not_working(struct mlx5_core_dev *dev) 93 { 94 struct mlx5_core_health *health = &dev->priv.health; 95 struct health_buffer __iomem *h = health->health; 96 97 /* Offline PCI reads return 0xffffffff */ 98 return (ioread32be(&h->fw_ver) == 0xffffffff); 99 } 100 101 static bool sensor_fw_synd_rfr(struct mlx5_core_dev *dev) 102 { 103 struct mlx5_core_health *health = &dev->priv.health; 104 struct health_buffer __iomem *h = health->health; 105 u32 rfr = ioread32be(&h->rfr) >> MLX5_RFR_OFFSET; 106 u8 synd = ioread8(&h->synd); 107 108 if (rfr && synd) 109 mlx5_core_dbg(dev, "FW requests reset, synd: %d\n", synd); 110 return rfr && synd; 111 } 112 113 u32 mlx5_health_check_fatal_sensors(struct mlx5_core_dev *dev) 114 { 115 if (sensor_pci_not_working(dev)) 116 return MLX5_SENSOR_PCI_COMM_ERR; 117 if (pci_channel_offline(dev->pdev)) 118 return MLX5_SENSOR_PCI_ERR; 119 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) 120 return MLX5_SENSOR_NIC_DISABLED; 121 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET) 122 return MLX5_SENSOR_NIC_SW_RESET; 123 if (sensor_fw_synd_rfr(dev)) 124 return MLX5_SENSOR_FW_SYND_RFR; 125 126 return MLX5_SENSOR_NO_ERR; 127 } 128 129 static int lock_sem_sw_reset(struct mlx5_core_dev *dev, bool lock) 130 { 131 enum mlx5_vsc_state state; 132 int ret; 133 134 if (!mlx5_core_is_pf(dev)) 135 return -EBUSY; 136 137 /* Try to lock GW access, this stage doesn't return 138 * EBUSY because locked GW does not mean that other PF 139 * already started the reset. 140 */ 141 ret = mlx5_vsc_gw_lock(dev); 142 if (ret == -EBUSY) 143 return -EINVAL; 144 if (ret) 145 return ret; 146 147 state = lock ? MLX5_VSC_LOCK : MLX5_VSC_UNLOCK; 148 /* At this stage, if the return status == EBUSY, then we know 149 * for sure that another PF started the reset, so don't allow 150 * another reset. 151 */ 152 ret = mlx5_vsc_sem_set_space(dev, MLX5_SEMAPHORE_SW_RESET, state); 153 if (ret) 154 mlx5_core_warn(dev, "Failed to lock SW reset semaphore\n"); 155 156 /* Unlock GW access */ 157 mlx5_vsc_gw_unlock(dev); 158 159 return ret; 160 } 161 162 static bool reset_fw_if_needed(struct mlx5_core_dev *dev) 163 { 164 bool supported = (ioread32be(&dev->iseg->initializing) >> 165 MLX5_FW_RESET_SUPPORTED_OFFSET) & 1; 166 u32 fatal_error; 167 168 if (!supported) 169 return false; 170 171 /* The reset only needs to be issued by one PF. The health buffer is 172 * shared between all functions, and will be cleared during a reset. 173 * Check again to avoid a redundant 2nd reset. If the fatal erros was 174 * PCI related a reset won't help. 175 */ 176 fatal_error = mlx5_health_check_fatal_sensors(dev); 177 if (fatal_error == MLX5_SENSOR_PCI_COMM_ERR || 178 fatal_error == MLX5_SENSOR_NIC_DISABLED || 179 fatal_error == MLX5_SENSOR_NIC_SW_RESET) { 180 mlx5_core_warn(dev, "Not issuing FW reset. Either it's already done or won't help."); 181 return false; 182 } 183 184 mlx5_core_warn(dev, "Issuing FW Reset\n"); 185 /* Write the NIC interface field to initiate the reset, the command 186 * interface address also resides here, don't overwrite it. 187 */ 188 mlx5_set_nic_state(dev, MLX5_NIC_IFC_SW_RESET); 189 190 return true; 191 } 192 193 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) 194 { 195 bool err_detected = false; 196 197 /* Mark the device as fatal in order to abort FW commands */ 198 if ((mlx5_health_check_fatal_sensors(dev) || force) && 199 dev->state == MLX5_DEVICE_STATE_UP) { 200 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 201 err_detected = true; 202 } 203 mutex_lock(&dev->intf_state_mutex); 204 if (!err_detected && dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 205 goto unlock;/* a previous error is still being handled */ 206 if (dev->state == MLX5_DEVICE_STATE_UNINITIALIZED) { 207 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 208 goto unlock; 209 } 210 211 if (mlx5_health_check_fatal_sensors(dev) || force) { /* protected state setting */ 212 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 213 mlx5_cmd_flush(dev); 214 } 215 216 mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1); 217 unlock: 218 mutex_unlock(&dev->intf_state_mutex); 219 } 220 221 #define MLX5_CRDUMP_WAIT_MS 60000 222 #define MLX5_FW_RESET_WAIT_MS 1000 223 void mlx5_error_sw_reset(struct mlx5_core_dev *dev) 224 { 225 unsigned long end, delay_ms = MLX5_FW_RESET_WAIT_MS; 226 int lock = -EBUSY; 227 228 mutex_lock(&dev->intf_state_mutex); 229 if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) 230 goto unlock; 231 232 mlx5_core_err(dev, "start\n"); 233 234 if (mlx5_health_check_fatal_sensors(dev) == MLX5_SENSOR_FW_SYND_RFR) { 235 /* Get cr-dump and reset FW semaphore */ 236 lock = lock_sem_sw_reset(dev, true); 237 238 if (lock == -EBUSY) { 239 delay_ms = MLX5_CRDUMP_WAIT_MS; 240 goto recover_from_sw_reset; 241 } 242 /* Execute SW reset */ 243 reset_fw_if_needed(dev); 244 } 245 246 recover_from_sw_reset: 247 /* Recover from SW reset */ 248 end = jiffies + msecs_to_jiffies(delay_ms); 249 do { 250 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) 251 break; 252 253 msleep(20); 254 } while (!time_after(jiffies, end)); 255 256 if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) { 257 dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n", 258 mlx5_get_nic_state(dev), delay_ms); 259 } 260 261 /* Release FW semaphore if you are the lock owner */ 262 if (!lock) 263 lock_sem_sw_reset(dev, false); 264 265 mlx5_core_err(dev, "end\n"); 266 267 unlock: 268 mutex_unlock(&dev->intf_state_mutex); 269 } 270 271 static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) 272 { 273 u8 nic_interface = mlx5_get_nic_state(dev); 274 275 switch (nic_interface) { 276 case MLX5_NIC_IFC_FULL: 277 mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); 278 break; 279 280 case MLX5_NIC_IFC_DISABLED: 281 mlx5_core_warn(dev, "starting teardown\n"); 282 break; 283 284 case MLX5_NIC_IFC_NO_DRAM_NIC: 285 mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); 286 break; 287 288 case MLX5_NIC_IFC_SW_RESET: 289 /* The IFC mode field is 3 bits, so it will read 0x7 in 2 cases: 290 * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded 291 * and this is a VF), this is not recoverable by SW reset. 292 * Logging of this is handled elsewhere. 293 * 2. FW reset has been issued by another function, driver can 294 * be reloaded to recover after the mode switches to 295 * MLX5_NIC_IFC_DISABLED. 296 */ 297 if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR) 298 mlx5_core_warn(dev, "NIC SW reset in progress\n"); 299 break; 300 301 default: 302 mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", 303 nic_interface); 304 } 305 306 mlx5_disable_device(dev); 307 } 308 309 /* How much time to wait until health resetting the driver (in msecs) */ 310 #define MLX5_RECOVERY_WAIT_MSECS 60000 311 int mlx5_health_wait_pci_up(struct mlx5_core_dev *dev) 312 { 313 unsigned long end; 314 315 end = jiffies + msecs_to_jiffies(MLX5_RECOVERY_WAIT_MSECS); 316 while (sensor_pci_not_working(dev)) { 317 if (time_after(jiffies, end)) 318 return -ETIMEDOUT; 319 msleep(100); 320 } 321 return 0; 322 } 323 324 static int mlx5_health_try_recover(struct mlx5_core_dev *dev) 325 { 326 mlx5_core_warn(dev, "handling bad device here\n"); 327 mlx5_handle_bad_state(dev); 328 if (mlx5_health_wait_pci_up(dev)) { 329 mlx5_core_err(dev, "health recovery flow aborted, PCI reads still not working\n"); 330 return -EIO; 331 } 332 mlx5_core_err(dev, "starting health recovery flow\n"); 333 mlx5_recover_device(dev); 334 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state) || 335 mlx5_health_check_fatal_sensors(dev)) { 336 mlx5_core_err(dev, "health recovery failed\n"); 337 return -EIO; 338 } 339 return 0; 340 } 341 342 static const char *hsynd_str(u8 synd) 343 { 344 switch (synd) { 345 case MLX5_HEALTH_SYNDR_FW_ERR: 346 return "firmware internal error"; 347 case MLX5_HEALTH_SYNDR_IRISC_ERR: 348 return "irisc not responding"; 349 case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: 350 return "unrecoverable hardware error"; 351 case MLX5_HEALTH_SYNDR_CRC_ERR: 352 return "firmware CRC error"; 353 case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: 354 return "ICM fetch PCI error"; 355 case MLX5_HEALTH_SYNDR_HW_FTL_ERR: 356 return "HW fatal error\n"; 357 case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: 358 return "async EQ buffer overrun"; 359 case MLX5_HEALTH_SYNDR_EQ_ERR: 360 return "EQ error"; 361 case MLX5_HEALTH_SYNDR_EQ_INV: 362 return "Invalid EQ referenced"; 363 case MLX5_HEALTH_SYNDR_FFSER_ERR: 364 return "FFSER error"; 365 case MLX5_HEALTH_SYNDR_HIGH_TEMP: 366 return "High temperature"; 367 default: 368 return "unrecognized error"; 369 } 370 } 371 372 static void print_health_info(struct mlx5_core_dev *dev) 373 { 374 struct mlx5_core_health *health = &dev->priv.health; 375 struct health_buffer __iomem *h = health->health; 376 char fw_str[18]; 377 u32 fw; 378 int i; 379 380 /* If the syndrome is 0, the device is OK and no need to print buffer */ 381 if (!ioread8(&h->synd)) 382 return; 383 384 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) 385 mlx5_core_err(dev, "assert_var[%d] 0x%08x\n", i, 386 ioread32be(h->assert_var + i)); 387 388 mlx5_core_err(dev, "assert_exit_ptr 0x%08x\n", 389 ioread32be(&h->assert_exit_ptr)); 390 mlx5_core_err(dev, "assert_callra 0x%08x\n", 391 ioread32be(&h->assert_callra)); 392 sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 393 mlx5_core_err(dev, "fw_ver %s\n", fw_str); 394 mlx5_core_err(dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); 395 mlx5_core_err(dev, "irisc_index %d\n", ioread8(&h->irisc_index)); 396 mlx5_core_err(dev, "synd 0x%x: %s\n", ioread8(&h->synd), 397 hsynd_str(ioread8(&h->synd))); 398 mlx5_core_err(dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); 399 fw = ioread32be(&h->fw_ver); 400 mlx5_core_err(dev, "raw fw_ver 0x%08x\n", fw); 401 } 402 403 static int 404 mlx5_fw_reporter_diagnose(struct devlink_health_reporter *reporter, 405 struct devlink_fmsg *fmsg, 406 struct netlink_ext_ack *extack) 407 { 408 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 409 struct mlx5_core_health *health = &dev->priv.health; 410 struct health_buffer __iomem *h = health->health; 411 u8 synd; 412 int err; 413 414 synd = ioread8(&h->synd); 415 err = devlink_fmsg_u8_pair_put(fmsg, "Syndrome", synd); 416 if (err || !synd) 417 return err; 418 return devlink_fmsg_string_pair_put(fmsg, "Description", hsynd_str(synd)); 419 } 420 421 struct mlx5_fw_reporter_ctx { 422 u8 err_synd; 423 int miss_counter; 424 }; 425 426 static int 427 mlx5_fw_reporter_ctx_pairs_put(struct devlink_fmsg *fmsg, 428 struct mlx5_fw_reporter_ctx *fw_reporter_ctx) 429 { 430 int err; 431 432 err = devlink_fmsg_u8_pair_put(fmsg, "syndrome", 433 fw_reporter_ctx->err_synd); 434 if (err) 435 return err; 436 err = devlink_fmsg_u32_pair_put(fmsg, "fw_miss_counter", 437 fw_reporter_ctx->miss_counter); 438 if (err) 439 return err; 440 return 0; 441 } 442 443 static int 444 mlx5_fw_reporter_heath_buffer_data_put(struct mlx5_core_dev *dev, 445 struct devlink_fmsg *fmsg) 446 { 447 struct mlx5_core_health *health = &dev->priv.health; 448 struct health_buffer __iomem *h = health->health; 449 int err; 450 int i; 451 452 if (!ioread8(&h->synd)) 453 return 0; 454 455 err = devlink_fmsg_pair_nest_start(fmsg, "health buffer"); 456 if (err) 457 return err; 458 err = devlink_fmsg_obj_nest_start(fmsg); 459 if (err) 460 return err; 461 err = devlink_fmsg_arr_pair_nest_start(fmsg, "assert_var"); 462 if (err) 463 return err; 464 465 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) { 466 err = devlink_fmsg_u32_put(fmsg, ioread32be(h->assert_var + i)); 467 if (err) 468 return err; 469 } 470 err = devlink_fmsg_arr_pair_nest_end(fmsg); 471 if (err) 472 return err; 473 err = devlink_fmsg_u32_pair_put(fmsg, "assert_exit_ptr", 474 ioread32be(&h->assert_exit_ptr)); 475 if (err) 476 return err; 477 err = devlink_fmsg_u32_pair_put(fmsg, "assert_callra", 478 ioread32be(&h->assert_callra)); 479 if (err) 480 return err; 481 err = devlink_fmsg_u32_pair_put(fmsg, "hw_id", ioread32be(&h->hw_id)); 482 if (err) 483 return err; 484 err = devlink_fmsg_u8_pair_put(fmsg, "irisc_index", 485 ioread8(&h->irisc_index)); 486 if (err) 487 return err; 488 err = devlink_fmsg_u8_pair_put(fmsg, "synd", ioread8(&h->synd)); 489 if (err) 490 return err; 491 err = devlink_fmsg_u32_pair_put(fmsg, "ext_synd", 492 ioread16be(&h->ext_synd)); 493 if (err) 494 return err; 495 err = devlink_fmsg_u32_pair_put(fmsg, "raw_fw_ver", 496 ioread32be(&h->fw_ver)); 497 if (err) 498 return err; 499 err = devlink_fmsg_obj_nest_end(fmsg); 500 if (err) 501 return err; 502 return devlink_fmsg_pair_nest_end(fmsg); 503 } 504 505 static int 506 mlx5_fw_reporter_dump(struct devlink_health_reporter *reporter, 507 struct devlink_fmsg *fmsg, void *priv_ctx, 508 struct netlink_ext_ack *extack) 509 { 510 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 511 int err; 512 513 err = mlx5_fw_tracer_trigger_core_dump_general(dev); 514 if (err) 515 return err; 516 517 if (priv_ctx) { 518 struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx; 519 520 err = mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx); 521 if (err) 522 return err; 523 } 524 525 err = mlx5_fw_reporter_heath_buffer_data_put(dev, fmsg); 526 if (err) 527 return err; 528 return mlx5_fw_tracer_get_saved_traces_objects(dev->tracer, fmsg); 529 } 530 531 static void mlx5_fw_reporter_err_work(struct work_struct *work) 532 { 533 struct mlx5_fw_reporter_ctx fw_reporter_ctx; 534 struct mlx5_core_health *health; 535 536 health = container_of(work, struct mlx5_core_health, report_work); 537 538 if (IS_ERR_OR_NULL(health->fw_reporter)) 539 return; 540 541 fw_reporter_ctx.err_synd = health->synd; 542 fw_reporter_ctx.miss_counter = health->miss_counter; 543 if (fw_reporter_ctx.err_synd) { 544 devlink_health_report(health->fw_reporter, 545 "FW syndrom reported", &fw_reporter_ctx); 546 return; 547 } 548 if (fw_reporter_ctx.miss_counter) 549 devlink_health_report(health->fw_reporter, 550 "FW miss counter reported", 551 &fw_reporter_ctx); 552 } 553 554 static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = { 555 .name = "fw", 556 .diagnose = mlx5_fw_reporter_diagnose, 557 .dump = mlx5_fw_reporter_dump, 558 }; 559 560 static int 561 mlx5_fw_fatal_reporter_recover(struct devlink_health_reporter *reporter, 562 void *priv_ctx, 563 struct netlink_ext_ack *extack) 564 { 565 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 566 567 return mlx5_health_try_recover(dev); 568 } 569 570 static int 571 mlx5_fw_fatal_reporter_dump(struct devlink_health_reporter *reporter, 572 struct devlink_fmsg *fmsg, void *priv_ctx, 573 struct netlink_ext_ack *extack) 574 { 575 struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter); 576 u32 crdump_size = dev->priv.health.crdump_size; 577 u32 *cr_data; 578 int err; 579 580 if (!mlx5_core_is_pf(dev)) 581 return -EPERM; 582 583 cr_data = kvmalloc(crdump_size, GFP_KERNEL); 584 if (!cr_data) 585 return -ENOMEM; 586 err = mlx5_crdump_collect(dev, cr_data); 587 if (err) 588 goto free_data; 589 590 if (priv_ctx) { 591 struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx; 592 593 err = mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx); 594 if (err) 595 goto free_data; 596 } 597 598 err = devlink_fmsg_binary_pair_put(fmsg, "crdump_data", cr_data, crdump_size); 599 600 free_data: 601 kvfree(cr_data); 602 return err; 603 } 604 605 static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) 606 { 607 struct mlx5_fw_reporter_ctx fw_reporter_ctx; 608 struct mlx5_core_health *health; 609 struct mlx5_core_dev *dev; 610 struct mlx5_priv *priv; 611 612 health = container_of(work, struct mlx5_core_health, fatal_report_work); 613 priv = container_of(health, struct mlx5_priv, health); 614 dev = container_of(priv, struct mlx5_core_dev, priv); 615 616 mlx5_enter_error_state(dev, false); 617 if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) { 618 if (mlx5_health_try_recover(dev)) 619 mlx5_core_err(dev, "health recovery failed\n"); 620 return; 621 } 622 fw_reporter_ctx.err_synd = health->synd; 623 fw_reporter_ctx.miss_counter = health->miss_counter; 624 devlink_health_report(health->fw_fatal_reporter, 625 "FW fatal error reported", &fw_reporter_ctx); 626 } 627 628 static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = { 629 .name = "fw_fatal", 630 .recover = mlx5_fw_fatal_reporter_recover, 631 .dump = mlx5_fw_fatal_reporter_dump, 632 }; 633 634 #define MLX5_REPORTER_FW_GRACEFUL_PERIOD 1200000 635 static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev) 636 { 637 struct mlx5_core_health *health = &dev->priv.health; 638 struct devlink *devlink = priv_to_devlink(dev); 639 640 health->fw_reporter = 641 devlink_health_reporter_create(devlink, &mlx5_fw_reporter_ops, 642 0, dev); 643 if (IS_ERR(health->fw_reporter)) 644 mlx5_core_warn(dev, "Failed to create fw reporter, err = %ld\n", 645 PTR_ERR(health->fw_reporter)); 646 647 health->fw_fatal_reporter = 648 devlink_health_reporter_create(devlink, 649 &mlx5_fw_fatal_reporter_ops, 650 MLX5_REPORTER_FW_GRACEFUL_PERIOD, 651 dev); 652 if (IS_ERR(health->fw_fatal_reporter)) 653 mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %ld\n", 654 PTR_ERR(health->fw_fatal_reporter)); 655 } 656 657 static void mlx5_fw_reporters_destroy(struct mlx5_core_dev *dev) 658 { 659 struct mlx5_core_health *health = &dev->priv.health; 660 661 if (!IS_ERR_OR_NULL(health->fw_reporter)) 662 devlink_health_reporter_destroy(health->fw_reporter); 663 664 if (!IS_ERR_OR_NULL(health->fw_fatal_reporter)) 665 devlink_health_reporter_destroy(health->fw_fatal_reporter); 666 } 667 668 static unsigned long get_next_poll_jiffies(void) 669 { 670 unsigned long next; 671 672 get_random_bytes(&next, sizeof(next)); 673 next %= HZ; 674 next += jiffies + MLX5_HEALTH_POLL_INTERVAL; 675 676 return next; 677 } 678 679 void mlx5_trigger_health_work(struct mlx5_core_dev *dev) 680 { 681 struct mlx5_core_health *health = &dev->priv.health; 682 unsigned long flags; 683 684 spin_lock_irqsave(&health->wq_lock, flags); 685 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) 686 queue_work(health->wq, &health->fatal_report_work); 687 else 688 mlx5_core_err(dev, "new health works are not permitted at this stage\n"); 689 spin_unlock_irqrestore(&health->wq_lock, flags); 690 } 691 692 static void poll_health(struct timer_list *t) 693 { 694 struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer); 695 struct mlx5_core_health *health = &dev->priv.health; 696 struct health_buffer __iomem *h = health->health; 697 u32 fatal_error; 698 u8 prev_synd; 699 u32 count; 700 701 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 702 goto out; 703 704 fatal_error = mlx5_health_check_fatal_sensors(dev); 705 706 if (fatal_error && !health->fatal_error) { 707 mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error); 708 dev->priv.health.fatal_error = fatal_error; 709 print_health_info(dev); 710 mlx5_trigger_health_work(dev); 711 goto out; 712 } 713 714 count = ioread32be(health->health_counter); 715 if (count == health->prev) 716 ++health->miss_counter; 717 else 718 health->miss_counter = 0; 719 720 health->prev = count; 721 if (health->miss_counter == MAX_MISSES) { 722 mlx5_core_err(dev, "device's health compromised - reached miss count\n"); 723 print_health_info(dev); 724 queue_work(health->wq, &health->report_work); 725 } 726 727 prev_synd = health->synd; 728 health->synd = ioread8(&h->synd); 729 if (health->synd && health->synd != prev_synd) 730 queue_work(health->wq, &health->report_work); 731 732 out: 733 mod_timer(&health->timer, get_next_poll_jiffies()); 734 } 735 736 void mlx5_start_health_poll(struct mlx5_core_dev *dev) 737 { 738 struct mlx5_core_health *health = &dev->priv.health; 739 740 timer_setup(&health->timer, poll_health, 0); 741 health->fatal_error = MLX5_SENSOR_NO_ERR; 742 clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 743 health->health = &dev->iseg->health; 744 health->health_counter = &dev->iseg->health_counter; 745 746 health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL); 747 add_timer(&health->timer); 748 } 749 750 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health) 751 { 752 struct mlx5_core_health *health = &dev->priv.health; 753 unsigned long flags; 754 755 if (disable_health) { 756 spin_lock_irqsave(&health->wq_lock, flags); 757 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 758 spin_unlock_irqrestore(&health->wq_lock, flags); 759 } 760 761 del_timer_sync(&health->timer); 762 } 763 764 void mlx5_drain_health_wq(struct mlx5_core_dev *dev) 765 { 766 struct mlx5_core_health *health = &dev->priv.health; 767 unsigned long flags; 768 769 spin_lock_irqsave(&health->wq_lock, flags); 770 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 771 spin_unlock_irqrestore(&health->wq_lock, flags); 772 cancel_work_sync(&health->report_work); 773 cancel_work_sync(&health->fatal_report_work); 774 } 775 776 void mlx5_health_flush(struct mlx5_core_dev *dev) 777 { 778 struct mlx5_core_health *health = &dev->priv.health; 779 780 flush_workqueue(health->wq); 781 } 782 783 void mlx5_health_cleanup(struct mlx5_core_dev *dev) 784 { 785 struct mlx5_core_health *health = &dev->priv.health; 786 787 destroy_workqueue(health->wq); 788 mlx5_fw_reporters_destroy(dev); 789 } 790 791 int mlx5_health_init(struct mlx5_core_dev *dev) 792 { 793 struct mlx5_core_health *health; 794 char *name; 795 796 mlx5_fw_reporters_create(dev); 797 798 health = &dev->priv.health; 799 name = kmalloc(64, GFP_KERNEL); 800 if (!name) 801 goto out_err; 802 803 strcpy(name, "mlx5_health"); 804 strcat(name, dev_name(dev->device)); 805 health->wq = create_singlethread_workqueue(name); 806 kfree(name); 807 if (!health->wq) 808 goto out_err; 809 spin_lock_init(&health->wq_lock); 810 INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work); 811 INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work); 812 813 return 0; 814 815 out_err: 816 mlx5_fw_reporters_destroy(dev); 817 return -ENOMEM; 818 } 819