1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 4 #include "fw_reset.h" 5 #include "diag/fw_tracer.h" 6 #include "lib/tout.h" 7 8 enum { 9 MLX5_FW_RESET_FLAGS_RESET_REQUESTED, 10 MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, 11 MLX5_FW_RESET_FLAGS_PENDING_COMP, 12 MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS 13 }; 14 15 struct mlx5_fw_reset { 16 struct mlx5_core_dev *dev; 17 struct mlx5_nb nb; 18 struct workqueue_struct *wq; 19 struct work_struct fw_live_patch_work; 20 struct work_struct reset_request_work; 21 struct work_struct reset_reload_work; 22 struct work_struct reset_now_work; 23 struct work_struct reset_abort_work; 24 unsigned long reset_flags; 25 struct timer_list timer; 26 struct completion done; 27 int ret; 28 }; 29 30 void mlx5_fw_reset_enable_remote_dev_reset_set(struct mlx5_core_dev *dev, bool enable) 31 { 32 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 33 34 if (enable) 35 clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags); 36 else 37 set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags); 38 } 39 40 bool mlx5_fw_reset_enable_remote_dev_reset_get(struct mlx5_core_dev *dev) 41 { 42 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 43 44 return !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags); 45 } 46 47 static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level, 48 u8 reset_type_sel, u8 sync_resp, bool sync_start) 49 { 50 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 51 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 52 53 MLX5_SET(mfrl_reg, in, reset_level, reset_level); 54 MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel); 55 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp); 56 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start); 57 58 return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1); 59 } 60 61 static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level, 62 u8 *reset_type, u8 *reset_state) 63 { 64 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 65 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 66 int err; 67 68 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0); 69 if (err) 70 return err; 71 72 if (reset_level) 73 *reset_level = MLX5_GET(mfrl_reg, out, reset_level); 74 if (reset_type) 75 *reset_type = MLX5_GET(mfrl_reg, out, reset_type); 76 if (reset_state) 77 *reset_state = MLX5_GET(mfrl_reg, out, reset_state); 78 79 return 0; 80 } 81 82 int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type) 83 { 84 return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL); 85 } 86 87 static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev, 88 struct netlink_ext_ack *extack) 89 { 90 u8 reset_state; 91 92 if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state)) 93 goto out; 94 95 switch (reset_state) { 96 case MLX5_MFRL_REG_RESET_STATE_IN_NEGOTIATION: 97 case MLX5_MFRL_REG_RESET_STATE_RESET_IN_PROGRESS: 98 NL_SET_ERR_MSG_MOD(extack, "Sync reset was already triggered"); 99 return -EBUSY; 100 case MLX5_MFRL_REG_RESET_STATE_TIMEOUT: 101 NL_SET_ERR_MSG_MOD(extack, "Sync reset got timeout"); 102 return -ETIMEDOUT; 103 case MLX5_MFRL_REG_RESET_STATE_NACK: 104 NL_SET_ERR_MSG_MOD(extack, "One of the hosts disabled reset"); 105 return -EPERM; 106 } 107 108 out: 109 NL_SET_ERR_MSG_MOD(extack, "Sync reset failed"); 110 return -EIO; 111 } 112 113 int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel, 114 struct netlink_ext_ack *extack) 115 { 116 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 117 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 118 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 119 int err; 120 121 set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags); 122 123 MLX5_SET(mfrl_reg, in, reset_level, MLX5_MFRL_REG_RESET_LEVEL3); 124 MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel); 125 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, 1); 126 err = mlx5_access_reg(dev, in, sizeof(in), out, sizeof(out), 127 MLX5_REG_MFRL, 0, 1, false); 128 if (!err) 129 return 0; 130 131 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags); 132 if (err == -EREMOTEIO && MLX5_CAP_MCAM_FEATURE(dev, reset_state)) 133 return mlx5_fw_reset_get_reset_state_err(dev, extack); 134 135 NL_SET_ERR_MSG_MOD(extack, "Sync reset command failed"); 136 return mlx5_cmd_check(dev, err, in, out); 137 } 138 139 int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev) 140 { 141 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false); 142 } 143 144 static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) 145 { 146 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 147 148 /* if this is the driver that initiated the fw reset, devlink completed the reload */ 149 if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) { 150 complete(&fw_reset->done); 151 } else { 152 mlx5_unload_one(dev); 153 if (mlx5_health_wait_pci_up(dev)) 154 mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); 155 else 156 mlx5_load_one(dev, false); 157 devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0, 158 BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | 159 BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE)); 160 } 161 } 162 163 static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev) 164 { 165 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 166 167 del_timer_sync(&fw_reset->timer); 168 } 169 170 static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health) 171 { 172 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 173 174 if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) { 175 mlx5_core_warn(dev, "Reset request was already cleared\n"); 176 return -EALREADY; 177 } 178 179 mlx5_stop_sync_reset_poll(dev); 180 if (poll_health) 181 mlx5_start_health_poll(dev); 182 return 0; 183 } 184 185 static void mlx5_sync_reset_reload_work(struct work_struct *work) 186 { 187 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 188 reset_reload_work); 189 struct mlx5_core_dev *dev = fw_reset->dev; 190 191 mlx5_sync_reset_clear_reset_requested(dev, false); 192 mlx5_enter_error_state(dev, true); 193 mlx5_fw_reset_complete_reload(dev); 194 } 195 196 #define MLX5_RESET_POLL_INTERVAL (HZ / 10) 197 static void poll_sync_reset(struct timer_list *t) 198 { 199 struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer); 200 struct mlx5_core_dev *dev = fw_reset->dev; 201 u32 fatal_error; 202 203 if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) 204 return; 205 206 fatal_error = mlx5_health_check_fatal_sensors(dev); 207 208 if (fatal_error) { 209 mlx5_core_warn(dev, "Got Device Reset\n"); 210 if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags)) 211 queue_work(fw_reset->wq, &fw_reset->reset_reload_work); 212 else 213 mlx5_core_err(dev, "Device is being removed, Drop new reset work\n"); 214 return; 215 } 216 217 mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL)); 218 } 219 220 static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev) 221 { 222 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 223 224 timer_setup(&fw_reset->timer, poll_sync_reset, 0); 225 fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL); 226 add_timer(&fw_reset->timer); 227 } 228 229 static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev) 230 { 231 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false); 232 } 233 234 static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev) 235 { 236 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false); 237 } 238 239 static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev) 240 { 241 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 242 243 if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) { 244 mlx5_core_warn(dev, "Reset request was already set\n"); 245 return -EALREADY; 246 } 247 mlx5_stop_health_poll(dev, true); 248 mlx5_start_sync_reset_poll(dev); 249 return 0; 250 } 251 252 static void mlx5_fw_live_patch_event(struct work_struct *work) 253 { 254 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 255 fw_live_patch_work); 256 struct mlx5_core_dev *dev = fw_reset->dev; 257 258 mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev), 259 fw_rev_min(dev), fw_rev_sub(dev)); 260 261 if (mlx5_fw_tracer_reload(dev->tracer)) 262 mlx5_core_err(dev, "Failed to reload FW tracer\n"); 263 } 264 265 static void mlx5_sync_reset_request_event(struct work_struct *work) 266 { 267 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 268 reset_request_work); 269 struct mlx5_core_dev *dev = fw_reset->dev; 270 int err; 271 272 if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags)) { 273 err = mlx5_fw_reset_set_reset_sync_nack(dev); 274 mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s", 275 err ? "Failed" : "Sent"); 276 return; 277 } 278 if (mlx5_sync_reset_set_reset_requested(dev)) 279 return; 280 281 err = mlx5_fw_reset_set_reset_sync_ack(dev); 282 if (err) 283 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err); 284 else 285 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n"); 286 } 287 288 static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev) 289 { 290 struct pci_bus *bridge_bus = dev->pdev->bus; 291 struct pci_dev *bridge = bridge_bus->self; 292 u16 reg16, dev_id, sdev_id; 293 unsigned long timeout; 294 struct pci_dev *sdev; 295 int cap, err; 296 u32 reg32; 297 298 /* Check that all functions under the pci bridge are PFs of 299 * this device otherwise fail this function. 300 */ 301 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id); 302 if (err) 303 return err; 304 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { 305 err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id); 306 if (err) 307 return err; 308 if (sdev_id != dev_id) 309 return -EPERM; 310 } 311 312 cap = pci_find_capability(bridge, PCI_CAP_ID_EXP); 313 if (!cap) 314 return -EOPNOTSUPP; 315 316 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { 317 pci_save_state(sdev); 318 pci_cfg_access_lock(sdev); 319 } 320 /* PCI link toggle */ 321 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKCTL, ®16); 322 if (err) 323 return err; 324 reg16 |= PCI_EXP_LNKCTL_LD; 325 err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16); 326 if (err) 327 return err; 328 msleep(500); 329 reg16 &= ~PCI_EXP_LNKCTL_LD; 330 err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16); 331 if (err) 332 return err; 333 334 /* Check link */ 335 err = pci_read_config_dword(bridge, cap + PCI_EXP_LNKCAP, ®32); 336 if (err) 337 return err; 338 if (!(reg32 & PCI_EXP_LNKCAP_DLLLARC)) { 339 mlx5_core_warn(dev, "No PCI link reporting capability (0x%08x)\n", reg32); 340 msleep(1000); 341 goto restore; 342 } 343 344 timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE)); 345 do { 346 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, ®16); 347 if (err) 348 return err; 349 if (reg16 & PCI_EXP_LNKSTA_DLLLA) 350 break; 351 msleep(20); 352 } while (!time_after(jiffies, timeout)); 353 354 if (reg16 & PCI_EXP_LNKSTA_DLLLA) { 355 mlx5_core_info(dev, "PCI Link up\n"); 356 } else { 357 mlx5_core_err(dev, "PCI link not ready (0x%04x) after %llu ms\n", 358 reg16, mlx5_tout_ms(dev, PCI_TOGGLE)); 359 err = -ETIMEDOUT; 360 } 361 362 do { 363 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, ®16); 364 if (err) 365 return err; 366 if (reg16 == dev_id) 367 break; 368 msleep(20); 369 } while (!time_after(jiffies, timeout)); 370 371 if (reg16 == dev_id) { 372 mlx5_core_info(dev, "Firmware responds to PCI config cycles again\n"); 373 } else { 374 mlx5_core_err(dev, "Firmware is not responsive (0x%04x) after %llu ms\n", 375 reg16, mlx5_tout_ms(dev, PCI_TOGGLE)); 376 err = -ETIMEDOUT; 377 } 378 379 restore: 380 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { 381 pci_cfg_access_unlock(sdev); 382 pci_restore_state(sdev); 383 } 384 385 return err; 386 } 387 388 static void mlx5_sync_reset_now_event(struct work_struct *work) 389 { 390 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 391 reset_now_work); 392 struct mlx5_core_dev *dev = fw_reset->dev; 393 int err; 394 395 if (mlx5_sync_reset_clear_reset_requested(dev, false)) 396 return; 397 398 mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n"); 399 400 err = mlx5_cmd_fast_teardown_hca(dev); 401 if (err) { 402 mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err); 403 goto done; 404 } 405 406 err = mlx5_pci_link_toggle(dev); 407 if (err) { 408 mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err); 409 goto done; 410 } 411 412 mlx5_enter_error_state(dev, true); 413 done: 414 fw_reset->ret = err; 415 mlx5_fw_reset_complete_reload(dev); 416 } 417 418 static void mlx5_sync_reset_abort_event(struct work_struct *work) 419 { 420 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 421 reset_abort_work); 422 struct mlx5_core_dev *dev = fw_reset->dev; 423 424 if (mlx5_sync_reset_clear_reset_requested(dev, true)) 425 return; 426 mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n"); 427 } 428 429 static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe) 430 { 431 struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe; 432 u8 sync_event_rst_type; 433 434 sync_fw_update_eqe = &eqe->data.sync_fw_update; 435 sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK; 436 switch (sync_event_rst_type) { 437 case MLX5_SYNC_RST_STATE_RESET_REQUEST: 438 queue_work(fw_reset->wq, &fw_reset->reset_request_work); 439 break; 440 case MLX5_SYNC_RST_STATE_RESET_NOW: 441 queue_work(fw_reset->wq, &fw_reset->reset_now_work); 442 break; 443 case MLX5_SYNC_RST_STATE_RESET_ABORT: 444 queue_work(fw_reset->wq, &fw_reset->reset_abort_work); 445 break; 446 } 447 } 448 449 static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data) 450 { 451 struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb); 452 struct mlx5_eqe *eqe = data; 453 454 if (test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags)) 455 return NOTIFY_DONE; 456 457 switch (eqe->sub_type) { 458 case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT: 459 queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work); 460 break; 461 case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT: 462 mlx5_sync_reset_events_handle(fw_reset, eqe); 463 break; 464 default: 465 return NOTIFY_DONE; 466 } 467 468 return NOTIFY_OK; 469 } 470 471 int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev) 472 { 473 unsigned long pci_sync_update_timeout = mlx5_tout_ms(dev, PCI_SYNC_UPDATE); 474 unsigned long timeout = msecs_to_jiffies(pci_sync_update_timeout); 475 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 476 int err; 477 478 if (!wait_for_completion_timeout(&fw_reset->done, timeout)) { 479 mlx5_core_warn(dev, "FW sync reset timeout after %lu seconds\n", 480 pci_sync_update_timeout / 1000); 481 err = -ETIMEDOUT; 482 goto out; 483 } 484 err = fw_reset->ret; 485 out: 486 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags); 487 return err; 488 } 489 490 void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev) 491 { 492 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 493 494 MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT); 495 mlx5_eq_notifier_register(dev, &fw_reset->nb); 496 } 497 498 void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev) 499 { 500 mlx5_eq_notifier_unregister(dev, &dev->priv.fw_reset->nb); 501 } 502 503 void mlx5_drain_fw_reset(struct mlx5_core_dev *dev) 504 { 505 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 506 507 set_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags); 508 cancel_work_sync(&fw_reset->fw_live_patch_work); 509 cancel_work_sync(&fw_reset->reset_request_work); 510 cancel_work_sync(&fw_reset->reset_reload_work); 511 cancel_work_sync(&fw_reset->reset_now_work); 512 cancel_work_sync(&fw_reset->reset_abort_work); 513 } 514 515 int mlx5_fw_reset_init(struct mlx5_core_dev *dev) 516 { 517 struct mlx5_fw_reset *fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL); 518 519 if (!fw_reset) 520 return -ENOMEM; 521 fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events"); 522 if (!fw_reset->wq) { 523 kfree(fw_reset); 524 return -ENOMEM; 525 } 526 527 fw_reset->dev = dev; 528 dev->priv.fw_reset = fw_reset; 529 530 INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event); 531 INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event); 532 INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work); 533 INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event); 534 INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event); 535 536 init_completion(&fw_reset->done); 537 return 0; 538 } 539 540 void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev) 541 { 542 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 543 544 destroy_workqueue(fw_reset->wq); 545 kfree(dev->priv.fw_reset); 546 } 547