1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 4 #include "fw_reset.h" 5 #include "diag/fw_tracer.h" 6 7 enum { 8 MLX5_FW_RESET_FLAGS_RESET_REQUESTED, 9 MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, 10 MLX5_FW_RESET_FLAGS_PENDING_COMP 11 }; 12 13 struct mlx5_fw_reset { 14 struct mlx5_core_dev *dev; 15 struct mlx5_nb nb; 16 struct workqueue_struct *wq; 17 struct work_struct fw_live_patch_work; 18 struct work_struct reset_request_work; 19 struct work_struct reset_reload_work; 20 struct work_struct reset_now_work; 21 struct work_struct reset_abort_work; 22 unsigned long reset_flags; 23 struct timer_list timer; 24 struct completion done; 25 int ret; 26 }; 27 28 void mlx5_fw_reset_enable_remote_dev_reset_set(struct mlx5_core_dev *dev, bool enable) 29 { 30 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 31 32 if (enable) 33 clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags); 34 else 35 set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags); 36 } 37 38 bool mlx5_fw_reset_enable_remote_dev_reset_get(struct mlx5_core_dev *dev) 39 { 40 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 41 42 return !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags); 43 } 44 45 static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level, 46 u8 reset_type_sel, u8 sync_resp, bool sync_start) 47 { 48 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 49 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 50 51 MLX5_SET(mfrl_reg, in, reset_level, reset_level); 52 MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel); 53 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp); 54 MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start); 55 56 return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1); 57 } 58 59 static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type) 60 { 61 u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 62 u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {}; 63 int err; 64 65 err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0); 66 if (err) 67 return err; 68 69 if (reset_level) 70 *reset_level = MLX5_GET(mfrl_reg, out, reset_level); 71 if (reset_type) 72 *reset_type = MLX5_GET(mfrl_reg, out, reset_type); 73 74 return 0; 75 } 76 77 int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type) 78 { 79 return mlx5_reg_mfrl_query(dev, reset_level, reset_type); 80 } 81 82 int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel) 83 { 84 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 85 int err; 86 87 set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags); 88 err = mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, reset_type_sel, 0, true); 89 if (err) 90 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags); 91 return err; 92 } 93 94 int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev) 95 { 96 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false); 97 } 98 99 static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) 100 { 101 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 102 103 /* if this is the driver that initiated the fw reset, devlink completed the reload */ 104 if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) { 105 complete(&fw_reset->done); 106 } else { 107 mlx5_load_one(dev); 108 devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0, 109 BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | 110 BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE)); 111 } 112 } 113 114 static void mlx5_sync_reset_reload_work(struct work_struct *work) 115 { 116 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 117 reset_reload_work); 118 struct mlx5_core_dev *dev = fw_reset->dev; 119 int err; 120 121 mlx5_enter_error_state(dev, true); 122 mlx5_unload_one(dev); 123 err = mlx5_health_wait_pci_up(dev); 124 if (err) 125 mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); 126 fw_reset->ret = err; 127 mlx5_fw_reset_complete_reload(dev); 128 } 129 130 static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev) 131 { 132 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 133 134 del_timer(&fw_reset->timer); 135 } 136 137 static void mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health) 138 { 139 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 140 141 mlx5_stop_sync_reset_poll(dev); 142 clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags); 143 if (poll_health) 144 mlx5_start_health_poll(dev); 145 } 146 147 #define MLX5_RESET_POLL_INTERVAL (HZ / 10) 148 static void poll_sync_reset(struct timer_list *t) 149 { 150 struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer); 151 struct mlx5_core_dev *dev = fw_reset->dev; 152 u32 fatal_error; 153 154 if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) 155 return; 156 157 fatal_error = mlx5_health_check_fatal_sensors(dev); 158 159 if (fatal_error) { 160 mlx5_core_warn(dev, "Got Device Reset\n"); 161 mlx5_sync_reset_clear_reset_requested(dev, false); 162 queue_work(fw_reset->wq, &fw_reset->reset_reload_work); 163 return; 164 } 165 166 mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL)); 167 } 168 169 static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev) 170 { 171 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 172 173 timer_setup(&fw_reset->timer, poll_sync_reset, 0); 174 fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL); 175 add_timer(&fw_reset->timer); 176 } 177 178 static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev) 179 { 180 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false); 181 } 182 183 static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev) 184 { 185 return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false); 186 } 187 188 static void mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev) 189 { 190 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 191 192 mlx5_stop_health_poll(dev, true); 193 set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags); 194 mlx5_start_sync_reset_poll(dev); 195 } 196 197 static void mlx5_fw_live_patch_event(struct work_struct *work) 198 { 199 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 200 fw_live_patch_work); 201 struct mlx5_core_dev *dev = fw_reset->dev; 202 203 mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev), 204 fw_rev_min(dev), fw_rev_sub(dev)); 205 206 if (mlx5_fw_tracer_reload(dev->tracer)) 207 mlx5_core_err(dev, "Failed to reload FW tracer\n"); 208 } 209 210 static void mlx5_sync_reset_request_event(struct work_struct *work) 211 { 212 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 213 reset_request_work); 214 struct mlx5_core_dev *dev = fw_reset->dev; 215 int err; 216 217 if (test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags)) { 218 err = mlx5_fw_reset_set_reset_sync_nack(dev); 219 mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s", 220 err ? "Failed" : "Sent"); 221 return; 222 } 223 mlx5_sync_reset_set_reset_requested(dev); 224 err = mlx5_fw_reset_set_reset_sync_ack(dev); 225 if (err) 226 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err); 227 else 228 mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n"); 229 } 230 231 #define MLX5_PCI_LINK_UP_TIMEOUT 2000 232 233 static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev) 234 { 235 struct pci_bus *bridge_bus = dev->pdev->bus; 236 struct pci_dev *bridge = bridge_bus->self; 237 u16 reg16, dev_id, sdev_id; 238 unsigned long timeout; 239 struct pci_dev *sdev; 240 int cap, err; 241 u32 reg32; 242 243 /* Check that all functions under the pci bridge are PFs of 244 * this device otherwise fail this function. 245 */ 246 err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id); 247 if (err) 248 return err; 249 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { 250 err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id); 251 if (err) 252 return err; 253 if (sdev_id != dev_id) 254 return -EPERM; 255 } 256 257 cap = pci_find_capability(bridge, PCI_CAP_ID_EXP); 258 if (!cap) 259 return -EOPNOTSUPP; 260 261 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { 262 pci_save_state(sdev); 263 pci_cfg_access_lock(sdev); 264 } 265 /* PCI link toggle */ 266 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKCTL, ®16); 267 if (err) 268 return err; 269 reg16 |= PCI_EXP_LNKCTL_LD; 270 err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16); 271 if (err) 272 return err; 273 msleep(500); 274 reg16 &= ~PCI_EXP_LNKCTL_LD; 275 err = pci_write_config_word(bridge, cap + PCI_EXP_LNKCTL, reg16); 276 if (err) 277 return err; 278 279 /* Check link */ 280 err = pci_read_config_dword(bridge, cap + PCI_EXP_LNKCAP, ®32); 281 if (err) 282 return err; 283 if (!(reg32 & PCI_EXP_LNKCAP_DLLLARC)) { 284 mlx5_core_warn(dev, "No PCI link reporting capability (0x%08x)\n", reg32); 285 msleep(1000); 286 goto restore; 287 } 288 289 timeout = jiffies + msecs_to_jiffies(MLX5_PCI_LINK_UP_TIMEOUT); 290 do { 291 err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, ®16); 292 if (err) 293 return err; 294 if (reg16 & PCI_EXP_LNKSTA_DLLLA) 295 break; 296 msleep(20); 297 } while (!time_after(jiffies, timeout)); 298 299 if (reg16 & PCI_EXP_LNKSTA_DLLLA) { 300 mlx5_core_info(dev, "PCI Link up\n"); 301 } else { 302 mlx5_core_err(dev, "PCI link not ready (0x%04x) after %d ms\n", 303 reg16, MLX5_PCI_LINK_UP_TIMEOUT); 304 err = -ETIMEDOUT; 305 } 306 307 restore: 308 list_for_each_entry(sdev, &bridge_bus->devices, bus_list) { 309 pci_cfg_access_unlock(sdev); 310 pci_restore_state(sdev); 311 } 312 313 return err; 314 } 315 316 static void mlx5_sync_reset_now_event(struct work_struct *work) 317 { 318 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 319 reset_now_work); 320 struct mlx5_core_dev *dev = fw_reset->dev; 321 int err; 322 323 mlx5_sync_reset_clear_reset_requested(dev, false); 324 325 mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n"); 326 327 err = mlx5_cmd_fast_teardown_hca(dev); 328 if (err) { 329 mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err); 330 goto done; 331 } 332 333 err = mlx5_pci_link_toggle(dev); 334 if (err) { 335 mlx5_core_warn(dev, "mlx5_pci_link_toggle failed, no reset done, err %d\n", err); 336 goto done; 337 } 338 339 mlx5_enter_error_state(dev, true); 340 mlx5_unload_one(dev); 341 done: 342 fw_reset->ret = err; 343 mlx5_fw_reset_complete_reload(dev); 344 } 345 346 static void mlx5_sync_reset_abort_event(struct work_struct *work) 347 { 348 struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, 349 reset_abort_work); 350 struct mlx5_core_dev *dev = fw_reset->dev; 351 352 mlx5_sync_reset_clear_reset_requested(dev, true); 353 mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n"); 354 } 355 356 static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe) 357 { 358 struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe; 359 u8 sync_event_rst_type; 360 361 sync_fw_update_eqe = &eqe->data.sync_fw_update; 362 sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK; 363 switch (sync_event_rst_type) { 364 case MLX5_SYNC_RST_STATE_RESET_REQUEST: 365 queue_work(fw_reset->wq, &fw_reset->reset_request_work); 366 break; 367 case MLX5_SYNC_RST_STATE_RESET_NOW: 368 queue_work(fw_reset->wq, &fw_reset->reset_now_work); 369 break; 370 case MLX5_SYNC_RST_STATE_RESET_ABORT: 371 queue_work(fw_reset->wq, &fw_reset->reset_abort_work); 372 break; 373 } 374 } 375 376 static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data) 377 { 378 struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb); 379 struct mlx5_eqe *eqe = data; 380 381 switch (eqe->sub_type) { 382 case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT: 383 queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work); 384 break; 385 case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT: 386 mlx5_sync_reset_events_handle(fw_reset, eqe); 387 break; 388 default: 389 return NOTIFY_DONE; 390 } 391 392 return NOTIFY_OK; 393 } 394 395 #define MLX5_FW_RESET_TIMEOUT_MSEC 5000 396 int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev) 397 { 398 unsigned long timeout = msecs_to_jiffies(MLX5_FW_RESET_TIMEOUT_MSEC); 399 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 400 int err; 401 402 if (!wait_for_completion_timeout(&fw_reset->done, timeout)) { 403 mlx5_core_warn(dev, "FW sync reset timeout after %d seconds\n", 404 MLX5_FW_RESET_TIMEOUT_MSEC / 1000); 405 err = -ETIMEDOUT; 406 goto out; 407 } 408 err = fw_reset->ret; 409 out: 410 clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags); 411 return err; 412 } 413 414 void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev) 415 { 416 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 417 418 MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT); 419 mlx5_eq_notifier_register(dev, &fw_reset->nb); 420 } 421 422 void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev) 423 { 424 mlx5_eq_notifier_unregister(dev, &dev->priv.fw_reset->nb); 425 } 426 427 int mlx5_fw_reset_init(struct mlx5_core_dev *dev) 428 { 429 struct mlx5_fw_reset *fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL); 430 431 if (!fw_reset) 432 return -ENOMEM; 433 fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events"); 434 if (!fw_reset->wq) { 435 kfree(fw_reset); 436 return -ENOMEM; 437 } 438 439 fw_reset->dev = dev; 440 dev->priv.fw_reset = fw_reset; 441 442 INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event); 443 INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event); 444 INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work); 445 INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event); 446 INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event); 447 448 init_completion(&fw_reset->done); 449 return 0; 450 } 451 452 void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev) 453 { 454 struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset; 455 456 destroy_workqueue(fw_reset->wq); 457 kfree(dev->priv.fw_reset); 458 } 459