1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/kernel.h> 34 #include <linux/module.h> 35 #include <linux/random.h> 36 #include <linux/vmalloc.h> 37 #include <linux/hardirq.h> 38 #include <linux/mlx5/driver.h> 39 #include <linux/mlx5/cmd.h> 40 #include "mlx5_core.h" 41 42 enum { 43 MLX5_HEALTH_POLL_INTERVAL = 2 * HZ, 44 MAX_MISSES = 3, 45 }; 46 47 enum { 48 MLX5_HEALTH_SYNDR_FW_ERR = 0x1, 49 MLX5_HEALTH_SYNDR_IRISC_ERR = 0x7, 50 MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR = 0x8, 51 MLX5_HEALTH_SYNDR_CRC_ERR = 0x9, 52 MLX5_HEALTH_SYNDR_FETCH_PCI_ERR = 0xa, 53 MLX5_HEALTH_SYNDR_HW_FTL_ERR = 0xb, 54 MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR = 0xc, 55 MLX5_HEALTH_SYNDR_EQ_ERR = 0xd, 56 MLX5_HEALTH_SYNDR_EQ_INV = 0xe, 57 MLX5_HEALTH_SYNDR_FFSER_ERR = 0xf, 58 MLX5_HEALTH_SYNDR_HIGH_TEMP = 0x10 59 }; 60 61 enum { 62 MLX5_NIC_IFC_FULL = 0, 63 MLX5_NIC_IFC_DISABLED = 1, 64 MLX5_NIC_IFC_NO_DRAM_NIC = 2, 65 MLX5_NIC_IFC_INVALID = 3 66 }; 67 68 enum { 69 MLX5_DROP_NEW_HEALTH_WORK, 70 MLX5_DROP_NEW_RECOVERY_WORK, 71 }; 72 73 static u8 get_nic_state(struct mlx5_core_dev *dev) 74 { 75 return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 3; 76 } 77 78 static void trigger_cmd_completions(struct mlx5_core_dev *dev) 79 { 80 unsigned long flags; 81 u64 vector; 82 83 /* wait for pending handlers to complete */ 84 synchronize_irq(pci_irq_vector(dev->pdev, MLX5_EQ_VEC_CMD)); 85 spin_lock_irqsave(&dev->cmd.alloc_lock, flags); 86 vector = ~dev->cmd.bitmask & ((1ul << (1 << dev->cmd.log_sz)) - 1); 87 if (!vector) 88 goto no_trig; 89 90 vector |= MLX5_TRIGGERED_CMD_COMP; 91 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 92 93 mlx5_core_dbg(dev, "vector 0x%llx\n", vector); 94 mlx5_cmd_comp_handler(dev, vector, true); 95 return; 96 97 no_trig: 98 spin_unlock_irqrestore(&dev->cmd.alloc_lock, flags); 99 } 100 101 static int in_fatal(struct mlx5_core_dev *dev) 102 { 103 struct mlx5_core_health *health = &dev->priv.health; 104 struct health_buffer __iomem *h = health->health; 105 106 if (get_nic_state(dev) == MLX5_NIC_IFC_DISABLED) 107 return 1; 108 109 if (ioread32be(&h->fw_ver) == 0xffffffff) 110 return 1; 111 112 return 0; 113 } 114 115 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force) 116 { 117 mutex_lock(&dev->intf_state_mutex); 118 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 119 goto unlock; 120 121 mlx5_core_err(dev, "start\n"); 122 if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) { 123 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; 124 trigger_cmd_completions(dev); 125 } 126 127 mlx5_core_event(dev, MLX5_DEV_EVENT_SYS_ERROR, 0); 128 mlx5_core_err(dev, "end\n"); 129 130 unlock: 131 mutex_unlock(&dev->intf_state_mutex); 132 } 133 134 static void mlx5_handle_bad_state(struct mlx5_core_dev *dev) 135 { 136 u8 nic_interface = get_nic_state(dev); 137 138 switch (nic_interface) { 139 case MLX5_NIC_IFC_FULL: 140 mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n"); 141 break; 142 143 case MLX5_NIC_IFC_DISABLED: 144 mlx5_core_warn(dev, "starting teardown\n"); 145 break; 146 147 case MLX5_NIC_IFC_NO_DRAM_NIC: 148 mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n"); 149 break; 150 default: 151 mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n", 152 nic_interface); 153 } 154 155 mlx5_disable_device(dev); 156 } 157 158 static void health_recover(struct work_struct *work) 159 { 160 struct mlx5_core_health *health; 161 struct delayed_work *dwork; 162 struct mlx5_core_dev *dev; 163 struct mlx5_priv *priv; 164 u8 nic_state; 165 166 dwork = container_of(work, struct delayed_work, work); 167 health = container_of(dwork, struct mlx5_core_health, recover_work); 168 priv = container_of(health, struct mlx5_priv, health); 169 dev = container_of(priv, struct mlx5_core_dev, priv); 170 171 nic_state = get_nic_state(dev); 172 if (nic_state == MLX5_NIC_IFC_INVALID) { 173 dev_err(&dev->pdev->dev, "health recovery flow aborted since the nic state is invalid\n"); 174 return; 175 } 176 177 dev_err(&dev->pdev->dev, "starting health recovery flow\n"); 178 mlx5_recover_device(dev); 179 } 180 181 /* How much time to wait until health resetting the driver (in msecs) */ 182 #define MLX5_RECOVERY_DELAY_MSECS 60000 183 static void health_care(struct work_struct *work) 184 { 185 unsigned long recover_delay = msecs_to_jiffies(MLX5_RECOVERY_DELAY_MSECS); 186 struct mlx5_core_health *health; 187 struct mlx5_core_dev *dev; 188 struct mlx5_priv *priv; 189 unsigned long flags; 190 191 health = container_of(work, struct mlx5_core_health, work); 192 priv = container_of(health, struct mlx5_priv, health); 193 dev = container_of(priv, struct mlx5_core_dev, priv); 194 mlx5_core_warn(dev, "handling bad device here\n"); 195 mlx5_handle_bad_state(dev); 196 197 spin_lock_irqsave(&health->wq_lock, flags); 198 if (!test_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags)) 199 schedule_delayed_work(&health->recover_work, recover_delay); 200 else 201 dev_err(&dev->pdev->dev, 202 "new health works are not permitted at this stage\n"); 203 spin_unlock_irqrestore(&health->wq_lock, flags); 204 } 205 206 static const char *hsynd_str(u8 synd) 207 { 208 switch (synd) { 209 case MLX5_HEALTH_SYNDR_FW_ERR: 210 return "firmware internal error"; 211 case MLX5_HEALTH_SYNDR_IRISC_ERR: 212 return "irisc not responding"; 213 case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR: 214 return "unrecoverable hardware error"; 215 case MLX5_HEALTH_SYNDR_CRC_ERR: 216 return "firmware CRC error"; 217 case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR: 218 return "ICM fetch PCI error"; 219 case MLX5_HEALTH_SYNDR_HW_FTL_ERR: 220 return "HW fatal error\n"; 221 case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR: 222 return "async EQ buffer overrun"; 223 case MLX5_HEALTH_SYNDR_EQ_ERR: 224 return "EQ error"; 225 case MLX5_HEALTH_SYNDR_EQ_INV: 226 return "Invalid EQ referenced"; 227 case MLX5_HEALTH_SYNDR_FFSER_ERR: 228 return "FFSER error"; 229 case MLX5_HEALTH_SYNDR_HIGH_TEMP: 230 return "High temperature"; 231 default: 232 return "unrecognized error"; 233 } 234 } 235 236 static void print_health_info(struct mlx5_core_dev *dev) 237 { 238 struct mlx5_core_health *health = &dev->priv.health; 239 struct health_buffer __iomem *h = health->health; 240 char fw_str[18]; 241 u32 fw; 242 int i; 243 244 /* If the syndrom is 0, the device is OK and no need to print buffer */ 245 if (!ioread8(&h->synd)) 246 return; 247 248 for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) 249 dev_err(&dev->pdev->dev, "assert_var[%d] 0x%08x\n", i, ioread32be(h->assert_var + i)); 250 251 dev_err(&dev->pdev->dev, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr)); 252 dev_err(&dev->pdev->dev, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra)); 253 sprintf(fw_str, "%d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev)); 254 dev_err(&dev->pdev->dev, "fw_ver %s\n", fw_str); 255 dev_err(&dev->pdev->dev, "hw_id 0x%08x\n", ioread32be(&h->hw_id)); 256 dev_err(&dev->pdev->dev, "irisc_index %d\n", ioread8(&h->irisc_index)); 257 dev_err(&dev->pdev->dev, "synd 0x%x: %s\n", ioread8(&h->synd), hsynd_str(ioread8(&h->synd))); 258 dev_err(&dev->pdev->dev, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd)); 259 fw = ioread32be(&h->fw_ver); 260 dev_err(&dev->pdev->dev, "raw fw_ver 0x%08x\n", fw); 261 } 262 263 static unsigned long get_next_poll_jiffies(void) 264 { 265 unsigned long next; 266 267 get_random_bytes(&next, sizeof(next)); 268 next %= HZ; 269 next += jiffies + MLX5_HEALTH_POLL_INTERVAL; 270 271 return next; 272 } 273 274 void mlx5_trigger_health_work(struct mlx5_core_dev *dev) 275 { 276 struct mlx5_core_health *health = &dev->priv.health; 277 unsigned long flags; 278 279 spin_lock_irqsave(&health->wq_lock, flags); 280 if (!test_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags)) 281 queue_work(health->wq, &health->work); 282 else 283 dev_err(&dev->pdev->dev, 284 "new health works are not permitted at this stage\n"); 285 spin_unlock_irqrestore(&health->wq_lock, flags); 286 } 287 288 static void poll_health(unsigned long data) 289 { 290 struct mlx5_core_dev *dev = (struct mlx5_core_dev *)data; 291 struct mlx5_core_health *health = &dev->priv.health; 292 u32 count; 293 294 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 295 goto out; 296 297 count = ioread32be(health->health_counter); 298 if (count == health->prev) 299 ++health->miss_counter; 300 else 301 health->miss_counter = 0; 302 303 health->prev = count; 304 if (health->miss_counter == MAX_MISSES) { 305 dev_err(&dev->pdev->dev, "device's health compromised - reached miss count\n"); 306 print_health_info(dev); 307 } 308 309 if (in_fatal(dev) && !health->sick) { 310 health->sick = true; 311 print_health_info(dev); 312 mlx5_trigger_health_work(dev); 313 } 314 315 out: 316 mod_timer(&health->timer, get_next_poll_jiffies()); 317 } 318 319 void mlx5_start_health_poll(struct mlx5_core_dev *dev) 320 { 321 struct mlx5_core_health *health = &dev->priv.health; 322 323 init_timer(&health->timer); 324 health->sick = 0; 325 clear_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 326 clear_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); 327 health->health = &dev->iseg->health; 328 health->health_counter = &dev->iseg->health_counter; 329 330 health->timer.data = (unsigned long)dev; 331 health->timer.function = poll_health; 332 health->timer.expires = round_jiffies(jiffies + MLX5_HEALTH_POLL_INTERVAL); 333 add_timer(&health->timer); 334 } 335 336 void mlx5_stop_health_poll(struct mlx5_core_dev *dev) 337 { 338 struct mlx5_core_health *health = &dev->priv.health; 339 340 del_timer_sync(&health->timer); 341 } 342 343 void mlx5_drain_health_wq(struct mlx5_core_dev *dev) 344 { 345 struct mlx5_core_health *health = &dev->priv.health; 346 unsigned long flags; 347 348 spin_lock_irqsave(&health->wq_lock, flags); 349 set_bit(MLX5_DROP_NEW_HEALTH_WORK, &health->flags); 350 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); 351 spin_unlock_irqrestore(&health->wq_lock, flags); 352 cancel_delayed_work_sync(&health->recover_work); 353 cancel_work_sync(&health->work); 354 } 355 356 void mlx5_drain_health_recovery(struct mlx5_core_dev *dev) 357 { 358 struct mlx5_core_health *health = &dev->priv.health; 359 360 spin_lock(&health->wq_lock); 361 set_bit(MLX5_DROP_NEW_RECOVERY_WORK, &health->flags); 362 spin_unlock(&health->wq_lock); 363 cancel_delayed_work_sync(&dev->priv.health.recover_work); 364 } 365 366 void mlx5_health_cleanup(struct mlx5_core_dev *dev) 367 { 368 struct mlx5_core_health *health = &dev->priv.health; 369 370 destroy_workqueue(health->wq); 371 } 372 373 int mlx5_health_init(struct mlx5_core_dev *dev) 374 { 375 struct mlx5_core_health *health; 376 char *name; 377 378 health = &dev->priv.health; 379 name = kmalloc(64, GFP_KERNEL); 380 if (!name) 381 return -ENOMEM; 382 383 strcpy(name, "mlx5_health"); 384 strcat(name, dev_name(&dev->pdev->dev)); 385 health->wq = create_singlethread_workqueue(name); 386 kfree(name); 387 if (!health->wq) 388 return -ENOMEM; 389 spin_lock_init(&health->wq_lock); 390 INIT_WORK(&health->work, health_care); 391 INIT_DELAYED_WORK(&health->recover_work, health_recover); 392 393 return 0; 394 } 395