1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright IBM Corporation 2001, 2005, 2006 4 * Copyright Dave Engebretsen & Todd Inglett 2001 5 * Copyright Linas Vepstas 2005, 2006 6 * Copyright 2001-2012 IBM Corporation. 7 * 8 * Please address comments and feedback to Linas Vepstas <linas@austin.ibm.com> 9 */ 10 11 #include <linux/delay.h> 12 #include <linux/sched.h> 13 #include <linux/init.h> 14 #include <linux/list.h> 15 #include <linux/pci.h> 16 #include <linux/iommu.h> 17 #include <linux/proc_fs.h> 18 #include <linux/rbtree.h> 19 #include <linux/reboot.h> 20 #include <linux/seq_file.h> 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 #include <linux/of.h> 24 #include <linux/debugfs.h> 25 26 #include <linux/atomic.h> 27 #include <asm/eeh.h> 28 #include <asm/eeh_event.h> 29 #include <asm/io.h> 30 #include <asm/iommu.h> 31 #include <asm/machdep.h> 32 #include <asm/ppc-pci.h> 33 #include <asm/rtas.h> 34 #include <asm/pte-walk.h> 35 36 37 /** Overview: 38 * EEH, or "Enhanced Error Handling" is a PCI bridge technology for 39 * dealing with PCI bus errors that can't be dealt with within the 40 * usual PCI framework, except by check-stopping the CPU. Systems 41 * that are designed for high-availability/reliability cannot afford 42 * to crash due to a "mere" PCI error, thus the need for EEH. 43 * An EEH-capable bridge operates by converting a detected error 44 * into a "slot freeze", taking the PCI adapter off-line, making 45 * the slot behave, from the OS'es point of view, as if the slot 46 * were "empty": all reads return 0xff's and all writes are silently 47 * ignored. EEH slot isolation events can be triggered by parity 48 * errors on the address or data busses (e.g. during posted writes), 49 * which in turn might be caused by low voltage on the bus, dust, 50 * vibration, humidity, radioactivity or plain-old failed hardware. 51 * 52 * Note, however, that one of the leading causes of EEH slot 53 * freeze events are buggy device drivers, buggy device microcode, 54 * or buggy device hardware. This is because any attempt by the 55 * device to bus-master data to a memory address that is not 56 * assigned to the device will trigger a slot freeze. (The idea 57 * is to prevent devices-gone-wild from corrupting system memory). 58 * Buggy hardware/drivers will have a miserable time co-existing 59 * with EEH. 60 * 61 * Ideally, a PCI device driver, when suspecting that an isolation 62 * event has occurred (e.g. by reading 0xff's), will then ask EEH 63 * whether this is the case, and then take appropriate steps to 64 * reset the PCI slot, the PCI device, and then resume operations. 65 * However, until that day, the checking is done here, with the 66 * eeh_check_failure() routine embedded in the MMIO macros. If 67 * the slot is found to be isolated, an "EEH Event" is synthesized 68 * and sent out for processing. 69 */ 70 71 /* If a device driver keeps reading an MMIO register in an interrupt 72 * handler after a slot isolation event, it might be broken. 73 * This sets the threshold for how many read attempts we allow 74 * before printing an error message. 75 */ 76 #define EEH_MAX_FAILS 2100000 77 78 /* Time to wait for a PCI slot to report status, in milliseconds */ 79 #define PCI_BUS_RESET_WAIT_MSEC (5*60*1000) 80 81 /* 82 * EEH probe mode support, which is part of the flags, 83 * is to support multiple platforms for EEH. Some platforms 84 * like pSeries do PCI emunation based on device tree. 85 * However, other platforms like powernv probe PCI devices 86 * from hardware. The flag is used to distinguish that. 87 * In addition, struct eeh_ops::probe would be invoked for 88 * particular OF node or PCI device so that the corresponding 89 * PE would be created there. 90 */ 91 int eeh_subsystem_flags; 92 EXPORT_SYMBOL(eeh_subsystem_flags); 93 94 /* 95 * EEH allowed maximal frozen times. If one particular PE's 96 * frozen count in last hour exceeds this limit, the PE will 97 * be forced to be offline permanently. 98 */ 99 u32 eeh_max_freezes = 5; 100 101 /* 102 * Controls whether a recovery event should be scheduled when an 103 * isolated device is discovered. This is only really useful for 104 * debugging problems with the EEH core. 105 */ 106 bool eeh_debugfs_no_recover; 107 108 /* Platform dependent EEH operations */ 109 struct eeh_ops *eeh_ops = NULL; 110 111 /* Lock to avoid races due to multiple reports of an error */ 112 DEFINE_RAW_SPINLOCK(confirm_error_lock); 113 EXPORT_SYMBOL_GPL(confirm_error_lock); 114 115 /* Lock to protect passed flags */ 116 static DEFINE_MUTEX(eeh_dev_mutex); 117 118 /* Buffer for reporting pci register dumps. Its here in BSS, and 119 * not dynamically alloced, so that it ends up in RMO where RTAS 120 * can access it. 121 */ 122 #define EEH_PCI_REGS_LOG_LEN 8192 123 static unsigned char pci_regs_buf[EEH_PCI_REGS_LOG_LEN]; 124 125 /* 126 * The struct is used to maintain the EEH global statistic 127 * information. Besides, the EEH global statistics will be 128 * exported to user space through procfs 129 */ 130 struct eeh_stats { 131 u64 no_device; /* PCI device not found */ 132 u64 no_dn; /* OF node not found */ 133 u64 no_cfg_addr; /* Config address not found */ 134 u64 ignored_check; /* EEH check skipped */ 135 u64 total_mmio_ffs; /* Total EEH checks */ 136 u64 false_positives; /* Unnecessary EEH checks */ 137 u64 slot_resets; /* PE reset */ 138 }; 139 140 static struct eeh_stats eeh_stats; 141 142 static int __init eeh_setup(char *str) 143 { 144 if (!strcmp(str, "off")) 145 eeh_add_flag(EEH_FORCE_DISABLED); 146 else if (!strcmp(str, "early_log")) 147 eeh_add_flag(EEH_EARLY_DUMP_LOG); 148 149 return 1; 150 } 151 __setup("eeh=", eeh_setup); 152 153 void eeh_show_enabled(void) 154 { 155 if (eeh_has_flag(EEH_FORCE_DISABLED)) 156 pr_info("EEH: Recovery disabled by kernel parameter.\n"); 157 else if (eeh_has_flag(EEH_ENABLED)) 158 pr_info("EEH: Capable adapter found: recovery enabled.\n"); 159 else 160 pr_info("EEH: No capable adapters found: recovery disabled.\n"); 161 } 162 163 /* 164 * This routine captures assorted PCI configuration space data 165 * for the indicated PCI device, and puts them into a buffer 166 * for RTAS error logging. 167 */ 168 static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) 169 { 170 u32 cfg; 171 int cap, i; 172 int n = 0, l = 0; 173 char buffer[128]; 174 175 n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n", 176 edev->pe->phb->global_number, edev->bdfn >> 8, 177 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 178 pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n", 179 edev->pe->phb->global_number, edev->bdfn >> 8, 180 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 181 182 eeh_ops->read_config(edev, PCI_VENDOR_ID, 4, &cfg); 183 n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg); 184 pr_warn("EEH: PCI device/vendor: %08x\n", cfg); 185 186 eeh_ops->read_config(edev, PCI_COMMAND, 4, &cfg); 187 n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg); 188 pr_warn("EEH: PCI cmd/status register: %08x\n", cfg); 189 190 /* Gather bridge-specific registers */ 191 if (edev->mode & EEH_DEV_BRIDGE) { 192 eeh_ops->read_config(edev, PCI_SEC_STATUS, 2, &cfg); 193 n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg); 194 pr_warn("EEH: Bridge secondary status: %04x\n", cfg); 195 196 eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &cfg); 197 n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg); 198 pr_warn("EEH: Bridge control: %04x\n", cfg); 199 } 200 201 /* Dump out the PCI-X command and status regs */ 202 cap = edev->pcix_cap; 203 if (cap) { 204 eeh_ops->read_config(edev, cap, 4, &cfg); 205 n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg); 206 pr_warn("EEH: PCI-X cmd: %08x\n", cfg); 207 208 eeh_ops->read_config(edev, cap+4, 4, &cfg); 209 n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg); 210 pr_warn("EEH: PCI-X status: %08x\n", cfg); 211 } 212 213 /* If PCI-E capable, dump PCI-E cap 10 */ 214 cap = edev->pcie_cap; 215 if (cap) { 216 n += scnprintf(buf+n, len-n, "pci-e cap10:\n"); 217 pr_warn("EEH: PCI-E capabilities and status follow:\n"); 218 219 for (i=0; i<=8; i++) { 220 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 221 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 222 223 if ((i % 4) == 0) { 224 if (i != 0) 225 pr_warn("%s\n", buffer); 226 227 l = scnprintf(buffer, sizeof(buffer), 228 "EEH: PCI-E %02x: %08x ", 229 4*i, cfg); 230 } else { 231 l += scnprintf(buffer+l, sizeof(buffer)-l, 232 "%08x ", cfg); 233 } 234 235 } 236 237 pr_warn("%s\n", buffer); 238 } 239 240 /* If AER capable, dump it */ 241 cap = edev->aer_cap; 242 if (cap) { 243 n += scnprintf(buf+n, len-n, "pci-e AER:\n"); 244 pr_warn("EEH: PCI-E AER capability register set follows:\n"); 245 246 for (i=0; i<=13; i++) { 247 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 248 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 249 250 if ((i % 4) == 0) { 251 if (i != 0) 252 pr_warn("%s\n", buffer); 253 254 l = scnprintf(buffer, sizeof(buffer), 255 "EEH: PCI-E AER %02x: %08x ", 256 4*i, cfg); 257 } else { 258 l += scnprintf(buffer+l, sizeof(buffer)-l, 259 "%08x ", cfg); 260 } 261 } 262 263 pr_warn("%s\n", buffer); 264 } 265 266 return n; 267 } 268 269 static void *eeh_dump_pe_log(struct eeh_pe *pe, void *flag) 270 { 271 struct eeh_dev *edev, *tmp; 272 size_t *plen = flag; 273 274 eeh_pe_for_each_dev(pe, edev, tmp) 275 *plen += eeh_dump_dev_log(edev, pci_regs_buf + *plen, 276 EEH_PCI_REGS_LOG_LEN - *plen); 277 278 return NULL; 279 } 280 281 /** 282 * eeh_slot_error_detail - Generate combined log including driver log and error log 283 * @pe: EEH PE 284 * @severity: temporary or permanent error log 285 * 286 * This routine should be called to generate the combined log, which 287 * is comprised of driver log and error log. The driver log is figured 288 * out from the config space of the corresponding PCI device, while 289 * the error log is fetched through platform dependent function call. 290 */ 291 void eeh_slot_error_detail(struct eeh_pe *pe, int severity) 292 { 293 size_t loglen = 0; 294 295 /* 296 * When the PHB is fenced or dead, it's pointless to collect 297 * the data from PCI config space because it should return 298 * 0xFF's. For ER, we still retrieve the data from the PCI 299 * config space. 300 * 301 * For pHyp, we have to enable IO for log retrieval. Otherwise, 302 * 0xFF's is always returned from PCI config space. 303 * 304 * When the @severity is EEH_LOG_PERM, the PE is going to be 305 * removed. Prior to that, the drivers for devices included in 306 * the PE will be closed. The drivers rely on working IO path 307 * to bring the devices to quiet state. Otherwise, PCI traffic 308 * from those devices after they are removed is like to cause 309 * another unexpected EEH error. 310 */ 311 if (!(pe->type & EEH_PE_PHB)) { 312 if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG) || 313 severity == EEH_LOG_PERM) 314 eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 315 316 /* 317 * The config space of some PCI devices can't be accessed 318 * when their PEs are in frozen state. Otherwise, fenced 319 * PHB might be seen. Those PEs are identified with flag 320 * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED 321 * is set automatically when the PE is put to EEH_PE_ISOLATED. 322 * 323 * Restoring BARs possibly triggers PCI config access in 324 * (OPAL) firmware and then causes fenced PHB. If the 325 * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's 326 * pointless to restore BARs and dump config space. 327 */ 328 eeh_ops->configure_bridge(pe); 329 if (!(pe->state & EEH_PE_CFG_BLOCKED)) { 330 eeh_pe_restore_bars(pe); 331 332 pci_regs_buf[0] = 0; 333 eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen); 334 } 335 } 336 337 eeh_ops->get_log(pe, severity, pci_regs_buf, loglen); 338 } 339 340 /** 341 * eeh_token_to_phys - Convert EEH address token to phys address 342 * @token: I/O token, should be address in the form 0xA.... 343 * 344 * This routine should be called to convert virtual I/O address 345 * to physical one. 346 */ 347 static inline unsigned long eeh_token_to_phys(unsigned long token) 348 { 349 return ppc_find_vmap_phys(token); 350 } 351 352 /* 353 * On PowerNV platform, we might already have fenced PHB there. 354 * For that case, it's meaningless to recover frozen PE. Intead, 355 * We have to handle fenced PHB firstly. 356 */ 357 static int eeh_phb_check_failure(struct eeh_pe *pe) 358 { 359 struct eeh_pe *phb_pe; 360 unsigned long flags; 361 int ret; 362 363 if (!eeh_has_flag(EEH_PROBE_MODE_DEV)) 364 return -EPERM; 365 366 /* Find the PHB PE */ 367 phb_pe = eeh_phb_pe_get(pe->phb); 368 if (!phb_pe) { 369 pr_warn("%s Can't find PE for PHB#%x\n", 370 __func__, pe->phb->global_number); 371 return -EEXIST; 372 } 373 374 /* If the PHB has been in problematic state */ 375 eeh_serialize_lock(&flags); 376 if (phb_pe->state & EEH_PE_ISOLATED) { 377 ret = 0; 378 goto out; 379 } 380 381 /* Check PHB state */ 382 ret = eeh_ops->get_state(phb_pe, NULL); 383 if ((ret < 0) || 384 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 385 ret = 0; 386 goto out; 387 } 388 389 /* Isolate the PHB and send event */ 390 eeh_pe_mark_isolated(phb_pe); 391 eeh_serialize_unlock(flags); 392 393 pr_debug("EEH: PHB#%x failure detected, location: %s\n", 394 phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe)); 395 eeh_send_failure_event(phb_pe); 396 return 1; 397 out: 398 eeh_serialize_unlock(flags); 399 return ret; 400 } 401 402 /** 403 * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze 404 * @edev: eeh device 405 * 406 * Check for an EEH failure for the given device node. Call this 407 * routine if the result of a read was all 0xff's and you want to 408 * find out if this is due to an EEH slot freeze. This routine 409 * will query firmware for the EEH status. 410 * 411 * Returns 0 if there has not been an EEH error; otherwise returns 412 * a non-zero value and queues up a slot isolation event notification. 413 * 414 * It is safe to call this routine in an interrupt context. 415 */ 416 int eeh_dev_check_failure(struct eeh_dev *edev) 417 { 418 int ret; 419 unsigned long flags; 420 struct device_node *dn; 421 struct pci_dev *dev; 422 struct eeh_pe *pe, *parent_pe; 423 int rc = 0; 424 const char *location = NULL; 425 426 eeh_stats.total_mmio_ffs++; 427 428 if (!eeh_enabled()) 429 return 0; 430 431 if (!edev) { 432 eeh_stats.no_dn++; 433 return 0; 434 } 435 dev = eeh_dev_to_pci_dev(edev); 436 pe = eeh_dev_to_pe(edev); 437 438 /* Access to IO BARs might get this far and still not want checking. */ 439 if (!pe) { 440 eeh_stats.ignored_check++; 441 eeh_edev_dbg(edev, "Ignored check\n"); 442 return 0; 443 } 444 445 /* 446 * On PowerNV platform, we might already have fenced PHB 447 * there and we need take care of that firstly. 448 */ 449 ret = eeh_phb_check_failure(pe); 450 if (ret > 0) 451 return ret; 452 453 /* 454 * If the PE isn't owned by us, we shouldn't check the 455 * state. Instead, let the owner handle it if the PE has 456 * been frozen. 457 */ 458 if (eeh_pe_passed(pe)) 459 return 0; 460 461 /* If we already have a pending isolation event for this 462 * slot, we know it's bad already, we don't need to check. 463 * Do this checking under a lock; as multiple PCI devices 464 * in one slot might report errors simultaneously, and we 465 * only want one error recovery routine running. 466 */ 467 eeh_serialize_lock(&flags); 468 rc = 1; 469 if (pe->state & EEH_PE_ISOLATED) { 470 pe->check_count++; 471 if (pe->check_count == EEH_MAX_FAILS) { 472 dn = pci_device_to_OF_node(dev); 473 if (dn) 474 location = of_get_property(dn, "ibm,loc-code", 475 NULL); 476 eeh_edev_err(edev, "%d reads ignored for recovering device at location=%s driver=%s\n", 477 pe->check_count, 478 location ? location : "unknown", 479 eeh_driver_name(dev)); 480 eeh_edev_err(edev, "Might be infinite loop in %s driver\n", 481 eeh_driver_name(dev)); 482 dump_stack(); 483 } 484 goto dn_unlock; 485 } 486 487 /* 488 * Now test for an EEH failure. This is VERY expensive. 489 * Note that the eeh_config_addr may be a parent device 490 * in the case of a device behind a bridge, or it may be 491 * function zero of a multi-function device. 492 * In any case they must share a common PHB. 493 */ 494 ret = eeh_ops->get_state(pe, NULL); 495 496 /* Note that config-io to empty slots may fail; 497 * they are empty when they don't have children. 498 * We will punt with the following conditions: Failure to get 499 * PE's state, EEH not support and Permanently unavailable 500 * state, PE is in good state. 501 */ 502 if ((ret < 0) || 503 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 504 eeh_stats.false_positives++; 505 pe->false_positives++; 506 rc = 0; 507 goto dn_unlock; 508 } 509 510 /* 511 * It should be corner case that the parent PE has been 512 * put into frozen state as well. We should take care 513 * that at first. 514 */ 515 parent_pe = pe->parent; 516 while (parent_pe) { 517 /* Hit the ceiling ? */ 518 if (parent_pe->type & EEH_PE_PHB) 519 break; 520 521 /* Frozen parent PE ? */ 522 ret = eeh_ops->get_state(parent_pe, NULL); 523 if (ret > 0 && !eeh_state_active(ret)) { 524 pe = parent_pe; 525 pr_err("EEH: Failure of PHB#%x-PE#%x will be handled at parent PHB#%x-PE#%x.\n", 526 pe->phb->global_number, pe->addr, 527 pe->phb->global_number, parent_pe->addr); 528 } 529 530 /* Next parent level */ 531 parent_pe = parent_pe->parent; 532 } 533 534 eeh_stats.slot_resets++; 535 536 /* Avoid repeated reports of this failure, including problems 537 * with other functions on this device, and functions under 538 * bridges. 539 */ 540 eeh_pe_mark_isolated(pe); 541 eeh_serialize_unlock(flags); 542 543 /* Most EEH events are due to device driver bugs. Having 544 * a stack trace will help the device-driver authors figure 545 * out what happened. So print that out. 546 */ 547 pr_debug("EEH: %s: Frozen PHB#%x-PE#%x detected\n", 548 __func__, pe->phb->global_number, pe->addr); 549 eeh_send_failure_event(pe); 550 551 return 1; 552 553 dn_unlock: 554 eeh_serialize_unlock(flags); 555 return rc; 556 } 557 558 EXPORT_SYMBOL_GPL(eeh_dev_check_failure); 559 560 /** 561 * eeh_check_failure - Check if all 1's data is due to EEH slot freeze 562 * @token: I/O address 563 * 564 * Check for an EEH failure at the given I/O address. Call this 565 * routine if the result of a read was all 0xff's and you want to 566 * find out if this is due to an EEH slot freeze event. This routine 567 * will query firmware for the EEH status. 568 * 569 * Note this routine is safe to call in an interrupt context. 570 */ 571 int eeh_check_failure(const volatile void __iomem *token) 572 { 573 unsigned long addr; 574 struct eeh_dev *edev; 575 576 /* Finding the phys addr + pci device; this is pretty quick. */ 577 addr = eeh_token_to_phys((unsigned long __force) token); 578 edev = eeh_addr_cache_get_dev(addr); 579 if (!edev) { 580 eeh_stats.no_device++; 581 return 0; 582 } 583 584 return eeh_dev_check_failure(edev); 585 } 586 EXPORT_SYMBOL(eeh_check_failure); 587 588 589 /** 590 * eeh_pci_enable - Enable MMIO or DMA transfers for this slot 591 * @pe: EEH PE 592 * 593 * This routine should be called to reenable frozen MMIO or DMA 594 * so that it would work correctly again. It's useful while doing 595 * recovery or log collection on the indicated device. 596 */ 597 int eeh_pci_enable(struct eeh_pe *pe, int function) 598 { 599 int active_flag, rc; 600 601 /* 602 * pHyp doesn't allow to enable IO or DMA on unfrozen PE. 603 * Also, it's pointless to enable them on unfrozen PE. So 604 * we have to check before enabling IO or DMA. 605 */ 606 switch (function) { 607 case EEH_OPT_THAW_MMIO: 608 active_flag = EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED; 609 break; 610 case EEH_OPT_THAW_DMA: 611 active_flag = EEH_STATE_DMA_ACTIVE; 612 break; 613 case EEH_OPT_DISABLE: 614 case EEH_OPT_ENABLE: 615 case EEH_OPT_FREEZE_PE: 616 active_flag = 0; 617 break; 618 default: 619 pr_warn("%s: Invalid function %d\n", 620 __func__, function); 621 return -EINVAL; 622 } 623 624 /* 625 * Check if IO or DMA has been enabled before 626 * enabling them. 627 */ 628 if (active_flag) { 629 rc = eeh_ops->get_state(pe, NULL); 630 if (rc < 0) 631 return rc; 632 633 /* Needn't enable it at all */ 634 if (rc == EEH_STATE_NOT_SUPPORT) 635 return 0; 636 637 /* It's already enabled */ 638 if (rc & active_flag) 639 return 0; 640 } 641 642 643 /* Issue the request */ 644 rc = eeh_ops->set_option(pe, function); 645 if (rc) 646 pr_warn("%s: Unexpected state change %d on " 647 "PHB#%x-PE#%x, err=%d\n", 648 __func__, function, pe->phb->global_number, 649 pe->addr, rc); 650 651 /* Check if the request is finished successfully */ 652 if (active_flag) { 653 rc = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 654 if (rc < 0) 655 return rc; 656 657 if (rc & active_flag) 658 return 0; 659 660 return -EIO; 661 } 662 663 return rc; 664 } 665 666 static void eeh_disable_and_save_dev_state(struct eeh_dev *edev, 667 void *userdata) 668 { 669 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 670 struct pci_dev *dev = userdata; 671 672 /* 673 * The caller should have disabled and saved the 674 * state for the specified device 675 */ 676 if (!pdev || pdev == dev) 677 return; 678 679 /* Ensure we have D0 power state */ 680 pci_set_power_state(pdev, PCI_D0); 681 682 /* Save device state */ 683 pci_save_state(pdev); 684 685 /* 686 * Disable device to avoid any DMA traffic and 687 * interrupt from the device 688 */ 689 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 690 } 691 692 static void eeh_restore_dev_state(struct eeh_dev *edev, void *userdata) 693 { 694 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 695 struct pci_dev *dev = userdata; 696 697 if (!pdev) 698 return; 699 700 /* Apply customization from firmware */ 701 if (eeh_ops->restore_config) 702 eeh_ops->restore_config(edev); 703 704 /* The caller should restore state for the specified device */ 705 if (pdev != dev) 706 pci_restore_state(pdev); 707 } 708 709 /** 710 * pcibios_set_pcie_reset_state - Set PCI-E reset state 711 * @dev: pci device struct 712 * @state: reset state to enter 713 * 714 * Return value: 715 * 0 if success 716 */ 717 int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 718 { 719 struct eeh_dev *edev = pci_dev_to_eeh_dev(dev); 720 struct eeh_pe *pe = eeh_dev_to_pe(edev); 721 722 if (!pe) { 723 pr_err("%s: No PE found on PCI device %s\n", 724 __func__, pci_name(dev)); 725 return -EINVAL; 726 } 727 728 switch (state) { 729 case pcie_deassert_reset: 730 eeh_ops->reset(pe, EEH_RESET_DEACTIVATE); 731 eeh_unfreeze_pe(pe); 732 if (!(pe->type & EEH_PE_VF)) 733 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 734 eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev); 735 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 736 break; 737 case pcie_hot_reset: 738 eeh_pe_mark_isolated(pe); 739 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 740 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 741 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 742 if (!(pe->type & EEH_PE_VF)) 743 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 744 eeh_ops->reset(pe, EEH_RESET_HOT); 745 break; 746 case pcie_warm_reset: 747 eeh_pe_mark_isolated(pe); 748 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 749 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 750 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 751 if (!(pe->type & EEH_PE_VF)) 752 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 753 eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL); 754 break; 755 default: 756 eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true); 757 return -EINVAL; 758 } 759 760 return 0; 761 } 762 763 /** 764 * eeh_set_pe_freset - Check the required reset for the indicated device 765 * @data: EEH device 766 * @flag: return value 767 * 768 * Each device might have its preferred reset type: fundamental or 769 * hot reset. The routine is used to collected the information for 770 * the indicated device and its children so that the bunch of the 771 * devices could be reset properly. 772 */ 773 static void eeh_set_dev_freset(struct eeh_dev *edev, void *flag) 774 { 775 struct pci_dev *dev; 776 unsigned int *freset = (unsigned int *)flag; 777 778 dev = eeh_dev_to_pci_dev(edev); 779 if (dev) 780 *freset |= dev->needs_freset; 781 } 782 783 static void eeh_pe_refreeze_passed(struct eeh_pe *root) 784 { 785 struct eeh_pe *pe; 786 int state; 787 788 eeh_for_each_pe(root, pe) { 789 if (eeh_pe_passed(pe)) { 790 state = eeh_ops->get_state(pe, NULL); 791 if (state & 792 (EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED)) { 793 pr_info("EEH: Passed-through PE PHB#%x-PE#%x was thawed by reset, re-freezing for safety.\n", 794 pe->phb->global_number, pe->addr); 795 eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE); 796 } 797 } 798 } 799 } 800 801 /** 802 * eeh_pe_reset_full - Complete a full reset process on the indicated PE 803 * @pe: EEH PE 804 * 805 * This function executes a full reset procedure on a PE, including setting 806 * the appropriate flags, performing a fundamental or hot reset, and then 807 * deactivating the reset status. It is designed to be used within the EEH 808 * subsystem, as opposed to eeh_pe_reset which is exported to drivers and 809 * only performs a single operation at a time. 810 * 811 * This function will attempt to reset a PE three times before failing. 812 */ 813 int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed) 814 { 815 int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED); 816 int type = EEH_RESET_HOT; 817 unsigned int freset = 0; 818 int i, state = 0, ret; 819 820 /* 821 * Determine the type of reset to perform - hot or fundamental. 822 * Hot reset is the default operation, unless any device under the 823 * PE requires a fundamental reset. 824 */ 825 eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset); 826 827 if (freset) 828 type = EEH_RESET_FUNDAMENTAL; 829 830 /* Mark the PE as in reset state and block config space accesses */ 831 eeh_pe_state_mark(pe, reset_state); 832 833 /* Make three attempts at resetting the bus */ 834 for (i = 0; i < 3; i++) { 835 ret = eeh_pe_reset(pe, type, include_passed); 836 if (!ret) 837 ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, 838 include_passed); 839 if (ret) { 840 ret = -EIO; 841 pr_warn("EEH: Failure %d resetting PHB#%x-PE#%x (attempt %d)\n\n", 842 state, pe->phb->global_number, pe->addr, i + 1); 843 continue; 844 } 845 if (i) 846 pr_warn("EEH: PHB#%x-PE#%x: Successful reset (attempt %d)\n", 847 pe->phb->global_number, pe->addr, i + 1); 848 849 /* Wait until the PE is in a functioning state */ 850 state = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 851 if (state < 0) { 852 pr_warn("EEH: Unrecoverable slot failure on PHB#%x-PE#%x", 853 pe->phb->global_number, pe->addr); 854 ret = -ENOTRECOVERABLE; 855 break; 856 } 857 if (eeh_state_active(state)) 858 break; 859 else 860 pr_warn("EEH: PHB#%x-PE#%x: Slot inactive after reset: 0x%x (attempt %d)\n", 861 pe->phb->global_number, pe->addr, state, i + 1); 862 } 863 864 /* Resetting the PE may have unfrozen child PEs. If those PEs have been 865 * (potentially) passed through to a guest, re-freeze them: 866 */ 867 if (!include_passed) 868 eeh_pe_refreeze_passed(pe); 869 870 eeh_pe_state_clear(pe, reset_state, true); 871 return ret; 872 } 873 874 /** 875 * eeh_save_bars - Save device bars 876 * @edev: PCI device associated EEH device 877 * 878 * Save the values of the device bars. Unlike the restore 879 * routine, this routine is *not* recursive. This is because 880 * PCI devices are added individually; but, for the restore, 881 * an entire slot is reset at a time. 882 */ 883 void eeh_save_bars(struct eeh_dev *edev) 884 { 885 int i; 886 887 if (!edev) 888 return; 889 890 for (i = 0; i < 16; i++) 891 eeh_ops->read_config(edev, i * 4, 4, &edev->config_space[i]); 892 893 /* 894 * For PCI bridges including root port, we need enable bus 895 * master explicitly. Otherwise, it can't fetch IODA table 896 * entries correctly. So we cache the bit in advance so that 897 * we can restore it after reset, either PHB range or PE range. 898 */ 899 if (edev->mode & EEH_DEV_BRIDGE) 900 edev->config_space[1] |= PCI_COMMAND_MASTER; 901 } 902 903 static int eeh_reboot_notifier(struct notifier_block *nb, 904 unsigned long action, void *unused) 905 { 906 eeh_clear_flag(EEH_ENABLED); 907 return NOTIFY_DONE; 908 } 909 910 static struct notifier_block eeh_reboot_nb = { 911 .notifier_call = eeh_reboot_notifier, 912 }; 913 914 static int eeh_device_notifier(struct notifier_block *nb, 915 unsigned long action, void *data) 916 { 917 struct device *dev = data; 918 919 switch (action) { 920 /* 921 * Note: It's not possible to perform EEH device addition (i.e. 922 * {pseries,pnv}_pcibios_bus_add_device()) here because it depends on 923 * the device's resources, which have not yet been set up. 924 */ 925 case BUS_NOTIFY_DEL_DEVICE: 926 eeh_remove_device(to_pci_dev(dev)); 927 break; 928 default: 929 break; 930 } 931 return NOTIFY_DONE; 932 } 933 934 static struct notifier_block eeh_device_nb = { 935 .notifier_call = eeh_device_notifier, 936 }; 937 938 /** 939 * eeh_init - System wide EEH initialization 940 * 941 * It's the platform's job to call this from an arch_initcall(). 942 */ 943 int eeh_init(struct eeh_ops *ops) 944 { 945 struct pci_controller *hose, *tmp; 946 int ret = 0; 947 948 /* the platform should only initialise EEH once */ 949 if (WARN_ON(eeh_ops)) 950 return -EEXIST; 951 if (WARN_ON(!ops)) 952 return -ENOENT; 953 eeh_ops = ops; 954 955 /* Register reboot notifier */ 956 ret = register_reboot_notifier(&eeh_reboot_nb); 957 if (ret) { 958 pr_warn("%s: Failed to register reboot notifier (%d)\n", 959 __func__, ret); 960 return ret; 961 } 962 963 ret = bus_register_notifier(&pci_bus_type, &eeh_device_nb); 964 if (ret) { 965 pr_warn("%s: Failed to register bus notifier (%d)\n", 966 __func__, ret); 967 return ret; 968 } 969 970 /* Initialize PHB PEs */ 971 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) 972 eeh_phb_pe_create(hose); 973 974 eeh_addr_cache_init(); 975 976 /* Initialize EEH event */ 977 return eeh_event_init(); 978 } 979 980 /** 981 * eeh_probe_device() - Perform EEH initialization for the indicated pci device 982 * @dev: pci device for which to set up EEH 983 * 984 * This routine must be used to complete EEH initialization for PCI 985 * devices that were added after system boot (e.g. hotplug, dlpar). 986 */ 987 void eeh_probe_device(struct pci_dev *dev) 988 { 989 struct eeh_dev *edev; 990 991 pr_debug("EEH: Adding device %s\n", pci_name(dev)); 992 993 /* 994 * pci_dev_to_eeh_dev() can only work if eeh_probe_dev() was 995 * already called for this device. 996 */ 997 if (WARN_ON_ONCE(pci_dev_to_eeh_dev(dev))) { 998 pci_dbg(dev, "Already bound to an eeh_dev!\n"); 999 return; 1000 } 1001 1002 edev = eeh_ops->probe(dev); 1003 if (!edev) { 1004 pr_debug("EEH: Adding device failed\n"); 1005 return; 1006 } 1007 1008 /* 1009 * FIXME: We rely on pcibios_release_device() to remove the 1010 * existing EEH state. The release function is only called if 1011 * the pci_dev's refcount drops to zero so if something is 1012 * keeping a ref to a device (e.g. a filesystem) we need to 1013 * remove the old EEH state. 1014 * 1015 * FIXME: HEY MA, LOOK AT ME, NO LOCKING! 1016 */ 1017 if (edev->pdev && edev->pdev != dev) { 1018 eeh_pe_tree_remove(edev); 1019 eeh_addr_cache_rmv_dev(edev->pdev); 1020 eeh_sysfs_remove_device(edev->pdev); 1021 1022 /* 1023 * We definitely should have the PCI device removed 1024 * though it wasn't correctly. So we needn't call 1025 * into error handler afterwards. 1026 */ 1027 edev->mode |= EEH_DEV_NO_HANDLER; 1028 } 1029 1030 /* bind the pdev and the edev together */ 1031 edev->pdev = dev; 1032 dev->dev.archdata.edev = edev; 1033 eeh_addr_cache_insert_dev(dev); 1034 eeh_sysfs_add_device(dev); 1035 } 1036 1037 /** 1038 * eeh_remove_device - Undo EEH setup for the indicated pci device 1039 * @dev: pci device to be removed 1040 * 1041 * This routine should be called when a device is removed from 1042 * a running system (e.g. by hotplug or dlpar). It unregisters 1043 * the PCI device from the EEH subsystem. I/O errors affecting 1044 * this device will no longer be detected after this call; thus, 1045 * i/o errors affecting this slot may leave this device unusable. 1046 */ 1047 void eeh_remove_device(struct pci_dev *dev) 1048 { 1049 struct eeh_dev *edev; 1050 1051 if (!dev || !eeh_enabled()) 1052 return; 1053 edev = pci_dev_to_eeh_dev(dev); 1054 1055 /* Unregister the device with the EEH/PCI address search system */ 1056 dev_dbg(&dev->dev, "EEH: Removing device\n"); 1057 1058 if (!edev || !edev->pdev || !edev->pe) { 1059 dev_dbg(&dev->dev, "EEH: Device not referenced!\n"); 1060 return; 1061 } 1062 1063 /* 1064 * During the hotplug for EEH error recovery, we need the EEH 1065 * device attached to the parent PE in order for BAR restore 1066 * a bit later. So we keep it for BAR restore and remove it 1067 * from the parent PE during the BAR resotre. 1068 */ 1069 edev->pdev = NULL; 1070 1071 /* 1072 * eeh_sysfs_remove_device() uses pci_dev_to_eeh_dev() so we need to 1073 * remove the sysfs files before clearing dev.archdata.edev 1074 */ 1075 if (edev->mode & EEH_DEV_SYSFS) 1076 eeh_sysfs_remove_device(dev); 1077 1078 /* 1079 * We're removing from the PCI subsystem, that means 1080 * the PCI device driver can't support EEH or not 1081 * well. So we rely on hotplug completely to do recovery 1082 * for the specific PCI device. 1083 */ 1084 edev->mode |= EEH_DEV_NO_HANDLER; 1085 1086 eeh_addr_cache_rmv_dev(dev); 1087 1088 /* 1089 * The flag "in_error" is used to trace EEH devices for VFs 1090 * in error state or not. It's set in eeh_report_error(). If 1091 * it's not set, eeh_report_{reset,resume}() won't be called 1092 * for the VF EEH device. 1093 */ 1094 edev->in_error = false; 1095 dev->dev.archdata.edev = NULL; 1096 if (!(edev->pe->state & EEH_PE_KEEP)) 1097 eeh_pe_tree_remove(edev); 1098 else 1099 edev->mode |= EEH_DEV_DISCONNECTED; 1100 } 1101 1102 int eeh_unfreeze_pe(struct eeh_pe *pe) 1103 { 1104 int ret; 1105 1106 ret = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 1107 if (ret) { 1108 pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n", 1109 __func__, ret, pe->phb->global_number, pe->addr); 1110 return ret; 1111 } 1112 1113 ret = eeh_pci_enable(pe, EEH_OPT_THAW_DMA); 1114 if (ret) { 1115 pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n", 1116 __func__, ret, pe->phb->global_number, pe->addr); 1117 return ret; 1118 } 1119 1120 return ret; 1121 } 1122 1123 1124 static struct pci_device_id eeh_reset_ids[] = { 1125 { PCI_DEVICE(0x19a2, 0x0710) }, /* Emulex, BE */ 1126 { PCI_DEVICE(0x10df, 0xe220) }, /* Emulex, Lancer */ 1127 { PCI_DEVICE(0x14e4, 0x1657) }, /* Broadcom BCM5719 */ 1128 { 0 } 1129 }; 1130 1131 static int eeh_pe_change_owner(struct eeh_pe *pe) 1132 { 1133 struct eeh_dev *edev, *tmp; 1134 struct pci_dev *pdev; 1135 struct pci_device_id *id; 1136 int ret; 1137 1138 /* Check PE state */ 1139 ret = eeh_ops->get_state(pe, NULL); 1140 if (ret < 0 || ret == EEH_STATE_NOT_SUPPORT) 1141 return 0; 1142 1143 /* Unfrozen PE, nothing to do */ 1144 if (eeh_state_active(ret)) 1145 return 0; 1146 1147 /* Frozen PE, check if it needs PE level reset */ 1148 eeh_pe_for_each_dev(pe, edev, tmp) { 1149 pdev = eeh_dev_to_pci_dev(edev); 1150 if (!pdev) 1151 continue; 1152 1153 for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) { 1154 if (id->vendor != PCI_ANY_ID && 1155 id->vendor != pdev->vendor) 1156 continue; 1157 if (id->device != PCI_ANY_ID && 1158 id->device != pdev->device) 1159 continue; 1160 if (id->subvendor != PCI_ANY_ID && 1161 id->subvendor != pdev->subsystem_vendor) 1162 continue; 1163 if (id->subdevice != PCI_ANY_ID && 1164 id->subdevice != pdev->subsystem_device) 1165 continue; 1166 1167 return eeh_pe_reset_and_recover(pe); 1168 } 1169 } 1170 1171 ret = eeh_unfreeze_pe(pe); 1172 if (!ret) 1173 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 1174 return ret; 1175 } 1176 1177 /** 1178 * eeh_dev_open - Increase count of pass through devices for PE 1179 * @pdev: PCI device 1180 * 1181 * Increase count of passed through devices for the indicated 1182 * PE. In the result, the EEH errors detected on the PE won't be 1183 * reported. The PE owner will be responsible for detection 1184 * and recovery. 1185 */ 1186 int eeh_dev_open(struct pci_dev *pdev) 1187 { 1188 struct eeh_dev *edev; 1189 int ret = -ENODEV; 1190 1191 mutex_lock(&eeh_dev_mutex); 1192 1193 /* No PCI device ? */ 1194 if (!pdev) 1195 goto out; 1196 1197 /* No EEH device or PE ? */ 1198 edev = pci_dev_to_eeh_dev(pdev); 1199 if (!edev || !edev->pe) 1200 goto out; 1201 1202 /* 1203 * The PE might have been put into frozen state, but we 1204 * didn't detect that yet. The passed through PCI devices 1205 * in frozen PE won't work properly. Clear the frozen state 1206 * in advance. 1207 */ 1208 ret = eeh_pe_change_owner(edev->pe); 1209 if (ret) 1210 goto out; 1211 1212 /* Increase PE's pass through count */ 1213 atomic_inc(&edev->pe->pass_dev_cnt); 1214 mutex_unlock(&eeh_dev_mutex); 1215 1216 return 0; 1217 out: 1218 mutex_unlock(&eeh_dev_mutex); 1219 return ret; 1220 } 1221 EXPORT_SYMBOL_GPL(eeh_dev_open); 1222 1223 /** 1224 * eeh_dev_release - Decrease count of pass through devices for PE 1225 * @pdev: PCI device 1226 * 1227 * Decrease count of pass through devices for the indicated PE. If 1228 * there is no passed through device in PE, the EEH errors detected 1229 * on the PE will be reported and handled as usual. 1230 */ 1231 void eeh_dev_release(struct pci_dev *pdev) 1232 { 1233 struct eeh_dev *edev; 1234 1235 mutex_lock(&eeh_dev_mutex); 1236 1237 /* No PCI device ? */ 1238 if (!pdev) 1239 goto out; 1240 1241 /* No EEH device ? */ 1242 edev = pci_dev_to_eeh_dev(pdev); 1243 if (!edev || !edev->pe || !eeh_pe_passed(edev->pe)) 1244 goto out; 1245 1246 /* Decrease PE's pass through count */ 1247 WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0); 1248 eeh_pe_change_owner(edev->pe); 1249 out: 1250 mutex_unlock(&eeh_dev_mutex); 1251 } 1252 EXPORT_SYMBOL(eeh_dev_release); 1253 1254 #ifdef CONFIG_IOMMU_API 1255 1256 static int dev_has_iommu_table(struct device *dev, void *data) 1257 { 1258 struct pci_dev *pdev = to_pci_dev(dev); 1259 struct pci_dev **ppdev = data; 1260 1261 if (!dev) 1262 return 0; 1263 1264 if (device_iommu_mapped(dev)) { 1265 *ppdev = pdev; 1266 return 1; 1267 } 1268 1269 return 0; 1270 } 1271 1272 /** 1273 * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE 1274 * @group: IOMMU group 1275 * 1276 * The routine is called to convert IOMMU group to EEH PE. 1277 */ 1278 struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group) 1279 { 1280 struct pci_dev *pdev = NULL; 1281 struct eeh_dev *edev; 1282 int ret; 1283 1284 /* No IOMMU group ? */ 1285 if (!group) 1286 return NULL; 1287 1288 ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table); 1289 if (!ret || !pdev) 1290 return NULL; 1291 1292 /* No EEH device or PE ? */ 1293 edev = pci_dev_to_eeh_dev(pdev); 1294 if (!edev || !edev->pe) 1295 return NULL; 1296 1297 return edev->pe; 1298 } 1299 EXPORT_SYMBOL_GPL(eeh_iommu_group_to_pe); 1300 1301 #endif /* CONFIG_IOMMU_API */ 1302 1303 /** 1304 * eeh_pe_set_option - Set options for the indicated PE 1305 * @pe: EEH PE 1306 * @option: requested option 1307 * 1308 * The routine is called to enable or disable EEH functionality 1309 * on the indicated PE, to enable IO or DMA for the frozen PE. 1310 */ 1311 int eeh_pe_set_option(struct eeh_pe *pe, int option) 1312 { 1313 int ret = 0; 1314 1315 /* Invalid PE ? */ 1316 if (!pe) 1317 return -ENODEV; 1318 1319 /* 1320 * EEH functionality could possibly be disabled, just 1321 * return error for the case. And the EEH functinality 1322 * isn't expected to be disabled on one specific PE. 1323 */ 1324 switch (option) { 1325 case EEH_OPT_ENABLE: 1326 if (eeh_enabled()) { 1327 ret = eeh_pe_change_owner(pe); 1328 break; 1329 } 1330 ret = -EIO; 1331 break; 1332 case EEH_OPT_DISABLE: 1333 break; 1334 case EEH_OPT_THAW_MMIO: 1335 case EEH_OPT_THAW_DMA: 1336 case EEH_OPT_FREEZE_PE: 1337 if (!eeh_ops || !eeh_ops->set_option) { 1338 ret = -ENOENT; 1339 break; 1340 } 1341 1342 ret = eeh_pci_enable(pe, option); 1343 break; 1344 default: 1345 pr_debug("%s: Option %d out of range (%d, %d)\n", 1346 __func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA); 1347 ret = -EINVAL; 1348 } 1349 1350 return ret; 1351 } 1352 EXPORT_SYMBOL_GPL(eeh_pe_set_option); 1353 1354 /** 1355 * eeh_pe_get_state - Retrieve PE's state 1356 * @pe: EEH PE 1357 * 1358 * Retrieve the PE's state, which includes 3 aspects: enabled 1359 * DMA, enabled IO and asserted reset. 1360 */ 1361 int eeh_pe_get_state(struct eeh_pe *pe) 1362 { 1363 int result, ret = 0; 1364 bool rst_active, dma_en, mmio_en; 1365 1366 /* Existing PE ? */ 1367 if (!pe) 1368 return -ENODEV; 1369 1370 if (!eeh_ops || !eeh_ops->get_state) 1371 return -ENOENT; 1372 1373 /* 1374 * If the parent PE is owned by the host kernel and is undergoing 1375 * error recovery, we should return the PE state as temporarily 1376 * unavailable so that the error recovery on the guest is suspended 1377 * until the recovery completes on the host. 1378 */ 1379 if (pe->parent && 1380 !(pe->state & EEH_PE_REMOVED) && 1381 (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING))) 1382 return EEH_PE_STATE_UNAVAIL; 1383 1384 result = eeh_ops->get_state(pe, NULL); 1385 rst_active = !!(result & EEH_STATE_RESET_ACTIVE); 1386 dma_en = !!(result & EEH_STATE_DMA_ENABLED); 1387 mmio_en = !!(result & EEH_STATE_MMIO_ENABLED); 1388 1389 if (rst_active) 1390 ret = EEH_PE_STATE_RESET; 1391 else if (dma_en && mmio_en) 1392 ret = EEH_PE_STATE_NORMAL; 1393 else if (!dma_en && !mmio_en) 1394 ret = EEH_PE_STATE_STOPPED_IO_DMA; 1395 else if (!dma_en && mmio_en) 1396 ret = EEH_PE_STATE_STOPPED_DMA; 1397 else 1398 ret = EEH_PE_STATE_UNAVAIL; 1399 1400 return ret; 1401 } 1402 EXPORT_SYMBOL_GPL(eeh_pe_get_state); 1403 1404 static int eeh_pe_reenable_devices(struct eeh_pe *pe, bool include_passed) 1405 { 1406 struct eeh_dev *edev, *tmp; 1407 struct pci_dev *pdev; 1408 int ret = 0; 1409 1410 eeh_pe_restore_bars(pe); 1411 1412 /* 1413 * Reenable PCI devices as the devices passed 1414 * through are always enabled before the reset. 1415 */ 1416 eeh_pe_for_each_dev(pe, edev, tmp) { 1417 pdev = eeh_dev_to_pci_dev(edev); 1418 if (!pdev) 1419 continue; 1420 1421 ret = pci_reenable_device(pdev); 1422 if (ret) { 1423 pr_warn("%s: Failure %d reenabling %s\n", 1424 __func__, ret, pci_name(pdev)); 1425 return ret; 1426 } 1427 } 1428 1429 /* The PE is still in frozen state */ 1430 if (include_passed || !eeh_pe_passed(pe)) { 1431 ret = eeh_unfreeze_pe(pe); 1432 } else 1433 pr_info("EEH: Note: Leaving passthrough PHB#%x-PE#%x frozen.\n", 1434 pe->phb->global_number, pe->addr); 1435 if (!ret) 1436 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, include_passed); 1437 return ret; 1438 } 1439 1440 1441 /** 1442 * eeh_pe_reset - Issue PE reset according to specified type 1443 * @pe: EEH PE 1444 * @option: reset type 1445 * 1446 * The routine is called to reset the specified PE with the 1447 * indicated type, either fundamental reset or hot reset. 1448 * PE reset is the most important part for error recovery. 1449 */ 1450 int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed) 1451 { 1452 int ret = 0; 1453 1454 /* Invalid PE ? */ 1455 if (!pe) 1456 return -ENODEV; 1457 1458 if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset) 1459 return -ENOENT; 1460 1461 switch (option) { 1462 case EEH_RESET_DEACTIVATE: 1463 ret = eeh_ops->reset(pe, option); 1464 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, include_passed); 1465 if (ret) 1466 break; 1467 1468 ret = eeh_pe_reenable_devices(pe, include_passed); 1469 break; 1470 case EEH_RESET_HOT: 1471 case EEH_RESET_FUNDAMENTAL: 1472 /* 1473 * Proactively freeze the PE to drop all MMIO access 1474 * during reset, which should be banned as it's always 1475 * cause recursive EEH error. 1476 */ 1477 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 1478 1479 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 1480 ret = eeh_ops->reset(pe, option); 1481 break; 1482 default: 1483 pr_debug("%s: Unsupported option %d\n", 1484 __func__, option); 1485 ret = -EINVAL; 1486 } 1487 1488 return ret; 1489 } 1490 EXPORT_SYMBOL_GPL(eeh_pe_reset); 1491 1492 /** 1493 * eeh_pe_configure - Configure PCI bridges after PE reset 1494 * @pe: EEH PE 1495 * 1496 * The routine is called to restore the PCI config space for 1497 * those PCI devices, especially PCI bridges affected by PE 1498 * reset issued previously. 1499 */ 1500 int eeh_pe_configure(struct eeh_pe *pe) 1501 { 1502 int ret = 0; 1503 1504 /* Invalid PE ? */ 1505 if (!pe) 1506 return -ENODEV; 1507 1508 return ret; 1509 } 1510 EXPORT_SYMBOL_GPL(eeh_pe_configure); 1511 1512 /** 1513 * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE 1514 * @pe: the indicated PE 1515 * @type: error type 1516 * @function: error function 1517 * @addr: address 1518 * @mask: address mask 1519 * 1520 * The routine is called to inject the specified PCI error, which 1521 * is determined by @type and @function, to the indicated PE for 1522 * testing purpose. 1523 */ 1524 int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, 1525 unsigned long addr, unsigned long mask) 1526 { 1527 /* Invalid PE ? */ 1528 if (!pe) 1529 return -ENODEV; 1530 1531 /* Unsupported operation ? */ 1532 if (!eeh_ops || !eeh_ops->err_inject) 1533 return -ENOENT; 1534 1535 /* Check on PCI error type */ 1536 if (type != EEH_ERR_TYPE_32 && type != EEH_ERR_TYPE_64) 1537 return -EINVAL; 1538 1539 /* Check on PCI error function */ 1540 if (func < EEH_ERR_FUNC_MIN || func > EEH_ERR_FUNC_MAX) 1541 return -EINVAL; 1542 1543 return eeh_ops->err_inject(pe, type, func, addr, mask); 1544 } 1545 EXPORT_SYMBOL_GPL(eeh_pe_inject_err); 1546 1547 #ifdef CONFIG_PROC_FS 1548 static int proc_eeh_show(struct seq_file *m, void *v) 1549 { 1550 if (!eeh_enabled()) { 1551 seq_printf(m, "EEH Subsystem is globally disabled\n"); 1552 seq_printf(m, "eeh_total_mmio_ffs=%llu\n", eeh_stats.total_mmio_ffs); 1553 } else { 1554 seq_printf(m, "EEH Subsystem is enabled\n"); 1555 seq_printf(m, 1556 "no device=%llu\n" 1557 "no device node=%llu\n" 1558 "no config address=%llu\n" 1559 "check not wanted=%llu\n" 1560 "eeh_total_mmio_ffs=%llu\n" 1561 "eeh_false_positives=%llu\n" 1562 "eeh_slot_resets=%llu\n", 1563 eeh_stats.no_device, 1564 eeh_stats.no_dn, 1565 eeh_stats.no_cfg_addr, 1566 eeh_stats.ignored_check, 1567 eeh_stats.total_mmio_ffs, 1568 eeh_stats.false_positives, 1569 eeh_stats.slot_resets); 1570 } 1571 1572 return 0; 1573 } 1574 #endif /* CONFIG_PROC_FS */ 1575 1576 #ifdef CONFIG_DEBUG_FS 1577 1578 1579 static struct pci_dev *eeh_debug_lookup_pdev(struct file *filp, 1580 const char __user *user_buf, 1581 size_t count, loff_t *ppos) 1582 { 1583 uint32_t domain, bus, dev, fn; 1584 struct pci_dev *pdev; 1585 char buf[20]; 1586 int ret; 1587 1588 memset(buf, 0, sizeof(buf)); 1589 ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); 1590 if (!ret) 1591 return ERR_PTR(-EFAULT); 1592 1593 ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); 1594 if (ret != 4) { 1595 pr_err("%s: expected 4 args, got %d\n", __func__, ret); 1596 return ERR_PTR(-EINVAL); 1597 } 1598 1599 pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); 1600 if (!pdev) 1601 return ERR_PTR(-ENODEV); 1602 1603 return pdev; 1604 } 1605 1606 static int eeh_enable_dbgfs_set(void *data, u64 val) 1607 { 1608 if (val) 1609 eeh_clear_flag(EEH_FORCE_DISABLED); 1610 else 1611 eeh_add_flag(EEH_FORCE_DISABLED); 1612 1613 return 0; 1614 } 1615 1616 static int eeh_enable_dbgfs_get(void *data, u64 *val) 1617 { 1618 if (eeh_enabled()) 1619 *val = 0x1ul; 1620 else 1621 *val = 0x0ul; 1622 return 0; 1623 } 1624 1625 DEFINE_DEBUGFS_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get, 1626 eeh_enable_dbgfs_set, "0x%llx\n"); 1627 1628 static ssize_t eeh_force_recover_write(struct file *filp, 1629 const char __user *user_buf, 1630 size_t count, loff_t *ppos) 1631 { 1632 struct pci_controller *hose; 1633 uint32_t phbid, pe_no; 1634 struct eeh_pe *pe; 1635 char buf[20]; 1636 int ret; 1637 1638 ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count); 1639 if (!ret) 1640 return -EFAULT; 1641 1642 /* 1643 * When PE is NULL the event is a "special" event. Rather than 1644 * recovering a specific PE it forces the EEH core to scan for failed 1645 * PHBs and recovers each. This needs to be done before any device 1646 * recoveries can occur. 1647 */ 1648 if (!strncmp(buf, "hwcheck", 7)) { 1649 __eeh_send_failure_event(NULL); 1650 return count; 1651 } 1652 1653 ret = sscanf(buf, "%x:%x", &phbid, &pe_no); 1654 if (ret != 2) 1655 return -EINVAL; 1656 1657 hose = pci_find_controller_for_domain(phbid); 1658 if (!hose) 1659 return -ENODEV; 1660 1661 /* Retrieve PE */ 1662 pe = eeh_pe_get(hose, pe_no); 1663 if (!pe) 1664 return -ENODEV; 1665 1666 /* 1667 * We don't do any state checking here since the detection 1668 * process is async to the recovery process. The recovery 1669 * thread *should* not break even if we schedule a recovery 1670 * from an odd state (e.g. PE removed, or recovery of a 1671 * non-isolated PE) 1672 */ 1673 __eeh_send_failure_event(pe); 1674 1675 return ret < 0 ? ret : count; 1676 } 1677 1678 static const struct file_operations eeh_force_recover_fops = { 1679 .open = simple_open, 1680 .llseek = no_llseek, 1681 .write = eeh_force_recover_write, 1682 }; 1683 1684 static ssize_t eeh_debugfs_dev_usage(struct file *filp, 1685 char __user *user_buf, 1686 size_t count, loff_t *ppos) 1687 { 1688 static const char usage[] = "input format: <domain>:<bus>:<dev>.<fn>\n"; 1689 1690 return simple_read_from_buffer(user_buf, count, ppos, 1691 usage, sizeof(usage) - 1); 1692 } 1693 1694 static ssize_t eeh_dev_check_write(struct file *filp, 1695 const char __user *user_buf, 1696 size_t count, loff_t *ppos) 1697 { 1698 struct pci_dev *pdev; 1699 struct eeh_dev *edev; 1700 int ret; 1701 1702 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1703 if (IS_ERR(pdev)) 1704 return PTR_ERR(pdev); 1705 1706 edev = pci_dev_to_eeh_dev(pdev); 1707 if (!edev) { 1708 pci_err(pdev, "No eeh_dev for this device!\n"); 1709 pci_dev_put(pdev); 1710 return -ENODEV; 1711 } 1712 1713 ret = eeh_dev_check_failure(edev); 1714 pci_info(pdev, "eeh_dev_check_failure(%s) = %d\n", 1715 pci_name(pdev), ret); 1716 1717 pci_dev_put(pdev); 1718 1719 return count; 1720 } 1721 1722 static const struct file_operations eeh_dev_check_fops = { 1723 .open = simple_open, 1724 .llseek = no_llseek, 1725 .write = eeh_dev_check_write, 1726 .read = eeh_debugfs_dev_usage, 1727 }; 1728 1729 static int eeh_debugfs_break_device(struct pci_dev *pdev) 1730 { 1731 struct resource *bar = NULL; 1732 void __iomem *mapped; 1733 u16 old, bit; 1734 int i, pos; 1735 1736 /* Do we have an MMIO BAR to disable? */ 1737 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { 1738 struct resource *r = &pdev->resource[i]; 1739 1740 if (!r->flags || !r->start) 1741 continue; 1742 if (r->flags & IORESOURCE_IO) 1743 continue; 1744 if (r->flags & IORESOURCE_UNSET) 1745 continue; 1746 1747 bar = r; 1748 break; 1749 } 1750 1751 if (!bar) { 1752 pci_err(pdev, "Unable to find Memory BAR to cause EEH with\n"); 1753 return -ENXIO; 1754 } 1755 1756 pci_err(pdev, "Going to break: %pR\n", bar); 1757 1758 if (pdev->is_virtfn) { 1759 #ifndef CONFIG_PCI_IOV 1760 return -ENXIO; 1761 #else 1762 /* 1763 * VFs don't have a per-function COMMAND register, so the best 1764 * we can do is clear the Memory Space Enable bit in the PF's 1765 * SRIOV control reg. 1766 * 1767 * Unfortunately, this requires that we have a PF (i.e doesn't 1768 * work for a passed-through VF) and it has the potential side 1769 * effect of also causing an EEH on every other VF under the 1770 * PF. Oh well. 1771 */ 1772 pdev = pdev->physfn; 1773 if (!pdev) 1774 return -ENXIO; /* passed through VFs have no PF */ 1775 1776 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 1777 pos += PCI_SRIOV_CTRL; 1778 bit = PCI_SRIOV_CTRL_MSE; 1779 #endif /* !CONFIG_PCI_IOV */ 1780 } else { 1781 bit = PCI_COMMAND_MEMORY; 1782 pos = PCI_COMMAND; 1783 } 1784 1785 /* 1786 * Process here is: 1787 * 1788 * 1. Disable Memory space. 1789 * 1790 * 2. Perform an MMIO to the device. This should result in an error 1791 * (CA / UR) being raised by the device which results in an EEH 1792 * PE freeze. Using the in_8() accessor skips the eeh detection hook 1793 * so the freeze hook so the EEH Detection machinery won't be 1794 * triggered here. This is to match the usual behaviour of EEH 1795 * where the HW will asyncronously freeze a PE and it's up to 1796 * the kernel to notice and deal with it. 1797 * 1798 * 3. Turn Memory space back on. This is more important for VFs 1799 * since recovery will probably fail if we don't. For normal 1800 * the COMMAND register is reset as a part of re-initialising 1801 * the device. 1802 * 1803 * Breaking stuff is the point so who cares if it's racy ;) 1804 */ 1805 pci_read_config_word(pdev, pos, &old); 1806 1807 mapped = ioremap(bar->start, PAGE_SIZE); 1808 if (!mapped) { 1809 pci_err(pdev, "Unable to map MMIO BAR %pR\n", bar); 1810 return -ENXIO; 1811 } 1812 1813 pci_write_config_word(pdev, pos, old & ~bit); 1814 in_8(mapped); 1815 pci_write_config_word(pdev, pos, old); 1816 1817 iounmap(mapped); 1818 1819 return 0; 1820 } 1821 1822 static ssize_t eeh_dev_break_write(struct file *filp, 1823 const char __user *user_buf, 1824 size_t count, loff_t *ppos) 1825 { 1826 struct pci_dev *pdev; 1827 int ret; 1828 1829 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1830 if (IS_ERR(pdev)) 1831 return PTR_ERR(pdev); 1832 1833 ret = eeh_debugfs_break_device(pdev); 1834 pci_dev_put(pdev); 1835 1836 if (ret < 0) 1837 return ret; 1838 1839 return count; 1840 } 1841 1842 static const struct file_operations eeh_dev_break_fops = { 1843 .open = simple_open, 1844 .llseek = no_llseek, 1845 .write = eeh_dev_break_write, 1846 .read = eeh_debugfs_dev_usage, 1847 }; 1848 1849 static ssize_t eeh_dev_can_recover(struct file *filp, 1850 const char __user *user_buf, 1851 size_t count, loff_t *ppos) 1852 { 1853 struct pci_driver *drv; 1854 struct pci_dev *pdev; 1855 size_t ret; 1856 1857 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1858 if (IS_ERR(pdev)) 1859 return PTR_ERR(pdev); 1860 1861 /* 1862 * In order for error recovery to work the driver needs to implement 1863 * .error_detected(), so it can quiesce IO to the device, and 1864 * .slot_reset() so it can re-initialise the device after a reset. 1865 * 1866 * Ideally they'd implement .resume() too, but some drivers which 1867 * we need to support (notably IPR) don't so I guess we can tolerate 1868 * that. 1869 * 1870 * .mmio_enabled() is mostly there as a work-around for devices which 1871 * take forever to re-init after a hot reset. Implementing that is 1872 * strictly optional. 1873 */ 1874 drv = pci_dev_driver(pdev); 1875 if (drv && 1876 drv->err_handler && 1877 drv->err_handler->error_detected && 1878 drv->err_handler->slot_reset) { 1879 ret = count; 1880 } else { 1881 ret = -EOPNOTSUPP; 1882 } 1883 1884 pci_dev_put(pdev); 1885 1886 return ret; 1887 } 1888 1889 static const struct file_operations eeh_dev_can_recover_fops = { 1890 .open = simple_open, 1891 .llseek = no_llseek, 1892 .write = eeh_dev_can_recover, 1893 .read = eeh_debugfs_dev_usage, 1894 }; 1895 1896 #endif 1897 1898 static int __init eeh_init_proc(void) 1899 { 1900 if (machine_is(pseries) || machine_is(powernv)) { 1901 proc_create_single("powerpc/eeh", 0, NULL, proc_eeh_show); 1902 #ifdef CONFIG_DEBUG_FS 1903 debugfs_create_file_unsafe("eeh_enable", 0600, 1904 arch_debugfs_dir, NULL, 1905 &eeh_enable_dbgfs_ops); 1906 debugfs_create_u32("eeh_max_freezes", 0600, 1907 arch_debugfs_dir, &eeh_max_freezes); 1908 debugfs_create_bool("eeh_disable_recovery", 0600, 1909 arch_debugfs_dir, 1910 &eeh_debugfs_no_recover); 1911 debugfs_create_file_unsafe("eeh_dev_check", 0600, 1912 arch_debugfs_dir, NULL, 1913 &eeh_dev_check_fops); 1914 debugfs_create_file_unsafe("eeh_dev_break", 0600, 1915 arch_debugfs_dir, NULL, 1916 &eeh_dev_break_fops); 1917 debugfs_create_file_unsafe("eeh_force_recover", 0600, 1918 arch_debugfs_dir, NULL, 1919 &eeh_force_recover_fops); 1920 debugfs_create_file_unsafe("eeh_dev_can_recover", 0600, 1921 arch_debugfs_dir, NULL, 1922 &eeh_dev_can_recover_fops); 1923 eeh_cache_debugfs_init(); 1924 #endif 1925 } 1926 1927 return 0; 1928 } 1929 __initcall(eeh_init_proc); 1930