1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright IBM Corporation 2001, 2005, 2006 4 * Copyright Dave Engebretsen & Todd Inglett 2001 5 * Copyright Linas Vepstas 2005, 2006 6 * Copyright 2001-2012 IBM Corporation. 7 * 8 * Please address comments and feedback to Linas Vepstas <linas@austin.ibm.com> 9 */ 10 11 #include <linux/delay.h> 12 #include <linux/sched.h> 13 #include <linux/init.h> 14 #include <linux/list.h> 15 #include <linux/pci.h> 16 #include <linux/iommu.h> 17 #include <linux/proc_fs.h> 18 #include <linux/rbtree.h> 19 #include <linux/reboot.h> 20 #include <linux/seq_file.h> 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 #include <linux/of.h> 24 25 #include <linux/atomic.h> 26 #include <asm/debugfs.h> 27 #include <asm/eeh.h> 28 #include <asm/eeh_event.h> 29 #include <asm/io.h> 30 #include <asm/iommu.h> 31 #include <asm/machdep.h> 32 #include <asm/ppc-pci.h> 33 #include <asm/rtas.h> 34 #include <asm/pte-walk.h> 35 36 37 /** Overview: 38 * EEH, or "Enhanced Error Handling" is a PCI bridge technology for 39 * dealing with PCI bus errors that can't be dealt with within the 40 * usual PCI framework, except by check-stopping the CPU. Systems 41 * that are designed for high-availability/reliability cannot afford 42 * to crash due to a "mere" PCI error, thus the need for EEH. 43 * An EEH-capable bridge operates by converting a detected error 44 * into a "slot freeze", taking the PCI adapter off-line, making 45 * the slot behave, from the OS'es point of view, as if the slot 46 * were "empty": all reads return 0xff's and all writes are silently 47 * ignored. EEH slot isolation events can be triggered by parity 48 * errors on the address or data busses (e.g. during posted writes), 49 * which in turn might be caused by low voltage on the bus, dust, 50 * vibration, humidity, radioactivity or plain-old failed hardware. 51 * 52 * Note, however, that one of the leading causes of EEH slot 53 * freeze events are buggy device drivers, buggy device microcode, 54 * or buggy device hardware. This is because any attempt by the 55 * device to bus-master data to a memory address that is not 56 * assigned to the device will trigger a slot freeze. (The idea 57 * is to prevent devices-gone-wild from corrupting system memory). 58 * Buggy hardware/drivers will have a miserable time co-existing 59 * with EEH. 60 * 61 * Ideally, a PCI device driver, when suspecting that an isolation 62 * event has occurred (e.g. by reading 0xff's), will then ask EEH 63 * whether this is the case, and then take appropriate steps to 64 * reset the PCI slot, the PCI device, and then resume operations. 65 * However, until that day, the checking is done here, with the 66 * eeh_check_failure() routine embedded in the MMIO macros. If 67 * the slot is found to be isolated, an "EEH Event" is synthesized 68 * and sent out for processing. 69 */ 70 71 /* If a device driver keeps reading an MMIO register in an interrupt 72 * handler after a slot isolation event, it might be broken. 73 * This sets the threshold for how many read attempts we allow 74 * before printing an error message. 75 */ 76 #define EEH_MAX_FAILS 2100000 77 78 /* Time to wait for a PCI slot to report status, in milliseconds */ 79 #define PCI_BUS_RESET_WAIT_MSEC (5*60*1000) 80 81 /* 82 * EEH probe mode support, which is part of the flags, 83 * is to support multiple platforms for EEH. Some platforms 84 * like pSeries do PCI emunation based on device tree. 85 * However, other platforms like powernv probe PCI devices 86 * from hardware. The flag is used to distinguish that. 87 * In addition, struct eeh_ops::probe would be invoked for 88 * particular OF node or PCI device so that the corresponding 89 * PE would be created there. 90 */ 91 int eeh_subsystem_flags; 92 EXPORT_SYMBOL(eeh_subsystem_flags); 93 94 /* 95 * EEH allowed maximal frozen times. If one particular PE's 96 * frozen count in last hour exceeds this limit, the PE will 97 * be forced to be offline permanently. 98 */ 99 u32 eeh_max_freezes = 5; 100 101 /* 102 * Controls whether a recovery event should be scheduled when an 103 * isolated device is discovered. This is only really useful for 104 * debugging problems with the EEH core. 105 */ 106 bool eeh_debugfs_no_recover; 107 108 /* Platform dependent EEH operations */ 109 struct eeh_ops *eeh_ops = NULL; 110 111 /* Lock to avoid races due to multiple reports of an error */ 112 DEFINE_RAW_SPINLOCK(confirm_error_lock); 113 EXPORT_SYMBOL_GPL(confirm_error_lock); 114 115 /* Lock to protect passed flags */ 116 static DEFINE_MUTEX(eeh_dev_mutex); 117 118 /* Buffer for reporting pci register dumps. Its here in BSS, and 119 * not dynamically alloced, so that it ends up in RMO where RTAS 120 * can access it. 121 */ 122 #define EEH_PCI_REGS_LOG_LEN 8192 123 static unsigned char pci_regs_buf[EEH_PCI_REGS_LOG_LEN]; 124 125 /* 126 * The struct is used to maintain the EEH global statistic 127 * information. Besides, the EEH global statistics will be 128 * exported to user space through procfs 129 */ 130 struct eeh_stats { 131 u64 no_device; /* PCI device not found */ 132 u64 no_dn; /* OF node not found */ 133 u64 no_cfg_addr; /* Config address not found */ 134 u64 ignored_check; /* EEH check skipped */ 135 u64 total_mmio_ffs; /* Total EEH checks */ 136 u64 false_positives; /* Unnecessary EEH checks */ 137 u64 slot_resets; /* PE reset */ 138 }; 139 140 static struct eeh_stats eeh_stats; 141 142 static int __init eeh_setup(char *str) 143 { 144 if (!strcmp(str, "off")) 145 eeh_add_flag(EEH_FORCE_DISABLED); 146 else if (!strcmp(str, "early_log")) 147 eeh_add_flag(EEH_EARLY_DUMP_LOG); 148 149 return 1; 150 } 151 __setup("eeh=", eeh_setup); 152 153 void eeh_show_enabled(void) 154 { 155 if (eeh_has_flag(EEH_FORCE_DISABLED)) 156 pr_info("EEH: Recovery disabled by kernel parameter.\n"); 157 else if (eeh_has_flag(EEH_ENABLED)) 158 pr_info("EEH: Capable adapter found: recovery enabled.\n"); 159 else 160 pr_info("EEH: No capable adapters found: recovery disabled.\n"); 161 } 162 163 /* 164 * This routine captures assorted PCI configuration space data 165 * for the indicated PCI device, and puts them into a buffer 166 * for RTAS error logging. 167 */ 168 static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) 169 { 170 u32 cfg; 171 int cap, i; 172 int n = 0, l = 0; 173 char buffer[128]; 174 175 n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n", 176 edev->pe->phb->global_number, edev->bdfn >> 8, 177 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 178 pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n", 179 edev->pe->phb->global_number, edev->bdfn >> 8, 180 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 181 182 eeh_ops->read_config(edev, PCI_VENDOR_ID, 4, &cfg); 183 n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg); 184 pr_warn("EEH: PCI device/vendor: %08x\n", cfg); 185 186 eeh_ops->read_config(edev, PCI_COMMAND, 4, &cfg); 187 n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg); 188 pr_warn("EEH: PCI cmd/status register: %08x\n", cfg); 189 190 /* Gather bridge-specific registers */ 191 if (edev->mode & EEH_DEV_BRIDGE) { 192 eeh_ops->read_config(edev, PCI_SEC_STATUS, 2, &cfg); 193 n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg); 194 pr_warn("EEH: Bridge secondary status: %04x\n", cfg); 195 196 eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &cfg); 197 n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg); 198 pr_warn("EEH: Bridge control: %04x\n", cfg); 199 } 200 201 /* Dump out the PCI-X command and status regs */ 202 cap = edev->pcix_cap; 203 if (cap) { 204 eeh_ops->read_config(edev, cap, 4, &cfg); 205 n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg); 206 pr_warn("EEH: PCI-X cmd: %08x\n", cfg); 207 208 eeh_ops->read_config(edev, cap+4, 4, &cfg); 209 n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg); 210 pr_warn("EEH: PCI-X status: %08x\n", cfg); 211 } 212 213 /* If PCI-E capable, dump PCI-E cap 10 */ 214 cap = edev->pcie_cap; 215 if (cap) { 216 n += scnprintf(buf+n, len-n, "pci-e cap10:\n"); 217 pr_warn("EEH: PCI-E capabilities and status follow:\n"); 218 219 for (i=0; i<=8; i++) { 220 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 221 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 222 223 if ((i % 4) == 0) { 224 if (i != 0) 225 pr_warn("%s\n", buffer); 226 227 l = scnprintf(buffer, sizeof(buffer), 228 "EEH: PCI-E %02x: %08x ", 229 4*i, cfg); 230 } else { 231 l += scnprintf(buffer+l, sizeof(buffer)-l, 232 "%08x ", cfg); 233 } 234 235 } 236 237 pr_warn("%s\n", buffer); 238 } 239 240 /* If AER capable, dump it */ 241 cap = edev->aer_cap; 242 if (cap) { 243 n += scnprintf(buf+n, len-n, "pci-e AER:\n"); 244 pr_warn("EEH: PCI-E AER capability register set follows:\n"); 245 246 for (i=0; i<=13; i++) { 247 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 248 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 249 250 if ((i % 4) == 0) { 251 if (i != 0) 252 pr_warn("%s\n", buffer); 253 254 l = scnprintf(buffer, sizeof(buffer), 255 "EEH: PCI-E AER %02x: %08x ", 256 4*i, cfg); 257 } else { 258 l += scnprintf(buffer+l, sizeof(buffer)-l, 259 "%08x ", cfg); 260 } 261 } 262 263 pr_warn("%s\n", buffer); 264 } 265 266 return n; 267 } 268 269 static void *eeh_dump_pe_log(struct eeh_pe *pe, void *flag) 270 { 271 struct eeh_dev *edev, *tmp; 272 size_t *plen = flag; 273 274 eeh_pe_for_each_dev(pe, edev, tmp) 275 *plen += eeh_dump_dev_log(edev, pci_regs_buf + *plen, 276 EEH_PCI_REGS_LOG_LEN - *plen); 277 278 return NULL; 279 } 280 281 /** 282 * eeh_slot_error_detail - Generate combined log including driver log and error log 283 * @pe: EEH PE 284 * @severity: temporary or permanent error log 285 * 286 * This routine should be called to generate the combined log, which 287 * is comprised of driver log and error log. The driver log is figured 288 * out from the config space of the corresponding PCI device, while 289 * the error log is fetched through platform dependent function call. 290 */ 291 void eeh_slot_error_detail(struct eeh_pe *pe, int severity) 292 { 293 size_t loglen = 0; 294 295 /* 296 * When the PHB is fenced or dead, it's pointless to collect 297 * the data from PCI config space because it should return 298 * 0xFF's. For ER, we still retrieve the data from the PCI 299 * config space. 300 * 301 * For pHyp, we have to enable IO for log retrieval. Otherwise, 302 * 0xFF's is always returned from PCI config space. 303 * 304 * When the @severity is EEH_LOG_PERM, the PE is going to be 305 * removed. Prior to that, the drivers for devices included in 306 * the PE will be closed. The drivers rely on working IO path 307 * to bring the devices to quiet state. Otherwise, PCI traffic 308 * from those devices after they are removed is like to cause 309 * another unexpected EEH error. 310 */ 311 if (!(pe->type & EEH_PE_PHB)) { 312 if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG) || 313 severity == EEH_LOG_PERM) 314 eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 315 316 /* 317 * The config space of some PCI devices can't be accessed 318 * when their PEs are in frozen state. Otherwise, fenced 319 * PHB might be seen. Those PEs are identified with flag 320 * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED 321 * is set automatically when the PE is put to EEH_PE_ISOLATED. 322 * 323 * Restoring BARs possibly triggers PCI config access in 324 * (OPAL) firmware and then causes fenced PHB. If the 325 * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's 326 * pointless to restore BARs and dump config space. 327 */ 328 eeh_ops->configure_bridge(pe); 329 if (!(pe->state & EEH_PE_CFG_BLOCKED)) { 330 eeh_pe_restore_bars(pe); 331 332 pci_regs_buf[0] = 0; 333 eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen); 334 } 335 } 336 337 eeh_ops->get_log(pe, severity, pci_regs_buf, loglen); 338 } 339 340 /** 341 * eeh_token_to_phys - Convert EEH address token to phys address 342 * @token: I/O token, should be address in the form 0xA.... 343 * 344 * This routine should be called to convert virtual I/O address 345 * to physical one. 346 */ 347 static inline unsigned long eeh_token_to_phys(unsigned long token) 348 { 349 pte_t *ptep; 350 unsigned long pa; 351 int hugepage_shift; 352 353 /* 354 * We won't find hugepages here(this is iomem). Hence we are not 355 * worried about _PAGE_SPLITTING/collapse. Also we will not hit 356 * page table free, because of init_mm. 357 */ 358 ptep = find_init_mm_pte(token, &hugepage_shift); 359 if (!ptep) 360 return token; 361 362 pa = pte_pfn(*ptep); 363 364 /* On radix we can do hugepage mappings for io, so handle that */ 365 if (!hugepage_shift) 366 hugepage_shift = PAGE_SHIFT; 367 368 pa <<= PAGE_SHIFT; 369 pa |= token & ((1ul << hugepage_shift) - 1); 370 return pa; 371 } 372 373 /* 374 * On PowerNV platform, we might already have fenced PHB there. 375 * For that case, it's meaningless to recover frozen PE. Intead, 376 * We have to handle fenced PHB firstly. 377 */ 378 static int eeh_phb_check_failure(struct eeh_pe *pe) 379 { 380 struct eeh_pe *phb_pe; 381 unsigned long flags; 382 int ret; 383 384 if (!eeh_has_flag(EEH_PROBE_MODE_DEV)) 385 return -EPERM; 386 387 /* Find the PHB PE */ 388 phb_pe = eeh_phb_pe_get(pe->phb); 389 if (!phb_pe) { 390 pr_warn("%s Can't find PE for PHB#%x\n", 391 __func__, pe->phb->global_number); 392 return -EEXIST; 393 } 394 395 /* If the PHB has been in problematic state */ 396 eeh_serialize_lock(&flags); 397 if (phb_pe->state & EEH_PE_ISOLATED) { 398 ret = 0; 399 goto out; 400 } 401 402 /* Check PHB state */ 403 ret = eeh_ops->get_state(phb_pe, NULL); 404 if ((ret < 0) || 405 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 406 ret = 0; 407 goto out; 408 } 409 410 /* Isolate the PHB and send event */ 411 eeh_pe_mark_isolated(phb_pe); 412 eeh_serialize_unlock(flags); 413 414 pr_debug("EEH: PHB#%x failure detected, location: %s\n", 415 phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe)); 416 eeh_send_failure_event(phb_pe); 417 return 1; 418 out: 419 eeh_serialize_unlock(flags); 420 return ret; 421 } 422 423 /** 424 * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze 425 * @edev: eeh device 426 * 427 * Check for an EEH failure for the given device node. Call this 428 * routine if the result of a read was all 0xff's and you want to 429 * find out if this is due to an EEH slot freeze. This routine 430 * will query firmware for the EEH status. 431 * 432 * Returns 0 if there has not been an EEH error; otherwise returns 433 * a non-zero value and queues up a slot isolation event notification. 434 * 435 * It is safe to call this routine in an interrupt context. 436 */ 437 int eeh_dev_check_failure(struct eeh_dev *edev) 438 { 439 int ret; 440 unsigned long flags; 441 struct device_node *dn; 442 struct pci_dev *dev; 443 struct eeh_pe *pe, *parent_pe; 444 int rc = 0; 445 const char *location = NULL; 446 447 eeh_stats.total_mmio_ffs++; 448 449 if (!eeh_enabled()) 450 return 0; 451 452 if (!edev) { 453 eeh_stats.no_dn++; 454 return 0; 455 } 456 dev = eeh_dev_to_pci_dev(edev); 457 pe = eeh_dev_to_pe(edev); 458 459 /* Access to IO BARs might get this far and still not want checking. */ 460 if (!pe) { 461 eeh_stats.ignored_check++; 462 eeh_edev_dbg(edev, "Ignored check\n"); 463 return 0; 464 } 465 466 /* 467 * On PowerNV platform, we might already have fenced PHB 468 * there and we need take care of that firstly. 469 */ 470 ret = eeh_phb_check_failure(pe); 471 if (ret > 0) 472 return ret; 473 474 /* 475 * If the PE isn't owned by us, we shouldn't check the 476 * state. Instead, let the owner handle it if the PE has 477 * been frozen. 478 */ 479 if (eeh_pe_passed(pe)) 480 return 0; 481 482 /* If we already have a pending isolation event for this 483 * slot, we know it's bad already, we don't need to check. 484 * Do this checking under a lock; as multiple PCI devices 485 * in one slot might report errors simultaneously, and we 486 * only want one error recovery routine running. 487 */ 488 eeh_serialize_lock(&flags); 489 rc = 1; 490 if (pe->state & EEH_PE_ISOLATED) { 491 pe->check_count++; 492 if (pe->check_count == EEH_MAX_FAILS) { 493 dn = pci_device_to_OF_node(dev); 494 if (dn) 495 location = of_get_property(dn, "ibm,loc-code", 496 NULL); 497 eeh_edev_err(edev, "%d reads ignored for recovering device at location=%s driver=%s\n", 498 pe->check_count, 499 location ? location : "unknown", 500 eeh_driver_name(dev)); 501 eeh_edev_err(edev, "Might be infinite loop in %s driver\n", 502 eeh_driver_name(dev)); 503 dump_stack(); 504 } 505 goto dn_unlock; 506 } 507 508 /* 509 * Now test for an EEH failure. This is VERY expensive. 510 * Note that the eeh_config_addr may be a parent device 511 * in the case of a device behind a bridge, or it may be 512 * function zero of a multi-function device. 513 * In any case they must share a common PHB. 514 */ 515 ret = eeh_ops->get_state(pe, NULL); 516 517 /* Note that config-io to empty slots may fail; 518 * they are empty when they don't have children. 519 * We will punt with the following conditions: Failure to get 520 * PE's state, EEH not support and Permanently unavailable 521 * state, PE is in good state. 522 */ 523 if ((ret < 0) || 524 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 525 eeh_stats.false_positives++; 526 pe->false_positives++; 527 rc = 0; 528 goto dn_unlock; 529 } 530 531 /* 532 * It should be corner case that the parent PE has been 533 * put into frozen state as well. We should take care 534 * that at first. 535 */ 536 parent_pe = pe->parent; 537 while (parent_pe) { 538 /* Hit the ceiling ? */ 539 if (parent_pe->type & EEH_PE_PHB) 540 break; 541 542 /* Frozen parent PE ? */ 543 ret = eeh_ops->get_state(parent_pe, NULL); 544 if (ret > 0 && !eeh_state_active(ret)) { 545 pe = parent_pe; 546 pr_err("EEH: Failure of PHB#%x-PE#%x will be handled at parent PHB#%x-PE#%x.\n", 547 pe->phb->global_number, pe->addr, 548 pe->phb->global_number, parent_pe->addr); 549 } 550 551 /* Next parent level */ 552 parent_pe = parent_pe->parent; 553 } 554 555 eeh_stats.slot_resets++; 556 557 /* Avoid repeated reports of this failure, including problems 558 * with other functions on this device, and functions under 559 * bridges. 560 */ 561 eeh_pe_mark_isolated(pe); 562 eeh_serialize_unlock(flags); 563 564 /* Most EEH events are due to device driver bugs. Having 565 * a stack trace will help the device-driver authors figure 566 * out what happened. So print that out. 567 */ 568 pr_debug("EEH: %s: Frozen PHB#%x-PE#%x detected\n", 569 __func__, pe->phb->global_number, pe->addr); 570 eeh_send_failure_event(pe); 571 572 return 1; 573 574 dn_unlock: 575 eeh_serialize_unlock(flags); 576 return rc; 577 } 578 579 EXPORT_SYMBOL_GPL(eeh_dev_check_failure); 580 581 /** 582 * eeh_check_failure - Check if all 1's data is due to EEH slot freeze 583 * @token: I/O address 584 * 585 * Check for an EEH failure at the given I/O address. Call this 586 * routine if the result of a read was all 0xff's and you want to 587 * find out if this is due to an EEH slot freeze event. This routine 588 * will query firmware for the EEH status. 589 * 590 * Note this routine is safe to call in an interrupt context. 591 */ 592 int eeh_check_failure(const volatile void __iomem *token) 593 { 594 unsigned long addr; 595 struct eeh_dev *edev; 596 597 /* Finding the phys addr + pci device; this is pretty quick. */ 598 addr = eeh_token_to_phys((unsigned long __force) token); 599 edev = eeh_addr_cache_get_dev(addr); 600 if (!edev) { 601 eeh_stats.no_device++; 602 return 0; 603 } 604 605 return eeh_dev_check_failure(edev); 606 } 607 EXPORT_SYMBOL(eeh_check_failure); 608 609 610 /** 611 * eeh_pci_enable - Enable MMIO or DMA transfers for this slot 612 * @pe: EEH PE 613 * 614 * This routine should be called to reenable frozen MMIO or DMA 615 * so that it would work correctly again. It's useful while doing 616 * recovery or log collection on the indicated device. 617 */ 618 int eeh_pci_enable(struct eeh_pe *pe, int function) 619 { 620 int active_flag, rc; 621 622 /* 623 * pHyp doesn't allow to enable IO or DMA on unfrozen PE. 624 * Also, it's pointless to enable them on unfrozen PE. So 625 * we have to check before enabling IO or DMA. 626 */ 627 switch (function) { 628 case EEH_OPT_THAW_MMIO: 629 active_flag = EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED; 630 break; 631 case EEH_OPT_THAW_DMA: 632 active_flag = EEH_STATE_DMA_ACTIVE; 633 break; 634 case EEH_OPT_DISABLE: 635 case EEH_OPT_ENABLE: 636 case EEH_OPT_FREEZE_PE: 637 active_flag = 0; 638 break; 639 default: 640 pr_warn("%s: Invalid function %d\n", 641 __func__, function); 642 return -EINVAL; 643 } 644 645 /* 646 * Check if IO or DMA has been enabled before 647 * enabling them. 648 */ 649 if (active_flag) { 650 rc = eeh_ops->get_state(pe, NULL); 651 if (rc < 0) 652 return rc; 653 654 /* Needn't enable it at all */ 655 if (rc == EEH_STATE_NOT_SUPPORT) 656 return 0; 657 658 /* It's already enabled */ 659 if (rc & active_flag) 660 return 0; 661 } 662 663 664 /* Issue the request */ 665 rc = eeh_ops->set_option(pe, function); 666 if (rc) 667 pr_warn("%s: Unexpected state change %d on " 668 "PHB#%x-PE#%x, err=%d\n", 669 __func__, function, pe->phb->global_number, 670 pe->addr, rc); 671 672 /* Check if the request is finished successfully */ 673 if (active_flag) { 674 rc = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 675 if (rc < 0) 676 return rc; 677 678 if (rc & active_flag) 679 return 0; 680 681 return -EIO; 682 } 683 684 return rc; 685 } 686 687 static void eeh_disable_and_save_dev_state(struct eeh_dev *edev, 688 void *userdata) 689 { 690 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 691 struct pci_dev *dev = userdata; 692 693 /* 694 * The caller should have disabled and saved the 695 * state for the specified device 696 */ 697 if (!pdev || pdev == dev) 698 return; 699 700 /* Ensure we have D0 power state */ 701 pci_set_power_state(pdev, PCI_D0); 702 703 /* Save device state */ 704 pci_save_state(pdev); 705 706 /* 707 * Disable device to avoid any DMA traffic and 708 * interrupt from the device 709 */ 710 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 711 } 712 713 static void eeh_restore_dev_state(struct eeh_dev *edev, void *userdata) 714 { 715 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 716 struct pci_dev *dev = userdata; 717 718 if (!pdev) 719 return; 720 721 /* Apply customization from firmware */ 722 if (eeh_ops->restore_config) 723 eeh_ops->restore_config(edev); 724 725 /* The caller should restore state for the specified device */ 726 if (pdev != dev) 727 pci_restore_state(pdev); 728 } 729 730 /** 731 * pcibios_set_pcie_reset_state - Set PCI-E reset state 732 * @dev: pci device struct 733 * @state: reset state to enter 734 * 735 * Return value: 736 * 0 if success 737 */ 738 int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 739 { 740 struct eeh_dev *edev = pci_dev_to_eeh_dev(dev); 741 struct eeh_pe *pe = eeh_dev_to_pe(edev); 742 743 if (!pe) { 744 pr_err("%s: No PE found on PCI device %s\n", 745 __func__, pci_name(dev)); 746 return -EINVAL; 747 } 748 749 switch (state) { 750 case pcie_deassert_reset: 751 eeh_ops->reset(pe, EEH_RESET_DEACTIVATE); 752 eeh_unfreeze_pe(pe); 753 if (!(pe->type & EEH_PE_VF)) 754 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 755 eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev); 756 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 757 break; 758 case pcie_hot_reset: 759 eeh_pe_mark_isolated(pe); 760 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 761 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 762 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 763 if (!(pe->type & EEH_PE_VF)) 764 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 765 eeh_ops->reset(pe, EEH_RESET_HOT); 766 break; 767 case pcie_warm_reset: 768 eeh_pe_mark_isolated(pe); 769 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 770 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 771 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 772 if (!(pe->type & EEH_PE_VF)) 773 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 774 eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL); 775 break; 776 default: 777 eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true); 778 return -EINVAL; 779 } 780 781 return 0; 782 } 783 784 /** 785 * eeh_set_pe_freset - Check the required reset for the indicated device 786 * @data: EEH device 787 * @flag: return value 788 * 789 * Each device might have its preferred reset type: fundamental or 790 * hot reset. The routine is used to collected the information for 791 * the indicated device and its children so that the bunch of the 792 * devices could be reset properly. 793 */ 794 static void eeh_set_dev_freset(struct eeh_dev *edev, void *flag) 795 { 796 struct pci_dev *dev; 797 unsigned int *freset = (unsigned int *)flag; 798 799 dev = eeh_dev_to_pci_dev(edev); 800 if (dev) 801 *freset |= dev->needs_freset; 802 } 803 804 static void eeh_pe_refreeze_passed(struct eeh_pe *root) 805 { 806 struct eeh_pe *pe; 807 int state; 808 809 eeh_for_each_pe(root, pe) { 810 if (eeh_pe_passed(pe)) { 811 state = eeh_ops->get_state(pe, NULL); 812 if (state & 813 (EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED)) { 814 pr_info("EEH: Passed-through PE PHB#%x-PE#%x was thawed by reset, re-freezing for safety.\n", 815 pe->phb->global_number, pe->addr); 816 eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE); 817 } 818 } 819 } 820 } 821 822 /** 823 * eeh_pe_reset_full - Complete a full reset process on the indicated PE 824 * @pe: EEH PE 825 * 826 * This function executes a full reset procedure on a PE, including setting 827 * the appropriate flags, performing a fundamental or hot reset, and then 828 * deactivating the reset status. It is designed to be used within the EEH 829 * subsystem, as opposed to eeh_pe_reset which is exported to drivers and 830 * only performs a single operation at a time. 831 * 832 * This function will attempt to reset a PE three times before failing. 833 */ 834 int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed) 835 { 836 int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED); 837 int type = EEH_RESET_HOT; 838 unsigned int freset = 0; 839 int i, state = 0, ret; 840 841 /* 842 * Determine the type of reset to perform - hot or fundamental. 843 * Hot reset is the default operation, unless any device under the 844 * PE requires a fundamental reset. 845 */ 846 eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset); 847 848 if (freset) 849 type = EEH_RESET_FUNDAMENTAL; 850 851 /* Mark the PE as in reset state and block config space accesses */ 852 eeh_pe_state_mark(pe, reset_state); 853 854 /* Make three attempts at resetting the bus */ 855 for (i = 0; i < 3; i++) { 856 ret = eeh_pe_reset(pe, type, include_passed); 857 if (!ret) 858 ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, 859 include_passed); 860 if (ret) { 861 ret = -EIO; 862 pr_warn("EEH: Failure %d resetting PHB#%x-PE#%x (attempt %d)\n\n", 863 state, pe->phb->global_number, pe->addr, i + 1); 864 continue; 865 } 866 if (i) 867 pr_warn("EEH: PHB#%x-PE#%x: Successful reset (attempt %d)\n", 868 pe->phb->global_number, pe->addr, i + 1); 869 870 /* Wait until the PE is in a functioning state */ 871 state = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 872 if (state < 0) { 873 pr_warn("EEH: Unrecoverable slot failure on PHB#%x-PE#%x", 874 pe->phb->global_number, pe->addr); 875 ret = -ENOTRECOVERABLE; 876 break; 877 } 878 if (eeh_state_active(state)) 879 break; 880 else 881 pr_warn("EEH: PHB#%x-PE#%x: Slot inactive after reset: 0x%x (attempt %d)\n", 882 pe->phb->global_number, pe->addr, state, i + 1); 883 } 884 885 /* Resetting the PE may have unfrozen child PEs. If those PEs have been 886 * (potentially) passed through to a guest, re-freeze them: 887 */ 888 if (!include_passed) 889 eeh_pe_refreeze_passed(pe); 890 891 eeh_pe_state_clear(pe, reset_state, true); 892 return ret; 893 } 894 895 /** 896 * eeh_save_bars - Save device bars 897 * @edev: PCI device associated EEH device 898 * 899 * Save the values of the device bars. Unlike the restore 900 * routine, this routine is *not* recursive. This is because 901 * PCI devices are added individually; but, for the restore, 902 * an entire slot is reset at a time. 903 */ 904 void eeh_save_bars(struct eeh_dev *edev) 905 { 906 int i; 907 908 if (!edev) 909 return; 910 911 for (i = 0; i < 16; i++) 912 eeh_ops->read_config(edev, i * 4, 4, &edev->config_space[i]); 913 914 /* 915 * For PCI bridges including root port, we need enable bus 916 * master explicitly. Otherwise, it can't fetch IODA table 917 * entries correctly. So we cache the bit in advance so that 918 * we can restore it after reset, either PHB range or PE range. 919 */ 920 if (edev->mode & EEH_DEV_BRIDGE) 921 edev->config_space[1] |= PCI_COMMAND_MASTER; 922 } 923 924 static int eeh_reboot_notifier(struct notifier_block *nb, 925 unsigned long action, void *unused) 926 { 927 eeh_clear_flag(EEH_ENABLED); 928 return NOTIFY_DONE; 929 } 930 931 static struct notifier_block eeh_reboot_nb = { 932 .notifier_call = eeh_reboot_notifier, 933 }; 934 935 static int eeh_device_notifier(struct notifier_block *nb, 936 unsigned long action, void *data) 937 { 938 struct device *dev = data; 939 940 switch (action) { 941 /* 942 * Note: It's not possible to perform EEH device addition (i.e. 943 * {pseries,pnv}_pcibios_bus_add_device()) here because it depends on 944 * the device's resources, which have not yet been set up. 945 */ 946 case BUS_NOTIFY_DEL_DEVICE: 947 eeh_remove_device(to_pci_dev(dev)); 948 break; 949 default: 950 break; 951 } 952 return NOTIFY_DONE; 953 } 954 955 static struct notifier_block eeh_device_nb = { 956 .notifier_call = eeh_device_notifier, 957 }; 958 959 /** 960 * eeh_init - System wide EEH initialization 961 * 962 * It's the platform's job to call this from an arch_initcall(). 963 */ 964 int eeh_init(struct eeh_ops *ops) 965 { 966 struct pci_controller *hose, *tmp; 967 int ret = 0; 968 969 /* the platform should only initialise EEH once */ 970 if (WARN_ON(eeh_ops)) 971 return -EEXIST; 972 if (WARN_ON(!ops)) 973 return -ENOENT; 974 eeh_ops = ops; 975 976 /* Register reboot notifier */ 977 ret = register_reboot_notifier(&eeh_reboot_nb); 978 if (ret) { 979 pr_warn("%s: Failed to register reboot notifier (%d)\n", 980 __func__, ret); 981 return ret; 982 } 983 984 ret = bus_register_notifier(&pci_bus_type, &eeh_device_nb); 985 if (ret) { 986 pr_warn("%s: Failed to register bus notifier (%d)\n", 987 __func__, ret); 988 return ret; 989 } 990 991 /* Initialize PHB PEs */ 992 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) 993 eeh_phb_pe_create(hose); 994 995 eeh_addr_cache_init(); 996 997 /* Initialize EEH event */ 998 return eeh_event_init(); 999 } 1000 1001 /** 1002 * eeh_probe_device() - Perform EEH initialization for the indicated pci device 1003 * @dev: pci device for which to set up EEH 1004 * 1005 * This routine must be used to complete EEH initialization for PCI 1006 * devices that were added after system boot (e.g. hotplug, dlpar). 1007 */ 1008 void eeh_probe_device(struct pci_dev *dev) 1009 { 1010 struct eeh_dev *edev; 1011 1012 pr_debug("EEH: Adding device %s\n", pci_name(dev)); 1013 1014 /* 1015 * pci_dev_to_eeh_dev() can only work if eeh_probe_dev() was 1016 * already called for this device. 1017 */ 1018 if (WARN_ON_ONCE(pci_dev_to_eeh_dev(dev))) { 1019 pci_dbg(dev, "Already bound to an eeh_dev!\n"); 1020 return; 1021 } 1022 1023 edev = eeh_ops->probe(dev); 1024 if (!edev) { 1025 pr_debug("EEH: Adding device failed\n"); 1026 return; 1027 } 1028 1029 /* 1030 * FIXME: We rely on pcibios_release_device() to remove the 1031 * existing EEH state. The release function is only called if 1032 * the pci_dev's refcount drops to zero so if something is 1033 * keeping a ref to a device (e.g. a filesystem) we need to 1034 * remove the old EEH state. 1035 * 1036 * FIXME: HEY MA, LOOK AT ME, NO LOCKING! 1037 */ 1038 if (edev->pdev && edev->pdev != dev) { 1039 eeh_pe_tree_remove(edev); 1040 eeh_addr_cache_rmv_dev(edev->pdev); 1041 eeh_sysfs_remove_device(edev->pdev); 1042 1043 /* 1044 * We definitely should have the PCI device removed 1045 * though it wasn't correctly. So we needn't call 1046 * into error handler afterwards. 1047 */ 1048 edev->mode |= EEH_DEV_NO_HANDLER; 1049 } 1050 1051 /* bind the pdev and the edev together */ 1052 edev->pdev = dev; 1053 dev->dev.archdata.edev = edev; 1054 eeh_addr_cache_insert_dev(dev); 1055 eeh_sysfs_add_device(dev); 1056 } 1057 1058 /** 1059 * eeh_remove_device - Undo EEH setup for the indicated pci device 1060 * @dev: pci device to be removed 1061 * 1062 * This routine should be called when a device is removed from 1063 * a running system (e.g. by hotplug or dlpar). It unregisters 1064 * the PCI device from the EEH subsystem. I/O errors affecting 1065 * this device will no longer be detected after this call; thus, 1066 * i/o errors affecting this slot may leave this device unusable. 1067 */ 1068 void eeh_remove_device(struct pci_dev *dev) 1069 { 1070 struct eeh_dev *edev; 1071 1072 if (!dev || !eeh_enabled()) 1073 return; 1074 edev = pci_dev_to_eeh_dev(dev); 1075 1076 /* Unregister the device with the EEH/PCI address search system */ 1077 dev_dbg(&dev->dev, "EEH: Removing device\n"); 1078 1079 if (!edev || !edev->pdev || !edev->pe) { 1080 dev_dbg(&dev->dev, "EEH: Device not referenced!\n"); 1081 return; 1082 } 1083 1084 /* 1085 * During the hotplug for EEH error recovery, we need the EEH 1086 * device attached to the parent PE in order for BAR restore 1087 * a bit later. So we keep it for BAR restore and remove it 1088 * from the parent PE during the BAR resotre. 1089 */ 1090 edev->pdev = NULL; 1091 1092 /* 1093 * eeh_sysfs_remove_device() uses pci_dev_to_eeh_dev() so we need to 1094 * remove the sysfs files before clearing dev.archdata.edev 1095 */ 1096 if (edev->mode & EEH_DEV_SYSFS) 1097 eeh_sysfs_remove_device(dev); 1098 1099 /* 1100 * We're removing from the PCI subsystem, that means 1101 * the PCI device driver can't support EEH or not 1102 * well. So we rely on hotplug completely to do recovery 1103 * for the specific PCI device. 1104 */ 1105 edev->mode |= EEH_DEV_NO_HANDLER; 1106 1107 eeh_addr_cache_rmv_dev(dev); 1108 1109 /* 1110 * The flag "in_error" is used to trace EEH devices for VFs 1111 * in error state or not. It's set in eeh_report_error(). If 1112 * it's not set, eeh_report_{reset,resume}() won't be called 1113 * for the VF EEH device. 1114 */ 1115 edev->in_error = false; 1116 dev->dev.archdata.edev = NULL; 1117 if (!(edev->pe->state & EEH_PE_KEEP)) 1118 eeh_pe_tree_remove(edev); 1119 else 1120 edev->mode |= EEH_DEV_DISCONNECTED; 1121 } 1122 1123 int eeh_unfreeze_pe(struct eeh_pe *pe) 1124 { 1125 int ret; 1126 1127 ret = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 1128 if (ret) { 1129 pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n", 1130 __func__, ret, pe->phb->global_number, pe->addr); 1131 return ret; 1132 } 1133 1134 ret = eeh_pci_enable(pe, EEH_OPT_THAW_DMA); 1135 if (ret) { 1136 pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n", 1137 __func__, ret, pe->phb->global_number, pe->addr); 1138 return ret; 1139 } 1140 1141 return ret; 1142 } 1143 1144 1145 static struct pci_device_id eeh_reset_ids[] = { 1146 { PCI_DEVICE(0x19a2, 0x0710) }, /* Emulex, BE */ 1147 { PCI_DEVICE(0x10df, 0xe220) }, /* Emulex, Lancer */ 1148 { PCI_DEVICE(0x14e4, 0x1657) }, /* Broadcom BCM5719 */ 1149 { 0 } 1150 }; 1151 1152 static int eeh_pe_change_owner(struct eeh_pe *pe) 1153 { 1154 struct eeh_dev *edev, *tmp; 1155 struct pci_dev *pdev; 1156 struct pci_device_id *id; 1157 int ret; 1158 1159 /* Check PE state */ 1160 ret = eeh_ops->get_state(pe, NULL); 1161 if (ret < 0 || ret == EEH_STATE_NOT_SUPPORT) 1162 return 0; 1163 1164 /* Unfrozen PE, nothing to do */ 1165 if (eeh_state_active(ret)) 1166 return 0; 1167 1168 /* Frozen PE, check if it needs PE level reset */ 1169 eeh_pe_for_each_dev(pe, edev, tmp) { 1170 pdev = eeh_dev_to_pci_dev(edev); 1171 if (!pdev) 1172 continue; 1173 1174 for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) { 1175 if (id->vendor != PCI_ANY_ID && 1176 id->vendor != pdev->vendor) 1177 continue; 1178 if (id->device != PCI_ANY_ID && 1179 id->device != pdev->device) 1180 continue; 1181 if (id->subvendor != PCI_ANY_ID && 1182 id->subvendor != pdev->subsystem_vendor) 1183 continue; 1184 if (id->subdevice != PCI_ANY_ID && 1185 id->subdevice != pdev->subsystem_device) 1186 continue; 1187 1188 return eeh_pe_reset_and_recover(pe); 1189 } 1190 } 1191 1192 ret = eeh_unfreeze_pe(pe); 1193 if (!ret) 1194 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 1195 return ret; 1196 } 1197 1198 /** 1199 * eeh_dev_open - Increase count of pass through devices for PE 1200 * @pdev: PCI device 1201 * 1202 * Increase count of passed through devices for the indicated 1203 * PE. In the result, the EEH errors detected on the PE won't be 1204 * reported. The PE owner will be responsible for detection 1205 * and recovery. 1206 */ 1207 int eeh_dev_open(struct pci_dev *pdev) 1208 { 1209 struct eeh_dev *edev; 1210 int ret = -ENODEV; 1211 1212 mutex_lock(&eeh_dev_mutex); 1213 1214 /* No PCI device ? */ 1215 if (!pdev) 1216 goto out; 1217 1218 /* No EEH device or PE ? */ 1219 edev = pci_dev_to_eeh_dev(pdev); 1220 if (!edev || !edev->pe) 1221 goto out; 1222 1223 /* 1224 * The PE might have been put into frozen state, but we 1225 * didn't detect that yet. The passed through PCI devices 1226 * in frozen PE won't work properly. Clear the frozen state 1227 * in advance. 1228 */ 1229 ret = eeh_pe_change_owner(edev->pe); 1230 if (ret) 1231 goto out; 1232 1233 /* Increase PE's pass through count */ 1234 atomic_inc(&edev->pe->pass_dev_cnt); 1235 mutex_unlock(&eeh_dev_mutex); 1236 1237 return 0; 1238 out: 1239 mutex_unlock(&eeh_dev_mutex); 1240 return ret; 1241 } 1242 EXPORT_SYMBOL_GPL(eeh_dev_open); 1243 1244 /** 1245 * eeh_dev_release - Decrease count of pass through devices for PE 1246 * @pdev: PCI device 1247 * 1248 * Decrease count of pass through devices for the indicated PE. If 1249 * there is no passed through device in PE, the EEH errors detected 1250 * on the PE will be reported and handled as usual. 1251 */ 1252 void eeh_dev_release(struct pci_dev *pdev) 1253 { 1254 struct eeh_dev *edev; 1255 1256 mutex_lock(&eeh_dev_mutex); 1257 1258 /* No PCI device ? */ 1259 if (!pdev) 1260 goto out; 1261 1262 /* No EEH device ? */ 1263 edev = pci_dev_to_eeh_dev(pdev); 1264 if (!edev || !edev->pe || !eeh_pe_passed(edev->pe)) 1265 goto out; 1266 1267 /* Decrease PE's pass through count */ 1268 WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0); 1269 eeh_pe_change_owner(edev->pe); 1270 out: 1271 mutex_unlock(&eeh_dev_mutex); 1272 } 1273 EXPORT_SYMBOL(eeh_dev_release); 1274 1275 #ifdef CONFIG_IOMMU_API 1276 1277 static int dev_has_iommu_table(struct device *dev, void *data) 1278 { 1279 struct pci_dev *pdev = to_pci_dev(dev); 1280 struct pci_dev **ppdev = data; 1281 1282 if (!dev) 1283 return 0; 1284 1285 if (device_iommu_mapped(dev)) { 1286 *ppdev = pdev; 1287 return 1; 1288 } 1289 1290 return 0; 1291 } 1292 1293 /** 1294 * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE 1295 * @group: IOMMU group 1296 * 1297 * The routine is called to convert IOMMU group to EEH PE. 1298 */ 1299 struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group) 1300 { 1301 struct pci_dev *pdev = NULL; 1302 struct eeh_dev *edev; 1303 int ret; 1304 1305 /* No IOMMU group ? */ 1306 if (!group) 1307 return NULL; 1308 1309 ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table); 1310 if (!ret || !pdev) 1311 return NULL; 1312 1313 /* No EEH device or PE ? */ 1314 edev = pci_dev_to_eeh_dev(pdev); 1315 if (!edev || !edev->pe) 1316 return NULL; 1317 1318 return edev->pe; 1319 } 1320 EXPORT_SYMBOL_GPL(eeh_iommu_group_to_pe); 1321 1322 #endif /* CONFIG_IOMMU_API */ 1323 1324 /** 1325 * eeh_pe_set_option - Set options for the indicated PE 1326 * @pe: EEH PE 1327 * @option: requested option 1328 * 1329 * The routine is called to enable or disable EEH functionality 1330 * on the indicated PE, to enable IO or DMA for the frozen PE. 1331 */ 1332 int eeh_pe_set_option(struct eeh_pe *pe, int option) 1333 { 1334 int ret = 0; 1335 1336 /* Invalid PE ? */ 1337 if (!pe) 1338 return -ENODEV; 1339 1340 /* 1341 * EEH functionality could possibly be disabled, just 1342 * return error for the case. And the EEH functinality 1343 * isn't expected to be disabled on one specific PE. 1344 */ 1345 switch (option) { 1346 case EEH_OPT_ENABLE: 1347 if (eeh_enabled()) { 1348 ret = eeh_pe_change_owner(pe); 1349 break; 1350 } 1351 ret = -EIO; 1352 break; 1353 case EEH_OPT_DISABLE: 1354 break; 1355 case EEH_OPT_THAW_MMIO: 1356 case EEH_OPT_THAW_DMA: 1357 case EEH_OPT_FREEZE_PE: 1358 if (!eeh_ops || !eeh_ops->set_option) { 1359 ret = -ENOENT; 1360 break; 1361 } 1362 1363 ret = eeh_pci_enable(pe, option); 1364 break; 1365 default: 1366 pr_debug("%s: Option %d out of range (%d, %d)\n", 1367 __func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA); 1368 ret = -EINVAL; 1369 } 1370 1371 return ret; 1372 } 1373 EXPORT_SYMBOL_GPL(eeh_pe_set_option); 1374 1375 /** 1376 * eeh_pe_get_state - Retrieve PE's state 1377 * @pe: EEH PE 1378 * 1379 * Retrieve the PE's state, which includes 3 aspects: enabled 1380 * DMA, enabled IO and asserted reset. 1381 */ 1382 int eeh_pe_get_state(struct eeh_pe *pe) 1383 { 1384 int result, ret = 0; 1385 bool rst_active, dma_en, mmio_en; 1386 1387 /* Existing PE ? */ 1388 if (!pe) 1389 return -ENODEV; 1390 1391 if (!eeh_ops || !eeh_ops->get_state) 1392 return -ENOENT; 1393 1394 /* 1395 * If the parent PE is owned by the host kernel and is undergoing 1396 * error recovery, we should return the PE state as temporarily 1397 * unavailable so that the error recovery on the guest is suspended 1398 * until the recovery completes on the host. 1399 */ 1400 if (pe->parent && 1401 !(pe->state & EEH_PE_REMOVED) && 1402 (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING))) 1403 return EEH_PE_STATE_UNAVAIL; 1404 1405 result = eeh_ops->get_state(pe, NULL); 1406 rst_active = !!(result & EEH_STATE_RESET_ACTIVE); 1407 dma_en = !!(result & EEH_STATE_DMA_ENABLED); 1408 mmio_en = !!(result & EEH_STATE_MMIO_ENABLED); 1409 1410 if (rst_active) 1411 ret = EEH_PE_STATE_RESET; 1412 else if (dma_en && mmio_en) 1413 ret = EEH_PE_STATE_NORMAL; 1414 else if (!dma_en && !mmio_en) 1415 ret = EEH_PE_STATE_STOPPED_IO_DMA; 1416 else if (!dma_en && mmio_en) 1417 ret = EEH_PE_STATE_STOPPED_DMA; 1418 else 1419 ret = EEH_PE_STATE_UNAVAIL; 1420 1421 return ret; 1422 } 1423 EXPORT_SYMBOL_GPL(eeh_pe_get_state); 1424 1425 static int eeh_pe_reenable_devices(struct eeh_pe *pe, bool include_passed) 1426 { 1427 struct eeh_dev *edev, *tmp; 1428 struct pci_dev *pdev; 1429 int ret = 0; 1430 1431 eeh_pe_restore_bars(pe); 1432 1433 /* 1434 * Reenable PCI devices as the devices passed 1435 * through are always enabled before the reset. 1436 */ 1437 eeh_pe_for_each_dev(pe, edev, tmp) { 1438 pdev = eeh_dev_to_pci_dev(edev); 1439 if (!pdev) 1440 continue; 1441 1442 ret = pci_reenable_device(pdev); 1443 if (ret) { 1444 pr_warn("%s: Failure %d reenabling %s\n", 1445 __func__, ret, pci_name(pdev)); 1446 return ret; 1447 } 1448 } 1449 1450 /* The PE is still in frozen state */ 1451 if (include_passed || !eeh_pe_passed(pe)) { 1452 ret = eeh_unfreeze_pe(pe); 1453 } else 1454 pr_info("EEH: Note: Leaving passthrough PHB#%x-PE#%x frozen.\n", 1455 pe->phb->global_number, pe->addr); 1456 if (!ret) 1457 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, include_passed); 1458 return ret; 1459 } 1460 1461 1462 /** 1463 * eeh_pe_reset - Issue PE reset according to specified type 1464 * @pe: EEH PE 1465 * @option: reset type 1466 * 1467 * The routine is called to reset the specified PE with the 1468 * indicated type, either fundamental reset or hot reset. 1469 * PE reset is the most important part for error recovery. 1470 */ 1471 int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed) 1472 { 1473 int ret = 0; 1474 1475 /* Invalid PE ? */ 1476 if (!pe) 1477 return -ENODEV; 1478 1479 if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset) 1480 return -ENOENT; 1481 1482 switch (option) { 1483 case EEH_RESET_DEACTIVATE: 1484 ret = eeh_ops->reset(pe, option); 1485 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, include_passed); 1486 if (ret) 1487 break; 1488 1489 ret = eeh_pe_reenable_devices(pe, include_passed); 1490 break; 1491 case EEH_RESET_HOT: 1492 case EEH_RESET_FUNDAMENTAL: 1493 /* 1494 * Proactively freeze the PE to drop all MMIO access 1495 * during reset, which should be banned as it's always 1496 * cause recursive EEH error. 1497 */ 1498 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 1499 1500 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 1501 ret = eeh_ops->reset(pe, option); 1502 break; 1503 default: 1504 pr_debug("%s: Unsupported option %d\n", 1505 __func__, option); 1506 ret = -EINVAL; 1507 } 1508 1509 return ret; 1510 } 1511 EXPORT_SYMBOL_GPL(eeh_pe_reset); 1512 1513 /** 1514 * eeh_pe_configure - Configure PCI bridges after PE reset 1515 * @pe: EEH PE 1516 * 1517 * The routine is called to restore the PCI config space for 1518 * those PCI devices, especially PCI bridges affected by PE 1519 * reset issued previously. 1520 */ 1521 int eeh_pe_configure(struct eeh_pe *pe) 1522 { 1523 int ret = 0; 1524 1525 /* Invalid PE ? */ 1526 if (!pe) 1527 return -ENODEV; 1528 1529 return ret; 1530 } 1531 EXPORT_SYMBOL_GPL(eeh_pe_configure); 1532 1533 /** 1534 * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE 1535 * @pe: the indicated PE 1536 * @type: error type 1537 * @function: error function 1538 * @addr: address 1539 * @mask: address mask 1540 * 1541 * The routine is called to inject the specified PCI error, which 1542 * is determined by @type and @function, to the indicated PE for 1543 * testing purpose. 1544 */ 1545 int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, 1546 unsigned long addr, unsigned long mask) 1547 { 1548 /* Invalid PE ? */ 1549 if (!pe) 1550 return -ENODEV; 1551 1552 /* Unsupported operation ? */ 1553 if (!eeh_ops || !eeh_ops->err_inject) 1554 return -ENOENT; 1555 1556 /* Check on PCI error type */ 1557 if (type != EEH_ERR_TYPE_32 && type != EEH_ERR_TYPE_64) 1558 return -EINVAL; 1559 1560 /* Check on PCI error function */ 1561 if (func < EEH_ERR_FUNC_MIN || func > EEH_ERR_FUNC_MAX) 1562 return -EINVAL; 1563 1564 return eeh_ops->err_inject(pe, type, func, addr, mask); 1565 } 1566 EXPORT_SYMBOL_GPL(eeh_pe_inject_err); 1567 1568 #ifdef CONFIG_PROC_FS 1569 static int proc_eeh_show(struct seq_file *m, void *v) 1570 { 1571 if (!eeh_enabled()) { 1572 seq_printf(m, "EEH Subsystem is globally disabled\n"); 1573 seq_printf(m, "eeh_total_mmio_ffs=%llu\n", eeh_stats.total_mmio_ffs); 1574 } else { 1575 seq_printf(m, "EEH Subsystem is enabled\n"); 1576 seq_printf(m, 1577 "no device=%llu\n" 1578 "no device node=%llu\n" 1579 "no config address=%llu\n" 1580 "check not wanted=%llu\n" 1581 "eeh_total_mmio_ffs=%llu\n" 1582 "eeh_false_positives=%llu\n" 1583 "eeh_slot_resets=%llu\n", 1584 eeh_stats.no_device, 1585 eeh_stats.no_dn, 1586 eeh_stats.no_cfg_addr, 1587 eeh_stats.ignored_check, 1588 eeh_stats.total_mmio_ffs, 1589 eeh_stats.false_positives, 1590 eeh_stats.slot_resets); 1591 } 1592 1593 return 0; 1594 } 1595 #endif /* CONFIG_PROC_FS */ 1596 1597 #ifdef CONFIG_DEBUG_FS 1598 1599 1600 static struct pci_dev *eeh_debug_lookup_pdev(struct file *filp, 1601 const char __user *user_buf, 1602 size_t count, loff_t *ppos) 1603 { 1604 uint32_t domain, bus, dev, fn; 1605 struct pci_dev *pdev; 1606 char buf[20]; 1607 int ret; 1608 1609 memset(buf, 0, sizeof(buf)); 1610 ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); 1611 if (!ret) 1612 return ERR_PTR(-EFAULT); 1613 1614 ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); 1615 if (ret != 4) { 1616 pr_err("%s: expected 4 args, got %d\n", __func__, ret); 1617 return ERR_PTR(-EINVAL); 1618 } 1619 1620 pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); 1621 if (!pdev) 1622 return ERR_PTR(-ENODEV); 1623 1624 return pdev; 1625 } 1626 1627 static int eeh_enable_dbgfs_set(void *data, u64 val) 1628 { 1629 if (val) 1630 eeh_clear_flag(EEH_FORCE_DISABLED); 1631 else 1632 eeh_add_flag(EEH_FORCE_DISABLED); 1633 1634 return 0; 1635 } 1636 1637 static int eeh_enable_dbgfs_get(void *data, u64 *val) 1638 { 1639 if (eeh_enabled()) 1640 *val = 0x1ul; 1641 else 1642 *val = 0x0ul; 1643 return 0; 1644 } 1645 1646 DEFINE_DEBUGFS_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get, 1647 eeh_enable_dbgfs_set, "0x%llx\n"); 1648 1649 static ssize_t eeh_force_recover_write(struct file *filp, 1650 const char __user *user_buf, 1651 size_t count, loff_t *ppos) 1652 { 1653 struct pci_controller *hose; 1654 uint32_t phbid, pe_no; 1655 struct eeh_pe *pe; 1656 char buf[20]; 1657 int ret; 1658 1659 ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count); 1660 if (!ret) 1661 return -EFAULT; 1662 1663 /* 1664 * When PE is NULL the event is a "special" event. Rather than 1665 * recovering a specific PE it forces the EEH core to scan for failed 1666 * PHBs and recovers each. This needs to be done before any device 1667 * recoveries can occur. 1668 */ 1669 if (!strncmp(buf, "hwcheck", 7)) { 1670 __eeh_send_failure_event(NULL); 1671 return count; 1672 } 1673 1674 ret = sscanf(buf, "%x:%x", &phbid, &pe_no); 1675 if (ret != 2) 1676 return -EINVAL; 1677 1678 hose = pci_find_controller_for_domain(phbid); 1679 if (!hose) 1680 return -ENODEV; 1681 1682 /* Retrieve PE */ 1683 pe = eeh_pe_get(hose, pe_no); 1684 if (!pe) 1685 return -ENODEV; 1686 1687 /* 1688 * We don't do any state checking here since the detection 1689 * process is async to the recovery process. The recovery 1690 * thread *should* not break even if we schedule a recovery 1691 * from an odd state (e.g. PE removed, or recovery of a 1692 * non-isolated PE) 1693 */ 1694 __eeh_send_failure_event(pe); 1695 1696 return ret < 0 ? ret : count; 1697 } 1698 1699 static const struct file_operations eeh_force_recover_fops = { 1700 .open = simple_open, 1701 .llseek = no_llseek, 1702 .write = eeh_force_recover_write, 1703 }; 1704 1705 static ssize_t eeh_debugfs_dev_usage(struct file *filp, 1706 char __user *user_buf, 1707 size_t count, loff_t *ppos) 1708 { 1709 static const char usage[] = "input format: <domain>:<bus>:<dev>.<fn>\n"; 1710 1711 return simple_read_from_buffer(user_buf, count, ppos, 1712 usage, sizeof(usage) - 1); 1713 } 1714 1715 static ssize_t eeh_dev_check_write(struct file *filp, 1716 const char __user *user_buf, 1717 size_t count, loff_t *ppos) 1718 { 1719 struct pci_dev *pdev; 1720 struct eeh_dev *edev; 1721 int ret; 1722 1723 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1724 if (IS_ERR(pdev)) 1725 return PTR_ERR(pdev); 1726 1727 edev = pci_dev_to_eeh_dev(pdev); 1728 if (!edev) { 1729 pci_err(pdev, "No eeh_dev for this device!\n"); 1730 pci_dev_put(pdev); 1731 return -ENODEV; 1732 } 1733 1734 ret = eeh_dev_check_failure(edev); 1735 pci_info(pdev, "eeh_dev_check_failure(%s) = %d\n", 1736 pci_name(pdev), ret); 1737 1738 pci_dev_put(pdev); 1739 1740 return count; 1741 } 1742 1743 static const struct file_operations eeh_dev_check_fops = { 1744 .open = simple_open, 1745 .llseek = no_llseek, 1746 .write = eeh_dev_check_write, 1747 .read = eeh_debugfs_dev_usage, 1748 }; 1749 1750 static int eeh_debugfs_break_device(struct pci_dev *pdev) 1751 { 1752 struct resource *bar = NULL; 1753 void __iomem *mapped; 1754 u16 old, bit; 1755 int i, pos; 1756 1757 /* Do we have an MMIO BAR to disable? */ 1758 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { 1759 struct resource *r = &pdev->resource[i]; 1760 1761 if (!r->flags || !r->start) 1762 continue; 1763 if (r->flags & IORESOURCE_IO) 1764 continue; 1765 if (r->flags & IORESOURCE_UNSET) 1766 continue; 1767 1768 bar = r; 1769 break; 1770 } 1771 1772 if (!bar) { 1773 pci_err(pdev, "Unable to find Memory BAR to cause EEH with\n"); 1774 return -ENXIO; 1775 } 1776 1777 pci_err(pdev, "Going to break: %pR\n", bar); 1778 1779 if (pdev->is_virtfn) { 1780 #ifndef CONFIG_PCI_IOV 1781 return -ENXIO; 1782 #else 1783 /* 1784 * VFs don't have a per-function COMMAND register, so the best 1785 * we can do is clear the Memory Space Enable bit in the PF's 1786 * SRIOV control reg. 1787 * 1788 * Unfortunately, this requires that we have a PF (i.e doesn't 1789 * work for a passed-through VF) and it has the potential side 1790 * effect of also causing an EEH on every other VF under the 1791 * PF. Oh well. 1792 */ 1793 pdev = pdev->physfn; 1794 if (!pdev) 1795 return -ENXIO; /* passed through VFs have no PF */ 1796 1797 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 1798 pos += PCI_SRIOV_CTRL; 1799 bit = PCI_SRIOV_CTRL_MSE; 1800 #endif /* !CONFIG_PCI_IOV */ 1801 } else { 1802 bit = PCI_COMMAND_MEMORY; 1803 pos = PCI_COMMAND; 1804 } 1805 1806 /* 1807 * Process here is: 1808 * 1809 * 1. Disable Memory space. 1810 * 1811 * 2. Perform an MMIO to the device. This should result in an error 1812 * (CA / UR) being raised by the device which results in an EEH 1813 * PE freeze. Using the in_8() accessor skips the eeh detection hook 1814 * so the freeze hook so the EEH Detection machinery won't be 1815 * triggered here. This is to match the usual behaviour of EEH 1816 * where the HW will asyncronously freeze a PE and it's up to 1817 * the kernel to notice and deal with it. 1818 * 1819 * 3. Turn Memory space back on. This is more important for VFs 1820 * since recovery will probably fail if we don't. For normal 1821 * the COMMAND register is reset as a part of re-initialising 1822 * the device. 1823 * 1824 * Breaking stuff is the point so who cares if it's racy ;) 1825 */ 1826 pci_read_config_word(pdev, pos, &old); 1827 1828 mapped = ioremap(bar->start, PAGE_SIZE); 1829 if (!mapped) { 1830 pci_err(pdev, "Unable to map MMIO BAR %pR\n", bar); 1831 return -ENXIO; 1832 } 1833 1834 pci_write_config_word(pdev, pos, old & ~bit); 1835 in_8(mapped); 1836 pci_write_config_word(pdev, pos, old); 1837 1838 iounmap(mapped); 1839 1840 return 0; 1841 } 1842 1843 static ssize_t eeh_dev_break_write(struct file *filp, 1844 const char __user *user_buf, 1845 size_t count, loff_t *ppos) 1846 { 1847 struct pci_dev *pdev; 1848 int ret; 1849 1850 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1851 if (IS_ERR(pdev)) 1852 return PTR_ERR(pdev); 1853 1854 ret = eeh_debugfs_break_device(pdev); 1855 pci_dev_put(pdev); 1856 1857 if (ret < 0) 1858 return ret; 1859 1860 return count; 1861 } 1862 1863 static const struct file_operations eeh_dev_break_fops = { 1864 .open = simple_open, 1865 .llseek = no_llseek, 1866 .write = eeh_dev_break_write, 1867 .read = eeh_debugfs_dev_usage, 1868 }; 1869 1870 static ssize_t eeh_dev_can_recover(struct file *filp, 1871 const char __user *user_buf, 1872 size_t count, loff_t *ppos) 1873 { 1874 struct pci_driver *drv; 1875 struct pci_dev *pdev; 1876 size_t ret; 1877 1878 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1879 if (IS_ERR(pdev)) 1880 return PTR_ERR(pdev); 1881 1882 /* 1883 * In order for error recovery to work the driver needs to implement 1884 * .error_detected(), so it can quiesce IO to the device, and 1885 * .slot_reset() so it can re-initialise the device after a reset. 1886 * 1887 * Ideally they'd implement .resume() too, but some drivers which 1888 * we need to support (notably IPR) don't so I guess we can tolerate 1889 * that. 1890 * 1891 * .mmio_enabled() is mostly there as a work-around for devices which 1892 * take forever to re-init after a hot reset. Implementing that is 1893 * strictly optional. 1894 */ 1895 drv = pci_dev_driver(pdev); 1896 if (drv && 1897 drv->err_handler && 1898 drv->err_handler->error_detected && 1899 drv->err_handler->slot_reset) { 1900 ret = count; 1901 } else { 1902 ret = -EOPNOTSUPP; 1903 } 1904 1905 pci_dev_put(pdev); 1906 1907 return ret; 1908 } 1909 1910 static const struct file_operations eeh_dev_can_recover_fops = { 1911 .open = simple_open, 1912 .llseek = no_llseek, 1913 .write = eeh_dev_can_recover, 1914 .read = eeh_debugfs_dev_usage, 1915 }; 1916 1917 #endif 1918 1919 static int __init eeh_init_proc(void) 1920 { 1921 if (machine_is(pseries) || machine_is(powernv)) { 1922 proc_create_single("powerpc/eeh", 0, NULL, proc_eeh_show); 1923 #ifdef CONFIG_DEBUG_FS 1924 debugfs_create_file_unsafe("eeh_enable", 0600, 1925 powerpc_debugfs_root, NULL, 1926 &eeh_enable_dbgfs_ops); 1927 debugfs_create_u32("eeh_max_freezes", 0600, 1928 powerpc_debugfs_root, &eeh_max_freezes); 1929 debugfs_create_bool("eeh_disable_recovery", 0600, 1930 powerpc_debugfs_root, 1931 &eeh_debugfs_no_recover); 1932 debugfs_create_file_unsafe("eeh_dev_check", 0600, 1933 powerpc_debugfs_root, NULL, 1934 &eeh_dev_check_fops); 1935 debugfs_create_file_unsafe("eeh_dev_break", 0600, 1936 powerpc_debugfs_root, NULL, 1937 &eeh_dev_break_fops); 1938 debugfs_create_file_unsafe("eeh_force_recover", 0600, 1939 powerpc_debugfs_root, NULL, 1940 &eeh_force_recover_fops); 1941 debugfs_create_file_unsafe("eeh_dev_can_recover", 0600, 1942 powerpc_debugfs_root, NULL, 1943 &eeh_dev_can_recover_fops); 1944 eeh_cache_debugfs_init(); 1945 #endif 1946 } 1947 1948 return 0; 1949 } 1950 __initcall(eeh_init_proc); 1951