xref: /openbmc/linux/arch/powerpc/kernel/eeh.c (revision 4cff79e9)
1 /*
2  * Copyright IBM Corporation 2001, 2005, 2006
3  * Copyright Dave Engebretsen & Todd Inglett 2001
4  * Copyright Linas Vepstas 2005, 2006
5  * Copyright 2001-2012 IBM Corporation.
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
20  *
21  * Please address comments and feedback to Linas Vepstas <linas@austin.ibm.com>
22  */
23 
24 #include <linux/delay.h>
25 #include <linux/sched.h>
26 #include <linux/init.h>
27 #include <linux/list.h>
28 #include <linux/pci.h>
29 #include <linux/iommu.h>
30 #include <linux/proc_fs.h>
31 #include <linux/rbtree.h>
32 #include <linux/reboot.h>
33 #include <linux/seq_file.h>
34 #include <linux/spinlock.h>
35 #include <linux/export.h>
36 #include <linux/of.h>
37 
38 #include <linux/atomic.h>
39 #include <asm/debugfs.h>
40 #include <asm/eeh.h>
41 #include <asm/eeh_event.h>
42 #include <asm/io.h>
43 #include <asm/iommu.h>
44 #include <asm/machdep.h>
45 #include <asm/ppc-pci.h>
46 #include <asm/rtas.h>
47 #include <asm/pte-walk.h>
48 
49 
50 /** Overview:
51  *  EEH, or "Enhanced Error Handling" is a PCI bridge technology for
52  *  dealing with PCI bus errors that can't be dealt with within the
53  *  usual PCI framework, except by check-stopping the CPU.  Systems
54  *  that are designed for high-availability/reliability cannot afford
55  *  to crash due to a "mere" PCI error, thus the need for EEH.
56  *  An EEH-capable bridge operates by converting a detected error
57  *  into a "slot freeze", taking the PCI adapter off-line, making
58  *  the slot behave, from the OS'es point of view, as if the slot
59  *  were "empty": all reads return 0xff's and all writes are silently
60  *  ignored.  EEH slot isolation events can be triggered by parity
61  *  errors on the address or data busses (e.g. during posted writes),
62  *  which in turn might be caused by low voltage on the bus, dust,
63  *  vibration, humidity, radioactivity or plain-old failed hardware.
64  *
65  *  Note, however, that one of the leading causes of EEH slot
66  *  freeze events are buggy device drivers, buggy device microcode,
67  *  or buggy device hardware.  This is because any attempt by the
68  *  device to bus-master data to a memory address that is not
69  *  assigned to the device will trigger a slot freeze.   (The idea
70  *  is to prevent devices-gone-wild from corrupting system memory).
71  *  Buggy hardware/drivers will have a miserable time co-existing
72  *  with EEH.
73  *
74  *  Ideally, a PCI device driver, when suspecting that an isolation
75  *  event has occurred (e.g. by reading 0xff's), will then ask EEH
76  *  whether this is the case, and then take appropriate steps to
77  *  reset the PCI slot, the PCI device, and then resume operations.
78  *  However, until that day,  the checking is done here, with the
79  *  eeh_check_failure() routine embedded in the MMIO macros.  If
80  *  the slot is found to be isolated, an "EEH Event" is synthesized
81  *  and sent out for processing.
82  */
83 
84 /* If a device driver keeps reading an MMIO register in an interrupt
85  * handler after a slot isolation event, it might be broken.
86  * This sets the threshold for how many read attempts we allow
87  * before printing an error message.
88  */
89 #define EEH_MAX_FAILS	2100000
90 
91 /* Time to wait for a PCI slot to report status, in milliseconds */
92 #define PCI_BUS_RESET_WAIT_MSEC (5*60*1000)
93 
94 /*
95  * EEH probe mode support, which is part of the flags,
96  * is to support multiple platforms for EEH. Some platforms
97  * like pSeries do PCI emunation based on device tree.
98  * However, other platforms like powernv probe PCI devices
99  * from hardware. The flag is used to distinguish that.
100  * In addition, struct eeh_ops::probe would be invoked for
101  * particular OF node or PCI device so that the corresponding
102  * PE would be created there.
103  */
104 int eeh_subsystem_flags;
105 EXPORT_SYMBOL(eeh_subsystem_flags);
106 
107 /*
108  * EEH allowed maximal frozen times. If one particular PE's
109  * frozen count in last hour exceeds this limit, the PE will
110  * be forced to be offline permanently.
111  */
112 int eeh_max_freezes = 5;
113 
114 /* Platform dependent EEH operations */
115 struct eeh_ops *eeh_ops = NULL;
116 
117 /* Lock to avoid races due to multiple reports of an error */
118 DEFINE_RAW_SPINLOCK(confirm_error_lock);
119 EXPORT_SYMBOL_GPL(confirm_error_lock);
120 
121 /* Lock to protect passed flags */
122 static DEFINE_MUTEX(eeh_dev_mutex);
123 
124 /* Buffer for reporting pci register dumps. Its here in BSS, and
125  * not dynamically alloced, so that it ends up in RMO where RTAS
126  * can access it.
127  */
128 #define EEH_PCI_REGS_LOG_LEN 8192
129 static unsigned char pci_regs_buf[EEH_PCI_REGS_LOG_LEN];
130 
131 /*
132  * The struct is used to maintain the EEH global statistic
133  * information. Besides, the EEH global statistics will be
134  * exported to user space through procfs
135  */
136 struct eeh_stats {
137 	u64 no_device;		/* PCI device not found		*/
138 	u64 no_dn;		/* OF node not found		*/
139 	u64 no_cfg_addr;	/* Config address not found	*/
140 	u64 ignored_check;	/* EEH check skipped		*/
141 	u64 total_mmio_ffs;	/* Total EEH checks		*/
142 	u64 false_positives;	/* Unnecessary EEH checks	*/
143 	u64 slot_resets;	/* PE reset			*/
144 };
145 
146 static struct eeh_stats eeh_stats;
147 
148 static int __init eeh_setup(char *str)
149 {
150 	if (!strcmp(str, "off"))
151 		eeh_add_flag(EEH_FORCE_DISABLED);
152 	else if (!strcmp(str, "early_log"))
153 		eeh_add_flag(EEH_EARLY_DUMP_LOG);
154 
155 	return 1;
156 }
157 __setup("eeh=", eeh_setup);
158 
159 /*
160  * This routine captures assorted PCI configuration space data
161  * for the indicated PCI device, and puts them into a buffer
162  * for RTAS error logging.
163  */
164 static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len)
165 {
166 	struct pci_dn *pdn = eeh_dev_to_pdn(edev);
167 	u32 cfg;
168 	int cap, i;
169 	int n = 0, l = 0;
170 	char buffer[128];
171 
172 	n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n",
173 		       pdn->phb->global_number, pdn->busno,
174 		       PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn));
175 	pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n",
176 		pdn->phb->global_number, pdn->busno,
177 		PCI_SLOT(pdn->devfn), PCI_FUNC(pdn->devfn));
178 
179 	eeh_ops->read_config(pdn, PCI_VENDOR_ID, 4, &cfg);
180 	n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg);
181 	pr_warn("EEH: PCI device/vendor: %08x\n", cfg);
182 
183 	eeh_ops->read_config(pdn, PCI_COMMAND, 4, &cfg);
184 	n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg);
185 	pr_warn("EEH: PCI cmd/status register: %08x\n", cfg);
186 
187 	/* Gather bridge-specific registers */
188 	if (edev->mode & EEH_DEV_BRIDGE) {
189 		eeh_ops->read_config(pdn, PCI_SEC_STATUS, 2, &cfg);
190 		n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg);
191 		pr_warn("EEH: Bridge secondary status: %04x\n", cfg);
192 
193 		eeh_ops->read_config(pdn, PCI_BRIDGE_CONTROL, 2, &cfg);
194 		n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg);
195 		pr_warn("EEH: Bridge control: %04x\n", cfg);
196 	}
197 
198 	/* Dump out the PCI-X command and status regs */
199 	cap = edev->pcix_cap;
200 	if (cap) {
201 		eeh_ops->read_config(pdn, cap, 4, &cfg);
202 		n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg);
203 		pr_warn("EEH: PCI-X cmd: %08x\n", cfg);
204 
205 		eeh_ops->read_config(pdn, cap+4, 4, &cfg);
206 		n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg);
207 		pr_warn("EEH: PCI-X status: %08x\n", cfg);
208 	}
209 
210 	/* If PCI-E capable, dump PCI-E cap 10 */
211 	cap = edev->pcie_cap;
212 	if (cap) {
213 		n += scnprintf(buf+n, len-n, "pci-e cap10:\n");
214 		pr_warn("EEH: PCI-E capabilities and status follow:\n");
215 
216 		for (i=0; i<=8; i++) {
217 			eeh_ops->read_config(pdn, cap+4*i, 4, &cfg);
218 			n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);
219 
220 			if ((i % 4) == 0) {
221 				if (i != 0)
222 					pr_warn("%s\n", buffer);
223 
224 				l = scnprintf(buffer, sizeof(buffer),
225 					      "EEH: PCI-E %02x: %08x ",
226 					      4*i, cfg);
227 			} else {
228 				l += scnprintf(buffer+l, sizeof(buffer)-l,
229 					       "%08x ", cfg);
230 			}
231 
232 		}
233 
234 		pr_warn("%s\n", buffer);
235 	}
236 
237 	/* If AER capable, dump it */
238 	cap = edev->aer_cap;
239 	if (cap) {
240 		n += scnprintf(buf+n, len-n, "pci-e AER:\n");
241 		pr_warn("EEH: PCI-E AER capability register set follows:\n");
242 
243 		for (i=0; i<=13; i++) {
244 			eeh_ops->read_config(pdn, cap+4*i, 4, &cfg);
245 			n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);
246 
247 			if ((i % 4) == 0) {
248 				if (i != 0)
249 					pr_warn("%s\n", buffer);
250 
251 				l = scnprintf(buffer, sizeof(buffer),
252 					      "EEH: PCI-E AER %02x: %08x ",
253 					      4*i, cfg);
254 			} else {
255 				l += scnprintf(buffer+l, sizeof(buffer)-l,
256 					       "%08x ", cfg);
257 			}
258 		}
259 
260 		pr_warn("%s\n", buffer);
261 	}
262 
263 	return n;
264 }
265 
266 static void *eeh_dump_pe_log(void *data, void *flag)
267 {
268 	struct eeh_pe *pe = data;
269 	struct eeh_dev *edev, *tmp;
270 	size_t *plen = flag;
271 
272 	eeh_pe_for_each_dev(pe, edev, tmp)
273 		*plen += eeh_dump_dev_log(edev, pci_regs_buf + *plen,
274 					  EEH_PCI_REGS_LOG_LEN - *plen);
275 
276 	return NULL;
277 }
278 
279 /**
280  * eeh_slot_error_detail - Generate combined log including driver log and error log
281  * @pe: EEH PE
282  * @severity: temporary or permanent error log
283  *
284  * This routine should be called to generate the combined log, which
285  * is comprised of driver log and error log. The driver log is figured
286  * out from the config space of the corresponding PCI device, while
287  * the error log is fetched through platform dependent function call.
288  */
289 void eeh_slot_error_detail(struct eeh_pe *pe, int severity)
290 {
291 	size_t loglen = 0;
292 
293 	/*
294 	 * When the PHB is fenced or dead, it's pointless to collect
295 	 * the data from PCI config space because it should return
296 	 * 0xFF's. For ER, we still retrieve the data from the PCI
297 	 * config space.
298 	 *
299 	 * For pHyp, we have to enable IO for log retrieval. Otherwise,
300 	 * 0xFF's is always returned from PCI config space.
301 	 *
302 	 * When the @severity is EEH_LOG_PERM, the PE is going to be
303 	 * removed. Prior to that, the drivers for devices included in
304 	 * the PE will be closed. The drivers rely on working IO path
305 	 * to bring the devices to quiet state. Otherwise, PCI traffic
306 	 * from those devices after they are removed is like to cause
307 	 * another unexpected EEH error.
308 	 */
309 	if (!(pe->type & EEH_PE_PHB)) {
310 		if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG) ||
311 		    severity == EEH_LOG_PERM)
312 			eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);
313 
314 		/*
315 		 * The config space of some PCI devices can't be accessed
316 		 * when their PEs are in frozen state. Otherwise, fenced
317 		 * PHB might be seen. Those PEs are identified with flag
318 		 * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED
319 		 * is set automatically when the PE is put to EEH_PE_ISOLATED.
320 		 *
321 		 * Restoring BARs possibly triggers PCI config access in
322 		 * (OPAL) firmware and then causes fenced PHB. If the
323 		 * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's
324 		 * pointless to restore BARs and dump config space.
325 		 */
326 		eeh_ops->configure_bridge(pe);
327 		if (!(pe->state & EEH_PE_CFG_BLOCKED)) {
328 			eeh_pe_restore_bars(pe);
329 
330 			pci_regs_buf[0] = 0;
331 			eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen);
332 		}
333 	}
334 
335 	eeh_ops->get_log(pe, severity, pci_regs_buf, loglen);
336 }
337 
338 /**
339  * eeh_token_to_phys - Convert EEH address token to phys address
340  * @token: I/O token, should be address in the form 0xA....
341  *
342  * This routine should be called to convert virtual I/O address
343  * to physical one.
344  */
345 static inline unsigned long eeh_token_to_phys(unsigned long token)
346 {
347 	pte_t *ptep;
348 	unsigned long pa;
349 	int hugepage_shift;
350 
351 	/*
352 	 * We won't find hugepages here(this is iomem). Hence we are not
353 	 * worried about _PAGE_SPLITTING/collapse. Also we will not hit
354 	 * page table free, because of init_mm.
355 	 */
356 	ptep = find_init_mm_pte(token, &hugepage_shift);
357 	if (!ptep)
358 		return token;
359 	WARN_ON(hugepage_shift);
360 	pa = pte_pfn(*ptep) << PAGE_SHIFT;
361 
362 	return pa | (token & (PAGE_SIZE-1));
363 }
364 
365 /*
366  * On PowerNV platform, we might already have fenced PHB there.
367  * For that case, it's meaningless to recover frozen PE. Intead,
368  * We have to handle fenced PHB firstly.
369  */
370 static int eeh_phb_check_failure(struct eeh_pe *pe)
371 {
372 	struct eeh_pe *phb_pe;
373 	unsigned long flags;
374 	int ret;
375 
376 	if (!eeh_has_flag(EEH_PROBE_MODE_DEV))
377 		return -EPERM;
378 
379 	/* Find the PHB PE */
380 	phb_pe = eeh_phb_pe_get(pe->phb);
381 	if (!phb_pe) {
382 		pr_warn("%s Can't find PE for PHB#%x\n",
383 			__func__, pe->phb->global_number);
384 		return -EEXIST;
385 	}
386 
387 	/* If the PHB has been in problematic state */
388 	eeh_serialize_lock(&flags);
389 	if (phb_pe->state & EEH_PE_ISOLATED) {
390 		ret = 0;
391 		goto out;
392 	}
393 
394 	/* Check PHB state */
395 	ret = eeh_ops->get_state(phb_pe, NULL);
396 	if ((ret < 0) ||
397 	    (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) {
398 		ret = 0;
399 		goto out;
400 	}
401 
402 	/* Isolate the PHB and send event */
403 	eeh_pe_state_mark(phb_pe, EEH_PE_ISOLATED);
404 	eeh_serialize_unlock(flags);
405 
406 	pr_err("EEH: PHB#%x failure detected, location: %s\n",
407 		phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe));
408 	dump_stack();
409 	eeh_send_failure_event(phb_pe);
410 
411 	return 1;
412 out:
413 	eeh_serialize_unlock(flags);
414 	return ret;
415 }
416 
417 /**
418  * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze
419  * @edev: eeh device
420  *
421  * Check for an EEH failure for the given device node.  Call this
422  * routine if the result of a read was all 0xff's and you want to
423  * find out if this is due to an EEH slot freeze.  This routine
424  * will query firmware for the EEH status.
425  *
426  * Returns 0 if there has not been an EEH error; otherwise returns
427  * a non-zero value and queues up a slot isolation event notification.
428  *
429  * It is safe to call this routine in an interrupt context.
430  */
431 int eeh_dev_check_failure(struct eeh_dev *edev)
432 {
433 	int ret;
434 	unsigned long flags;
435 	struct device_node *dn;
436 	struct pci_dev *dev;
437 	struct eeh_pe *pe, *parent_pe, *phb_pe;
438 	int rc = 0;
439 	const char *location = NULL;
440 
441 	eeh_stats.total_mmio_ffs++;
442 
443 	if (!eeh_enabled())
444 		return 0;
445 
446 	if (!edev) {
447 		eeh_stats.no_dn++;
448 		return 0;
449 	}
450 	dev = eeh_dev_to_pci_dev(edev);
451 	pe = eeh_dev_to_pe(edev);
452 
453 	/* Access to IO BARs might get this far and still not want checking. */
454 	if (!pe) {
455 		eeh_stats.ignored_check++;
456 		pr_debug("EEH: Ignored check for %s\n",
457 			eeh_pci_name(dev));
458 		return 0;
459 	}
460 
461 	if (!pe->addr && !pe->config_addr) {
462 		eeh_stats.no_cfg_addr++;
463 		return 0;
464 	}
465 
466 	/*
467 	 * On PowerNV platform, we might already have fenced PHB
468 	 * there and we need take care of that firstly.
469 	 */
470 	ret = eeh_phb_check_failure(pe);
471 	if (ret > 0)
472 		return ret;
473 
474 	/*
475 	 * If the PE isn't owned by us, we shouldn't check the
476 	 * state. Instead, let the owner handle it if the PE has
477 	 * been frozen.
478 	 */
479 	if (eeh_pe_passed(pe))
480 		return 0;
481 
482 	/* If we already have a pending isolation event for this
483 	 * slot, we know it's bad already, we don't need to check.
484 	 * Do this checking under a lock; as multiple PCI devices
485 	 * in one slot might report errors simultaneously, and we
486 	 * only want one error recovery routine running.
487 	 */
488 	eeh_serialize_lock(&flags);
489 	rc = 1;
490 	if (pe->state & EEH_PE_ISOLATED) {
491 		pe->check_count++;
492 		if (pe->check_count % EEH_MAX_FAILS == 0) {
493 			dn = pci_device_to_OF_node(dev);
494 			if (dn)
495 				location = of_get_property(dn, "ibm,loc-code",
496 						NULL);
497 			printk(KERN_ERR "EEH: %d reads ignored for recovering device at "
498 				"location=%s driver=%s pci addr=%s\n",
499 				pe->check_count,
500 				location ? location : "unknown",
501 				eeh_driver_name(dev), eeh_pci_name(dev));
502 			printk(KERN_ERR "EEH: Might be infinite loop in %s driver\n",
503 				eeh_driver_name(dev));
504 			dump_stack();
505 		}
506 		goto dn_unlock;
507 	}
508 
509 	/*
510 	 * Now test for an EEH failure.  This is VERY expensive.
511 	 * Note that the eeh_config_addr may be a parent device
512 	 * in the case of a device behind a bridge, or it may be
513 	 * function zero of a multi-function device.
514 	 * In any case they must share a common PHB.
515 	 */
516 	ret = eeh_ops->get_state(pe, NULL);
517 
518 	/* Note that config-io to empty slots may fail;
519 	 * they are empty when they don't have children.
520 	 * We will punt with the following conditions: Failure to get
521 	 * PE's state, EEH not support and Permanently unavailable
522 	 * state, PE is in good state.
523 	 */
524 	if ((ret < 0) ||
525 	    (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) {
526 		eeh_stats.false_positives++;
527 		pe->false_positives++;
528 		rc = 0;
529 		goto dn_unlock;
530 	}
531 
532 	/*
533 	 * It should be corner case that the parent PE has been
534 	 * put into frozen state as well. We should take care
535 	 * that at first.
536 	 */
537 	parent_pe = pe->parent;
538 	while (parent_pe) {
539 		/* Hit the ceiling ? */
540 		if (parent_pe->type & EEH_PE_PHB)
541 			break;
542 
543 		/* Frozen parent PE ? */
544 		ret = eeh_ops->get_state(parent_pe, NULL);
545 		if (ret > 0 && !eeh_state_active(ret))
546 			pe = parent_pe;
547 
548 		/* Next parent level */
549 		parent_pe = parent_pe->parent;
550 	}
551 
552 	eeh_stats.slot_resets++;
553 
554 	/* Avoid repeated reports of this failure, including problems
555 	 * with other functions on this device, and functions under
556 	 * bridges.
557 	 */
558 	eeh_pe_state_mark(pe, EEH_PE_ISOLATED);
559 	eeh_serialize_unlock(flags);
560 
561 	/* Most EEH events are due to device driver bugs.  Having
562 	 * a stack trace will help the device-driver authors figure
563 	 * out what happened.  So print that out.
564 	 */
565 	phb_pe = eeh_phb_pe_get(pe->phb);
566 	pr_err("EEH: Frozen PHB#%x-PE#%x detected\n",
567 	       pe->phb->global_number, pe->addr);
568 	pr_err("EEH: PE location: %s, PHB location: %s\n",
569 	       eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe));
570 	dump_stack();
571 
572 	eeh_send_failure_event(pe);
573 
574 	return 1;
575 
576 dn_unlock:
577 	eeh_serialize_unlock(flags);
578 	return rc;
579 }
580 
581 EXPORT_SYMBOL_GPL(eeh_dev_check_failure);
582 
583 /**
584  * eeh_check_failure - Check if all 1's data is due to EEH slot freeze
585  * @token: I/O address
586  *
587  * Check for an EEH failure at the given I/O address. Call this
588  * routine if the result of a read was all 0xff's and you want to
589  * find out if this is due to an EEH slot freeze event. This routine
590  * will query firmware for the EEH status.
591  *
592  * Note this routine is safe to call in an interrupt context.
593  */
594 int eeh_check_failure(const volatile void __iomem *token)
595 {
596 	unsigned long addr;
597 	struct eeh_dev *edev;
598 
599 	/* Finding the phys addr + pci device; this is pretty quick. */
600 	addr = eeh_token_to_phys((unsigned long __force) token);
601 	edev = eeh_addr_cache_get_dev(addr);
602 	if (!edev) {
603 		eeh_stats.no_device++;
604 		return 0;
605 	}
606 
607 	return eeh_dev_check_failure(edev);
608 }
609 EXPORT_SYMBOL(eeh_check_failure);
610 
611 
612 /**
613  * eeh_pci_enable - Enable MMIO or DMA transfers for this slot
614  * @pe: EEH PE
615  *
616  * This routine should be called to reenable frozen MMIO or DMA
617  * so that it would work correctly again. It's useful while doing
618  * recovery or log collection on the indicated device.
619  */
620 int eeh_pci_enable(struct eeh_pe *pe, int function)
621 {
622 	int active_flag, rc;
623 
624 	/*
625 	 * pHyp doesn't allow to enable IO or DMA on unfrozen PE.
626 	 * Also, it's pointless to enable them on unfrozen PE. So
627 	 * we have to check before enabling IO or DMA.
628 	 */
629 	switch (function) {
630 	case EEH_OPT_THAW_MMIO:
631 		active_flag = EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED;
632 		break;
633 	case EEH_OPT_THAW_DMA:
634 		active_flag = EEH_STATE_DMA_ACTIVE;
635 		break;
636 	case EEH_OPT_DISABLE:
637 	case EEH_OPT_ENABLE:
638 	case EEH_OPT_FREEZE_PE:
639 		active_flag = 0;
640 		break;
641 	default:
642 		pr_warn("%s: Invalid function %d\n",
643 			__func__, function);
644 		return -EINVAL;
645 	}
646 
647 	/*
648 	 * Check if IO or DMA has been enabled before
649 	 * enabling them.
650 	 */
651 	if (active_flag) {
652 		rc = eeh_ops->get_state(pe, NULL);
653 		if (rc < 0)
654 			return rc;
655 
656 		/* Needn't enable it at all */
657 		if (rc == EEH_STATE_NOT_SUPPORT)
658 			return 0;
659 
660 		/* It's already enabled */
661 		if (rc & active_flag)
662 			return 0;
663 	}
664 
665 
666 	/* Issue the request */
667 	rc = eeh_ops->set_option(pe, function);
668 	if (rc)
669 		pr_warn("%s: Unexpected state change %d on "
670 			"PHB#%x-PE#%x, err=%d\n",
671 			__func__, function, pe->phb->global_number,
672 			pe->addr, rc);
673 
674 	/* Check if the request is finished successfully */
675 	if (active_flag) {
676 		rc = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
677 		if (rc < 0)
678 			return rc;
679 
680 		if (rc & active_flag)
681 			return 0;
682 
683 		return -EIO;
684 	}
685 
686 	return rc;
687 }
688 
689 static void *eeh_disable_and_save_dev_state(void *data, void *userdata)
690 {
691 	struct eeh_dev *edev = data;
692 	struct pci_dev *pdev = eeh_dev_to_pci_dev(edev);
693 	struct pci_dev *dev = userdata;
694 
695 	/*
696 	 * The caller should have disabled and saved the
697 	 * state for the specified device
698 	 */
699 	if (!pdev || pdev == dev)
700 		return NULL;
701 
702 	/* Ensure we have D0 power state */
703 	pci_set_power_state(pdev, PCI_D0);
704 
705 	/* Save device state */
706 	pci_save_state(pdev);
707 
708 	/*
709 	 * Disable device to avoid any DMA traffic and
710 	 * interrupt from the device
711 	 */
712 	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
713 
714 	return NULL;
715 }
716 
717 static void *eeh_restore_dev_state(void *data, void *userdata)
718 {
719 	struct eeh_dev *edev = data;
720 	struct pci_dn *pdn = eeh_dev_to_pdn(edev);
721 	struct pci_dev *pdev = eeh_dev_to_pci_dev(edev);
722 	struct pci_dev *dev = userdata;
723 
724 	if (!pdev)
725 		return NULL;
726 
727 	/* Apply customization from firmware */
728 	if (pdn && eeh_ops->restore_config)
729 		eeh_ops->restore_config(pdn);
730 
731 	/* The caller should restore state for the specified device */
732 	if (pdev != dev)
733 		pci_restore_state(pdev);
734 
735 	return NULL;
736 }
737 
738 int eeh_restore_vf_config(struct pci_dn *pdn)
739 {
740 	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
741 	u32 devctl, cmd, cap2, aer_capctl;
742 	int old_mps;
743 
744 	if (edev->pcie_cap) {
745 		/* Restore MPS */
746 		old_mps = (ffs(pdn->mps) - 8) << 5;
747 		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
748 				     2, &devctl);
749 		devctl &= ~PCI_EXP_DEVCTL_PAYLOAD;
750 		devctl |= old_mps;
751 		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
752 				      2, devctl);
753 
754 		/* Disable Completion Timeout if possible */
755 		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCAP2,
756 				     4, &cap2);
757 		if (cap2 & PCI_EXP_DEVCAP2_COMP_TMOUT_DIS) {
758 			eeh_ops->read_config(pdn,
759 					     edev->pcie_cap + PCI_EXP_DEVCTL2,
760 					     4, &cap2);
761 			cap2 |= PCI_EXP_DEVCTL2_COMP_TMOUT_DIS;
762 			eeh_ops->write_config(pdn,
763 					      edev->pcie_cap + PCI_EXP_DEVCTL2,
764 					      4, cap2);
765 		}
766 	}
767 
768 	/* Enable SERR and parity checking */
769 	eeh_ops->read_config(pdn, PCI_COMMAND, 2, &cmd);
770 	cmd |= (PCI_COMMAND_PARITY | PCI_COMMAND_SERR);
771 	eeh_ops->write_config(pdn, PCI_COMMAND, 2, cmd);
772 
773 	/* Enable report various errors */
774 	if (edev->pcie_cap) {
775 		eeh_ops->read_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
776 				     2, &devctl);
777 		devctl &= ~PCI_EXP_DEVCTL_CERE;
778 		devctl |= (PCI_EXP_DEVCTL_NFERE |
779 			   PCI_EXP_DEVCTL_FERE |
780 			   PCI_EXP_DEVCTL_URRE);
781 		eeh_ops->write_config(pdn, edev->pcie_cap + PCI_EXP_DEVCTL,
782 				      2, devctl);
783 	}
784 
785 	/* Enable ECRC generation and check */
786 	if (edev->pcie_cap && edev->aer_cap) {
787 		eeh_ops->read_config(pdn, edev->aer_cap + PCI_ERR_CAP,
788 				     4, &aer_capctl);
789 		aer_capctl |= (PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE);
790 		eeh_ops->write_config(pdn, edev->aer_cap + PCI_ERR_CAP,
791 				      4, aer_capctl);
792 	}
793 
794 	return 0;
795 }
796 
797 /**
798  * pcibios_set_pcie_reset_state - Set PCI-E reset state
799  * @dev: pci device struct
800  * @state: reset state to enter
801  *
802  * Return value:
803  * 	0 if success
804  */
805 int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
806 {
807 	struct eeh_dev *edev = pci_dev_to_eeh_dev(dev);
808 	struct eeh_pe *pe = eeh_dev_to_pe(edev);
809 
810 	if (!pe) {
811 		pr_err("%s: No PE found on PCI device %s\n",
812 			__func__, pci_name(dev));
813 		return -EINVAL;
814 	}
815 
816 	switch (state) {
817 	case pcie_deassert_reset:
818 		eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);
819 		eeh_unfreeze_pe(pe, false);
820 		if (!(pe->type & EEH_PE_VF))
821 			eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
822 		eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev);
823 		eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
824 		break;
825 	case pcie_hot_reset:
826 		eeh_pe_state_mark_with_cfg(pe, EEH_PE_ISOLATED);
827 		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
828 		eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev);
829 		if (!(pe->type & EEH_PE_VF))
830 			eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED);
831 		eeh_ops->reset(pe, EEH_RESET_HOT);
832 		break;
833 	case pcie_warm_reset:
834 		eeh_pe_state_mark_with_cfg(pe, EEH_PE_ISOLATED);
835 		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
836 		eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev);
837 		if (!(pe->type & EEH_PE_VF))
838 			eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED);
839 		eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL);
840 		break;
841 	default:
842 		eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED);
843 		return -EINVAL;
844 	};
845 
846 	return 0;
847 }
848 
849 /**
850  * eeh_set_pe_freset - Check the required reset for the indicated device
851  * @data: EEH device
852  * @flag: return value
853  *
854  * Each device might have its preferred reset type: fundamental or
855  * hot reset. The routine is used to collected the information for
856  * the indicated device and its children so that the bunch of the
857  * devices could be reset properly.
858  */
859 static void *eeh_set_dev_freset(void *data, void *flag)
860 {
861 	struct pci_dev *dev;
862 	unsigned int *freset = (unsigned int *)flag;
863 	struct eeh_dev *edev = (struct eeh_dev *)data;
864 
865 	dev = eeh_dev_to_pci_dev(edev);
866 	if (dev)
867 		*freset |= dev->needs_freset;
868 
869 	return NULL;
870 }
871 
872 /**
873  * eeh_pe_reset_full - Complete a full reset process on the indicated PE
874  * @pe: EEH PE
875  *
876  * This function executes a full reset procedure on a PE, including setting
877  * the appropriate flags, performing a fundamental or hot reset, and then
878  * deactivating the reset status.  It is designed to be used within the EEH
879  * subsystem, as opposed to eeh_pe_reset which is exported to drivers and
880  * only performs a single operation at a time.
881  *
882  * This function will attempt to reset a PE three times before failing.
883  */
884 int eeh_pe_reset_full(struct eeh_pe *pe)
885 {
886 	int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED);
887 	int type = EEH_RESET_HOT;
888 	unsigned int freset = 0;
889 	int i, state, ret;
890 
891 	/*
892 	 * Determine the type of reset to perform - hot or fundamental.
893 	 * Hot reset is the default operation, unless any device under the
894 	 * PE requires a fundamental reset.
895 	 */
896 	eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset);
897 
898 	if (freset)
899 		type = EEH_RESET_FUNDAMENTAL;
900 
901 	/* Mark the PE as in reset state and block config space accesses */
902 	eeh_pe_state_mark(pe, reset_state);
903 
904 	/* Make three attempts at resetting the bus */
905 	for (i = 0; i < 3; i++) {
906 		ret = eeh_pe_reset(pe, type);
907 		if (ret)
908 			break;
909 
910 		ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE);
911 		if (ret)
912 			break;
913 
914 		/* Wait until the PE is in a functioning state */
915 		state = eeh_ops->wait_state(pe, PCI_BUS_RESET_WAIT_MSEC);
916 		if (eeh_state_active(state))
917 			break;
918 
919 		if (state < 0) {
920 			pr_warn("%s: Unrecoverable slot failure on PHB#%x-PE#%x",
921 				__func__, pe->phb->global_number, pe->addr);
922 			ret = -ENOTRECOVERABLE;
923 			break;
924 		}
925 
926 		/* Set error in case this is our last attempt */
927 		ret = -EIO;
928 		pr_warn("%s: Failure %d resetting PHB#%x-PE#%x\n (%d)\n",
929 			__func__, state, pe->phb->global_number, pe->addr, (i + 1));
930 	}
931 
932 	eeh_pe_state_clear(pe, reset_state);
933 	return ret;
934 }
935 
936 /**
937  * eeh_save_bars - Save device bars
938  * @edev: PCI device associated EEH device
939  *
940  * Save the values of the device bars. Unlike the restore
941  * routine, this routine is *not* recursive. This is because
942  * PCI devices are added individually; but, for the restore,
943  * an entire slot is reset at a time.
944  */
945 void eeh_save_bars(struct eeh_dev *edev)
946 {
947 	struct pci_dn *pdn;
948 	int i;
949 
950 	pdn = eeh_dev_to_pdn(edev);
951 	if (!pdn)
952 		return;
953 
954 	for (i = 0; i < 16; i++)
955 		eeh_ops->read_config(pdn, i * 4, 4, &edev->config_space[i]);
956 
957 	/*
958 	 * For PCI bridges including root port, we need enable bus
959 	 * master explicitly. Otherwise, it can't fetch IODA table
960 	 * entries correctly. So we cache the bit in advance so that
961 	 * we can restore it after reset, either PHB range or PE range.
962 	 */
963 	if (edev->mode & EEH_DEV_BRIDGE)
964 		edev->config_space[1] |= PCI_COMMAND_MASTER;
965 }
966 
967 /**
968  * eeh_ops_register - Register platform dependent EEH operations
969  * @ops: platform dependent EEH operations
970  *
971  * Register the platform dependent EEH operation callback
972  * functions. The platform should call this function before
973  * any other EEH operations.
974  */
975 int __init eeh_ops_register(struct eeh_ops *ops)
976 {
977 	if (!ops->name) {
978 		pr_warn("%s: Invalid EEH ops name for %p\n",
979 			__func__, ops);
980 		return -EINVAL;
981 	}
982 
983 	if (eeh_ops && eeh_ops != ops) {
984 		pr_warn("%s: EEH ops of platform %s already existing (%s)\n",
985 			__func__, eeh_ops->name, ops->name);
986 		return -EEXIST;
987 	}
988 
989 	eeh_ops = ops;
990 
991 	return 0;
992 }
993 
994 /**
995  * eeh_ops_unregister - Unreigster platform dependent EEH operations
996  * @name: name of EEH platform operations
997  *
998  * Unregister the platform dependent EEH operation callback
999  * functions.
1000  */
1001 int __exit eeh_ops_unregister(const char *name)
1002 {
1003 	if (!name || !strlen(name)) {
1004 		pr_warn("%s: Invalid EEH ops name\n",
1005 			__func__);
1006 		return -EINVAL;
1007 	}
1008 
1009 	if (eeh_ops && !strcmp(eeh_ops->name, name)) {
1010 		eeh_ops = NULL;
1011 		return 0;
1012 	}
1013 
1014 	return -EEXIST;
1015 }
1016 
1017 static int eeh_reboot_notifier(struct notifier_block *nb,
1018 			       unsigned long action, void *unused)
1019 {
1020 	eeh_clear_flag(EEH_ENABLED);
1021 	return NOTIFY_DONE;
1022 }
1023 
1024 static struct notifier_block eeh_reboot_nb = {
1025 	.notifier_call = eeh_reboot_notifier,
1026 };
1027 
1028 void eeh_probe_devices(void)
1029 {
1030 	struct pci_controller *hose, *tmp;
1031 	struct pci_dn *pdn;
1032 
1033 	/* Enable EEH for all adapters */
1034 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node) {
1035 		pdn = hose->pci_data;
1036 		traverse_pci_dn(pdn, eeh_ops->probe, NULL);
1037 	}
1038 }
1039 
1040 /**
1041  * eeh_init - EEH initialization
1042  *
1043  * Initialize EEH by trying to enable it for all of the adapters in the system.
1044  * As a side effect we can determine here if eeh is supported at all.
1045  * Note that we leave EEH on so failed config cycles won't cause a machine
1046  * check.  If a user turns off EEH for a particular adapter they are really
1047  * telling Linux to ignore errors.  Some hardware (e.g. POWER5) won't
1048  * grant access to a slot if EEH isn't enabled, and so we always enable
1049  * EEH for all slots/all devices.
1050  *
1051  * The eeh-force-off option disables EEH checking globally, for all slots.
1052  * Even if force-off is set, the EEH hardware is still enabled, so that
1053  * newer systems can boot.
1054  */
1055 static int eeh_init(void)
1056 {
1057 	struct pci_controller *hose, *tmp;
1058 	int ret = 0;
1059 
1060 	/* Register reboot notifier */
1061 	ret = register_reboot_notifier(&eeh_reboot_nb);
1062 	if (ret) {
1063 		pr_warn("%s: Failed to register notifier (%d)\n",
1064 			__func__, ret);
1065 		return ret;
1066 	}
1067 
1068 	/* call platform initialization function */
1069 	if (!eeh_ops) {
1070 		pr_warn("%s: Platform EEH operation not found\n",
1071 			__func__);
1072 		return -EEXIST;
1073 	} else if ((ret = eeh_ops->init()))
1074 		return ret;
1075 
1076 	/* Initialize PHB PEs */
1077 	list_for_each_entry_safe(hose, tmp, &hose_list, list_node)
1078 		eeh_dev_phb_init_dynamic(hose);
1079 
1080 	/* Initialize EEH event */
1081 	ret = eeh_event_init();
1082 	if (ret)
1083 		return ret;
1084 
1085 	eeh_probe_devices();
1086 
1087 	if (eeh_enabled())
1088 		pr_info("EEH: PCI Enhanced I/O Error Handling Enabled\n");
1089 	else
1090 		pr_info("EEH: No capable adapters found\n");
1091 
1092 	return ret;
1093 }
1094 
1095 core_initcall_sync(eeh_init);
1096 
1097 /**
1098  * eeh_add_device_early - Enable EEH for the indicated device node
1099  * @pdn: PCI device node for which to set up EEH
1100  *
1101  * This routine must be used to perform EEH initialization for PCI
1102  * devices that were added after system boot (e.g. hotplug, dlpar).
1103  * This routine must be called before any i/o is performed to the
1104  * adapter (inluding any config-space i/o).
1105  * Whether this actually enables EEH or not for this device depends
1106  * on the CEC architecture, type of the device, on earlier boot
1107  * command-line arguments & etc.
1108  */
1109 void eeh_add_device_early(struct pci_dn *pdn)
1110 {
1111 	struct pci_controller *phb = pdn ? pdn->phb : NULL;
1112 	struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
1113 
1114 	if (!edev)
1115 		return;
1116 
1117 	if (!eeh_has_flag(EEH_PROBE_MODE_DEVTREE))
1118 		return;
1119 
1120 	/* USB Bus children of PCI devices will not have BUID's */
1121 	if (NULL == phb ||
1122 	    (eeh_has_flag(EEH_PROBE_MODE_DEVTREE) && 0 == phb->buid))
1123 		return;
1124 
1125 	eeh_ops->probe(pdn, NULL);
1126 }
1127 
1128 /**
1129  * eeh_add_device_tree_early - Enable EEH for the indicated device
1130  * @pdn: PCI device node
1131  *
1132  * This routine must be used to perform EEH initialization for the
1133  * indicated PCI device that was added after system boot (e.g.
1134  * hotplug, dlpar).
1135  */
1136 void eeh_add_device_tree_early(struct pci_dn *pdn)
1137 {
1138 	struct pci_dn *n;
1139 
1140 	if (!pdn)
1141 		return;
1142 
1143 	list_for_each_entry(n, &pdn->child_list, list)
1144 		eeh_add_device_tree_early(n);
1145 	eeh_add_device_early(pdn);
1146 }
1147 EXPORT_SYMBOL_GPL(eeh_add_device_tree_early);
1148 
1149 /**
1150  * eeh_add_device_late - Perform EEH initialization for the indicated pci device
1151  * @dev: pci device for which to set up EEH
1152  *
1153  * This routine must be used to complete EEH initialization for PCI
1154  * devices that were added after system boot (e.g. hotplug, dlpar).
1155  */
1156 void eeh_add_device_late(struct pci_dev *dev)
1157 {
1158 	struct pci_dn *pdn;
1159 	struct eeh_dev *edev;
1160 
1161 	if (!dev || !eeh_enabled())
1162 		return;
1163 
1164 	pr_debug("EEH: Adding device %s\n", pci_name(dev));
1165 
1166 	pdn = pci_get_pdn_by_devfn(dev->bus, dev->devfn);
1167 	edev = pdn_to_eeh_dev(pdn);
1168 	if (edev->pdev == dev) {
1169 		pr_debug("EEH: Already referenced !\n");
1170 		return;
1171 	}
1172 
1173 	/*
1174 	 * The EEH cache might not be removed correctly because of
1175 	 * unbalanced kref to the device during unplug time, which
1176 	 * relies on pcibios_release_device(). So we have to remove
1177 	 * that here explicitly.
1178 	 */
1179 	if (edev->pdev) {
1180 		eeh_rmv_from_parent_pe(edev);
1181 		eeh_addr_cache_rmv_dev(edev->pdev);
1182 		eeh_sysfs_remove_device(edev->pdev);
1183 		edev->mode &= ~EEH_DEV_SYSFS;
1184 
1185 		/*
1186 		 * We definitely should have the PCI device removed
1187 		 * though it wasn't correctly. So we needn't call
1188 		 * into error handler afterwards.
1189 		 */
1190 		edev->mode |= EEH_DEV_NO_HANDLER;
1191 
1192 		edev->pdev = NULL;
1193 		dev->dev.archdata.edev = NULL;
1194 	}
1195 
1196 	if (eeh_has_flag(EEH_PROBE_MODE_DEV))
1197 		eeh_ops->probe(pdn, NULL);
1198 
1199 	edev->pdev = dev;
1200 	dev->dev.archdata.edev = edev;
1201 
1202 	eeh_addr_cache_insert_dev(dev);
1203 }
1204 
1205 /**
1206  * eeh_add_device_tree_late - Perform EEH initialization for the indicated PCI bus
1207  * @bus: PCI bus
1208  *
1209  * This routine must be used to perform EEH initialization for PCI
1210  * devices which are attached to the indicated PCI bus. The PCI bus
1211  * is added after system boot through hotplug or dlpar.
1212  */
1213 void eeh_add_device_tree_late(struct pci_bus *bus)
1214 {
1215 	struct pci_dev *dev;
1216 
1217 	list_for_each_entry(dev, &bus->devices, bus_list) {
1218 		eeh_add_device_late(dev);
1219 		if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1220 			struct pci_bus *subbus = dev->subordinate;
1221 			if (subbus)
1222 				eeh_add_device_tree_late(subbus);
1223 		}
1224 	}
1225 }
1226 EXPORT_SYMBOL_GPL(eeh_add_device_tree_late);
1227 
1228 /**
1229  * eeh_add_sysfs_files - Add EEH sysfs files for the indicated PCI bus
1230  * @bus: PCI bus
1231  *
1232  * This routine must be used to add EEH sysfs files for PCI
1233  * devices which are attached to the indicated PCI bus. The PCI bus
1234  * is added after system boot through hotplug or dlpar.
1235  */
1236 void eeh_add_sysfs_files(struct pci_bus *bus)
1237 {
1238 	struct pci_dev *dev;
1239 
1240 	list_for_each_entry(dev, &bus->devices, bus_list) {
1241 		eeh_sysfs_add_device(dev);
1242 		if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1243 			struct pci_bus *subbus = dev->subordinate;
1244 			if (subbus)
1245 				eeh_add_sysfs_files(subbus);
1246 		}
1247 	}
1248 }
1249 EXPORT_SYMBOL_GPL(eeh_add_sysfs_files);
1250 
1251 /**
1252  * eeh_remove_device - Undo EEH setup for the indicated pci device
1253  * @dev: pci device to be removed
1254  *
1255  * This routine should be called when a device is removed from
1256  * a running system (e.g. by hotplug or dlpar).  It unregisters
1257  * the PCI device from the EEH subsystem.  I/O errors affecting
1258  * this device will no longer be detected after this call; thus,
1259  * i/o errors affecting this slot may leave this device unusable.
1260  */
1261 void eeh_remove_device(struct pci_dev *dev)
1262 {
1263 	struct eeh_dev *edev;
1264 
1265 	if (!dev || !eeh_enabled())
1266 		return;
1267 	edev = pci_dev_to_eeh_dev(dev);
1268 
1269 	/* Unregister the device with the EEH/PCI address search system */
1270 	pr_debug("EEH: Removing device %s\n", pci_name(dev));
1271 
1272 	if (!edev || !edev->pdev || !edev->pe) {
1273 		pr_debug("EEH: Not referenced !\n");
1274 		return;
1275 	}
1276 
1277 	/*
1278 	 * During the hotplug for EEH error recovery, we need the EEH
1279 	 * device attached to the parent PE in order for BAR restore
1280 	 * a bit later. So we keep it for BAR restore and remove it
1281 	 * from the parent PE during the BAR resotre.
1282 	 */
1283 	edev->pdev = NULL;
1284 
1285 	/*
1286 	 * The flag "in_error" is used to trace EEH devices for VFs
1287 	 * in error state or not. It's set in eeh_report_error(). If
1288 	 * it's not set, eeh_report_{reset,resume}() won't be called
1289 	 * for the VF EEH device.
1290 	 */
1291 	edev->in_error = false;
1292 	dev->dev.archdata.edev = NULL;
1293 	if (!(edev->pe->state & EEH_PE_KEEP))
1294 		eeh_rmv_from_parent_pe(edev);
1295 	else
1296 		edev->mode |= EEH_DEV_DISCONNECTED;
1297 
1298 	/*
1299 	 * We're removing from the PCI subsystem, that means
1300 	 * the PCI device driver can't support EEH or not
1301 	 * well. So we rely on hotplug completely to do recovery
1302 	 * for the specific PCI device.
1303 	 */
1304 	edev->mode |= EEH_DEV_NO_HANDLER;
1305 
1306 	eeh_addr_cache_rmv_dev(dev);
1307 	eeh_sysfs_remove_device(dev);
1308 	edev->mode &= ~EEH_DEV_SYSFS;
1309 }
1310 
1311 int eeh_unfreeze_pe(struct eeh_pe *pe, bool sw_state)
1312 {
1313 	int ret;
1314 
1315 	ret = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO);
1316 	if (ret) {
1317 		pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n",
1318 			__func__, ret, pe->phb->global_number, pe->addr);
1319 		return ret;
1320 	}
1321 
1322 	ret = eeh_pci_enable(pe, EEH_OPT_THAW_DMA);
1323 	if (ret) {
1324 		pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n",
1325 			__func__, ret, pe->phb->global_number, pe->addr);
1326 		return ret;
1327 	}
1328 
1329 	/* Clear software isolated state */
1330 	if (sw_state && (pe->state & EEH_PE_ISOLATED))
1331 		eeh_pe_state_clear(pe, EEH_PE_ISOLATED);
1332 
1333 	return ret;
1334 }
1335 
1336 
1337 static struct pci_device_id eeh_reset_ids[] = {
1338 	{ PCI_DEVICE(0x19a2, 0x0710) },	/* Emulex, BE     */
1339 	{ PCI_DEVICE(0x10df, 0xe220) },	/* Emulex, Lancer */
1340 	{ PCI_DEVICE(0x14e4, 0x1657) }, /* Broadcom BCM5719 */
1341 	{ 0 }
1342 };
1343 
1344 static int eeh_pe_change_owner(struct eeh_pe *pe)
1345 {
1346 	struct eeh_dev *edev, *tmp;
1347 	struct pci_dev *pdev;
1348 	struct pci_device_id *id;
1349 	int ret;
1350 
1351 	/* Check PE state */
1352 	ret = eeh_ops->get_state(pe, NULL);
1353 	if (ret < 0 || ret == EEH_STATE_NOT_SUPPORT)
1354 		return 0;
1355 
1356 	/* Unfrozen PE, nothing to do */
1357 	if (eeh_state_active(ret))
1358 		return 0;
1359 
1360 	/* Frozen PE, check if it needs PE level reset */
1361 	eeh_pe_for_each_dev(pe, edev, tmp) {
1362 		pdev = eeh_dev_to_pci_dev(edev);
1363 		if (!pdev)
1364 			continue;
1365 
1366 		for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) {
1367 			if (id->vendor != PCI_ANY_ID &&
1368 			    id->vendor != pdev->vendor)
1369 				continue;
1370 			if (id->device != PCI_ANY_ID &&
1371 			    id->device != pdev->device)
1372 				continue;
1373 			if (id->subvendor != PCI_ANY_ID &&
1374 			    id->subvendor != pdev->subsystem_vendor)
1375 				continue;
1376 			if (id->subdevice != PCI_ANY_ID &&
1377 			    id->subdevice != pdev->subsystem_device)
1378 				continue;
1379 
1380 			return eeh_pe_reset_and_recover(pe);
1381 		}
1382 	}
1383 
1384 	return eeh_unfreeze_pe(pe, true);
1385 }
1386 
1387 /**
1388  * eeh_dev_open - Increase count of pass through devices for PE
1389  * @pdev: PCI device
1390  *
1391  * Increase count of passed through devices for the indicated
1392  * PE. In the result, the EEH errors detected on the PE won't be
1393  * reported. The PE owner will be responsible for detection
1394  * and recovery.
1395  */
1396 int eeh_dev_open(struct pci_dev *pdev)
1397 {
1398 	struct eeh_dev *edev;
1399 	int ret = -ENODEV;
1400 
1401 	mutex_lock(&eeh_dev_mutex);
1402 
1403 	/* No PCI device ? */
1404 	if (!pdev)
1405 		goto out;
1406 
1407 	/* No EEH device or PE ? */
1408 	edev = pci_dev_to_eeh_dev(pdev);
1409 	if (!edev || !edev->pe)
1410 		goto out;
1411 
1412 	/*
1413 	 * The PE might have been put into frozen state, but we
1414 	 * didn't detect that yet. The passed through PCI devices
1415 	 * in frozen PE won't work properly. Clear the frozen state
1416 	 * in advance.
1417 	 */
1418 	ret = eeh_pe_change_owner(edev->pe);
1419 	if (ret)
1420 		goto out;
1421 
1422 	/* Increase PE's pass through count */
1423 	atomic_inc(&edev->pe->pass_dev_cnt);
1424 	mutex_unlock(&eeh_dev_mutex);
1425 
1426 	return 0;
1427 out:
1428 	mutex_unlock(&eeh_dev_mutex);
1429 	return ret;
1430 }
1431 EXPORT_SYMBOL_GPL(eeh_dev_open);
1432 
1433 /**
1434  * eeh_dev_release - Decrease count of pass through devices for PE
1435  * @pdev: PCI device
1436  *
1437  * Decrease count of pass through devices for the indicated PE. If
1438  * there is no passed through device in PE, the EEH errors detected
1439  * on the PE will be reported and handled as usual.
1440  */
1441 void eeh_dev_release(struct pci_dev *pdev)
1442 {
1443 	struct eeh_dev *edev;
1444 
1445 	mutex_lock(&eeh_dev_mutex);
1446 
1447 	/* No PCI device ? */
1448 	if (!pdev)
1449 		goto out;
1450 
1451 	/* No EEH device ? */
1452 	edev = pci_dev_to_eeh_dev(pdev);
1453 	if (!edev || !edev->pe || !eeh_pe_passed(edev->pe))
1454 		goto out;
1455 
1456 	/* Decrease PE's pass through count */
1457 	WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0);
1458 	eeh_pe_change_owner(edev->pe);
1459 out:
1460 	mutex_unlock(&eeh_dev_mutex);
1461 }
1462 EXPORT_SYMBOL(eeh_dev_release);
1463 
1464 #ifdef CONFIG_IOMMU_API
1465 
1466 static int dev_has_iommu_table(struct device *dev, void *data)
1467 {
1468 	struct pci_dev *pdev = to_pci_dev(dev);
1469 	struct pci_dev **ppdev = data;
1470 
1471 	if (!dev)
1472 		return 0;
1473 
1474 	if (dev->iommu_group) {
1475 		*ppdev = pdev;
1476 		return 1;
1477 	}
1478 
1479 	return 0;
1480 }
1481 
1482 /**
1483  * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE
1484  * @group: IOMMU group
1485  *
1486  * The routine is called to convert IOMMU group to EEH PE.
1487  */
1488 struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group)
1489 {
1490 	struct pci_dev *pdev = NULL;
1491 	struct eeh_dev *edev;
1492 	int ret;
1493 
1494 	/* No IOMMU group ? */
1495 	if (!group)
1496 		return NULL;
1497 
1498 	ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table);
1499 	if (!ret || !pdev)
1500 		return NULL;
1501 
1502 	/* No EEH device or PE ? */
1503 	edev = pci_dev_to_eeh_dev(pdev);
1504 	if (!edev || !edev->pe)
1505 		return NULL;
1506 
1507 	return edev->pe;
1508 }
1509 EXPORT_SYMBOL_GPL(eeh_iommu_group_to_pe);
1510 
1511 #endif /* CONFIG_IOMMU_API */
1512 
1513 /**
1514  * eeh_pe_set_option - Set options for the indicated PE
1515  * @pe: EEH PE
1516  * @option: requested option
1517  *
1518  * The routine is called to enable or disable EEH functionality
1519  * on the indicated PE, to enable IO or DMA for the frozen PE.
1520  */
1521 int eeh_pe_set_option(struct eeh_pe *pe, int option)
1522 {
1523 	int ret = 0;
1524 
1525 	/* Invalid PE ? */
1526 	if (!pe)
1527 		return -ENODEV;
1528 
1529 	/*
1530 	 * EEH functionality could possibly be disabled, just
1531 	 * return error for the case. And the EEH functinality
1532 	 * isn't expected to be disabled on one specific PE.
1533 	 */
1534 	switch (option) {
1535 	case EEH_OPT_ENABLE:
1536 		if (eeh_enabled()) {
1537 			ret = eeh_pe_change_owner(pe);
1538 			break;
1539 		}
1540 		ret = -EIO;
1541 		break;
1542 	case EEH_OPT_DISABLE:
1543 		break;
1544 	case EEH_OPT_THAW_MMIO:
1545 	case EEH_OPT_THAW_DMA:
1546 	case EEH_OPT_FREEZE_PE:
1547 		if (!eeh_ops || !eeh_ops->set_option) {
1548 			ret = -ENOENT;
1549 			break;
1550 		}
1551 
1552 		ret = eeh_pci_enable(pe, option);
1553 		break;
1554 	default:
1555 		pr_debug("%s: Option %d out of range (%d, %d)\n",
1556 			__func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA);
1557 		ret = -EINVAL;
1558 	}
1559 
1560 	return ret;
1561 }
1562 EXPORT_SYMBOL_GPL(eeh_pe_set_option);
1563 
1564 /**
1565  * eeh_pe_get_state - Retrieve PE's state
1566  * @pe: EEH PE
1567  *
1568  * Retrieve the PE's state, which includes 3 aspects: enabled
1569  * DMA, enabled IO and asserted reset.
1570  */
1571 int eeh_pe_get_state(struct eeh_pe *pe)
1572 {
1573 	int result, ret = 0;
1574 	bool rst_active, dma_en, mmio_en;
1575 
1576 	/* Existing PE ? */
1577 	if (!pe)
1578 		return -ENODEV;
1579 
1580 	if (!eeh_ops || !eeh_ops->get_state)
1581 		return -ENOENT;
1582 
1583 	/*
1584 	 * If the parent PE is owned by the host kernel and is undergoing
1585 	 * error recovery, we should return the PE state as temporarily
1586 	 * unavailable so that the error recovery on the guest is suspended
1587 	 * until the recovery completes on the host.
1588 	 */
1589 	if (pe->parent &&
1590 	    !(pe->state & EEH_PE_REMOVED) &&
1591 	    (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING)))
1592 		return EEH_PE_STATE_UNAVAIL;
1593 
1594 	result = eeh_ops->get_state(pe, NULL);
1595 	rst_active = !!(result & EEH_STATE_RESET_ACTIVE);
1596 	dma_en = !!(result & EEH_STATE_DMA_ENABLED);
1597 	mmio_en = !!(result & EEH_STATE_MMIO_ENABLED);
1598 
1599 	if (rst_active)
1600 		ret = EEH_PE_STATE_RESET;
1601 	else if (dma_en && mmio_en)
1602 		ret = EEH_PE_STATE_NORMAL;
1603 	else if (!dma_en && !mmio_en)
1604 		ret = EEH_PE_STATE_STOPPED_IO_DMA;
1605 	else if (!dma_en && mmio_en)
1606 		ret = EEH_PE_STATE_STOPPED_DMA;
1607 	else
1608 		ret = EEH_PE_STATE_UNAVAIL;
1609 
1610 	return ret;
1611 }
1612 EXPORT_SYMBOL_GPL(eeh_pe_get_state);
1613 
1614 static int eeh_pe_reenable_devices(struct eeh_pe *pe)
1615 {
1616 	struct eeh_dev *edev, *tmp;
1617 	struct pci_dev *pdev;
1618 	int ret = 0;
1619 
1620 	/* Restore config space */
1621 	eeh_pe_restore_bars(pe);
1622 
1623 	/*
1624 	 * Reenable PCI devices as the devices passed
1625 	 * through are always enabled before the reset.
1626 	 */
1627 	eeh_pe_for_each_dev(pe, edev, tmp) {
1628 		pdev = eeh_dev_to_pci_dev(edev);
1629 		if (!pdev)
1630 			continue;
1631 
1632 		ret = pci_reenable_device(pdev);
1633 		if (ret) {
1634 			pr_warn("%s: Failure %d reenabling %s\n",
1635 				__func__, ret, pci_name(pdev));
1636 			return ret;
1637 		}
1638 	}
1639 
1640 	/* The PE is still in frozen state */
1641 	return eeh_unfreeze_pe(pe, true);
1642 }
1643 
1644 
1645 /**
1646  * eeh_pe_reset - Issue PE reset according to specified type
1647  * @pe: EEH PE
1648  * @option: reset type
1649  *
1650  * The routine is called to reset the specified PE with the
1651  * indicated type, either fundamental reset or hot reset.
1652  * PE reset is the most important part for error recovery.
1653  */
1654 int eeh_pe_reset(struct eeh_pe *pe, int option)
1655 {
1656 	int ret = 0;
1657 
1658 	/* Invalid PE ? */
1659 	if (!pe)
1660 		return -ENODEV;
1661 
1662 	if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset)
1663 		return -ENOENT;
1664 
1665 	switch (option) {
1666 	case EEH_RESET_DEACTIVATE:
1667 		ret = eeh_ops->reset(pe, option);
1668 		eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED);
1669 		if (ret)
1670 			break;
1671 
1672 		ret = eeh_pe_reenable_devices(pe);
1673 		break;
1674 	case EEH_RESET_HOT:
1675 	case EEH_RESET_FUNDAMENTAL:
1676 		/*
1677 		 * Proactively freeze the PE to drop all MMIO access
1678 		 * during reset, which should be banned as it's always
1679 		 * cause recursive EEH error.
1680 		 */
1681 		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);
1682 
1683 		eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED);
1684 		ret = eeh_ops->reset(pe, option);
1685 		break;
1686 	default:
1687 		pr_debug("%s: Unsupported option %d\n",
1688 			__func__, option);
1689 		ret = -EINVAL;
1690 	}
1691 
1692 	return ret;
1693 }
1694 EXPORT_SYMBOL_GPL(eeh_pe_reset);
1695 
1696 /**
1697  * eeh_pe_configure - Configure PCI bridges after PE reset
1698  * @pe: EEH PE
1699  *
1700  * The routine is called to restore the PCI config space for
1701  * those PCI devices, especially PCI bridges affected by PE
1702  * reset issued previously.
1703  */
1704 int eeh_pe_configure(struct eeh_pe *pe)
1705 {
1706 	int ret = 0;
1707 
1708 	/* Invalid PE ? */
1709 	if (!pe)
1710 		return -ENODEV;
1711 
1712 	return ret;
1713 }
1714 EXPORT_SYMBOL_GPL(eeh_pe_configure);
1715 
1716 /**
1717  * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE
1718  * @pe: the indicated PE
1719  * @type: error type
1720  * @function: error function
1721  * @addr: address
1722  * @mask: address mask
1723  *
1724  * The routine is called to inject the specified PCI error, which
1725  * is determined by @type and @function, to the indicated PE for
1726  * testing purpose.
1727  */
1728 int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func,
1729 		      unsigned long addr, unsigned long mask)
1730 {
1731 	/* Invalid PE ? */
1732 	if (!pe)
1733 		return -ENODEV;
1734 
1735 	/* Unsupported operation ? */
1736 	if (!eeh_ops || !eeh_ops->err_inject)
1737 		return -ENOENT;
1738 
1739 	/* Check on PCI error type */
1740 	if (type != EEH_ERR_TYPE_32 && type != EEH_ERR_TYPE_64)
1741 		return -EINVAL;
1742 
1743 	/* Check on PCI error function */
1744 	if (func < EEH_ERR_FUNC_MIN || func > EEH_ERR_FUNC_MAX)
1745 		return -EINVAL;
1746 
1747 	return eeh_ops->err_inject(pe, type, func, addr, mask);
1748 }
1749 EXPORT_SYMBOL_GPL(eeh_pe_inject_err);
1750 
1751 static int proc_eeh_show(struct seq_file *m, void *v)
1752 {
1753 	if (!eeh_enabled()) {
1754 		seq_printf(m, "EEH Subsystem is globally disabled\n");
1755 		seq_printf(m, "eeh_total_mmio_ffs=%llu\n", eeh_stats.total_mmio_ffs);
1756 	} else {
1757 		seq_printf(m, "EEH Subsystem is enabled\n");
1758 		seq_printf(m,
1759 				"no device=%llu\n"
1760 				"no device node=%llu\n"
1761 				"no config address=%llu\n"
1762 				"check not wanted=%llu\n"
1763 				"eeh_total_mmio_ffs=%llu\n"
1764 				"eeh_false_positives=%llu\n"
1765 				"eeh_slot_resets=%llu\n",
1766 				eeh_stats.no_device,
1767 				eeh_stats.no_dn,
1768 				eeh_stats.no_cfg_addr,
1769 				eeh_stats.ignored_check,
1770 				eeh_stats.total_mmio_ffs,
1771 				eeh_stats.false_positives,
1772 				eeh_stats.slot_resets);
1773 	}
1774 
1775 	return 0;
1776 }
1777 
1778 static int proc_eeh_open(struct inode *inode, struct file *file)
1779 {
1780 	return single_open(file, proc_eeh_show, NULL);
1781 }
1782 
1783 static const struct file_operations proc_eeh_operations = {
1784 	.open      = proc_eeh_open,
1785 	.read      = seq_read,
1786 	.llseek    = seq_lseek,
1787 	.release   = single_release,
1788 };
1789 
1790 #ifdef CONFIG_DEBUG_FS
1791 static int eeh_enable_dbgfs_set(void *data, u64 val)
1792 {
1793 	if (val)
1794 		eeh_clear_flag(EEH_FORCE_DISABLED);
1795 	else
1796 		eeh_add_flag(EEH_FORCE_DISABLED);
1797 
1798 	return 0;
1799 }
1800 
1801 static int eeh_enable_dbgfs_get(void *data, u64 *val)
1802 {
1803 	if (eeh_enabled())
1804 		*val = 0x1ul;
1805 	else
1806 		*val = 0x0ul;
1807 	return 0;
1808 }
1809 
1810 static int eeh_freeze_dbgfs_set(void *data, u64 val)
1811 {
1812 	eeh_max_freezes = val;
1813 	return 0;
1814 }
1815 
1816 static int eeh_freeze_dbgfs_get(void *data, u64 *val)
1817 {
1818 	*val = eeh_max_freezes;
1819 	return 0;
1820 }
1821 
1822 DEFINE_SIMPLE_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get,
1823 			eeh_enable_dbgfs_set, "0x%llx\n");
1824 DEFINE_SIMPLE_ATTRIBUTE(eeh_freeze_dbgfs_ops, eeh_freeze_dbgfs_get,
1825 			eeh_freeze_dbgfs_set, "0x%llx\n");
1826 #endif
1827 
1828 static int __init eeh_init_proc(void)
1829 {
1830 	if (machine_is(pseries) || machine_is(powernv)) {
1831 		proc_create("powerpc/eeh", 0, NULL, &proc_eeh_operations);
1832 #ifdef CONFIG_DEBUG_FS
1833 		debugfs_create_file("eeh_enable", 0600,
1834                                     powerpc_debugfs_root, NULL,
1835                                     &eeh_enable_dbgfs_ops);
1836 		debugfs_create_file("eeh_max_freezes", 0600,
1837 				    powerpc_debugfs_root, NULL,
1838 				    &eeh_freeze_dbgfs_ops);
1839 #endif
1840 	}
1841 
1842 	return 0;
1843 }
1844 __initcall(eeh_init_proc);
1845