xref: /openbmc/linux/drivers/pci/pci.c (revision d2999e1b)
1 /*
2  *	PCI Bus Services, see include/linux/pci.h for further explanation.
3  *
4  *	Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
5  *	David Mosberger-Tang
6  *
7  *	Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/delay.h>
12 #include <linux/init.h>
13 #include <linux/pci.h>
14 #include <linux/pm.h>
15 #include <linux/slab.h>
16 #include <linux/module.h>
17 #include <linux/spinlock.h>
18 #include <linux/string.h>
19 #include <linux/log2.h>
20 #include <linux/pci-aspm.h>
21 #include <linux/pm_wakeup.h>
22 #include <linux/interrupt.h>
23 #include <linux/device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/pci_hotplug.h>
26 #include <asm-generic/pci-bridge.h>
27 #include <asm/setup.h>
28 #include "pci.h"
29 
30 const char *pci_power_names[] = {
31 	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
32 };
33 EXPORT_SYMBOL_GPL(pci_power_names);
34 
35 int isa_dma_bridge_buggy;
36 EXPORT_SYMBOL(isa_dma_bridge_buggy);
37 
38 int pci_pci_problems;
39 EXPORT_SYMBOL(pci_pci_problems);
40 
41 unsigned int pci_pm_d3_delay;
42 
43 static void pci_pme_list_scan(struct work_struct *work);
44 
45 static LIST_HEAD(pci_pme_list);
46 static DEFINE_MUTEX(pci_pme_list_mutex);
47 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
48 
49 struct pci_pme_device {
50 	struct list_head list;
51 	struct pci_dev *dev;
52 };
53 
54 #define PME_TIMEOUT 1000 /* How long between PME checks */
55 
56 static void pci_dev_d3_sleep(struct pci_dev *dev)
57 {
58 	unsigned int delay = dev->d3_delay;
59 
60 	if (delay < pci_pm_d3_delay)
61 		delay = pci_pm_d3_delay;
62 
63 	msleep(delay);
64 }
65 
66 #ifdef CONFIG_PCI_DOMAINS
67 int pci_domains_supported = 1;
68 #endif
69 
70 #define DEFAULT_CARDBUS_IO_SIZE		(256)
71 #define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
72 /* pci=cbmemsize=nnM,cbiosize=nn can override this */
73 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
74 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
75 
76 #define DEFAULT_HOTPLUG_IO_SIZE		(256)
77 #define DEFAULT_HOTPLUG_MEM_SIZE	(2*1024*1024)
78 /* pci=hpmemsize=nnM,hpiosize=nn can override this */
79 unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
80 unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE;
81 
82 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
83 
84 /*
85  * The default CLS is used if arch didn't set CLS explicitly and not
86  * all pci devices agree on the same value.  Arch can override either
87  * the dfl or actual value as it sees fit.  Don't forget this is
88  * measured in 32-bit words, not bytes.
89  */
90 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
91 u8 pci_cache_line_size;
92 
93 /*
94  * If we set up a device for bus mastering, we need to check the latency
95  * timer as certain BIOSes forget to set it properly.
96  */
97 unsigned int pcibios_max_latency = 255;
98 
99 /* If set, the PCIe ARI capability will not be used. */
100 static bool pcie_ari_disabled;
101 
102 /**
103  * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
104  * @bus: pointer to PCI bus structure to search
105  *
106  * Given a PCI bus, returns the highest PCI bus number present in the set
107  * including the given PCI bus and its list of child PCI buses.
108  */
109 unsigned char pci_bus_max_busnr(struct pci_bus *bus)
110 {
111 	struct pci_bus *tmp;
112 	unsigned char max, n;
113 
114 	max = bus->busn_res.end;
115 	list_for_each_entry(tmp, &bus->children, node) {
116 		n = pci_bus_max_busnr(tmp);
117 		if (n > max)
118 			max = n;
119 	}
120 	return max;
121 }
122 EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
123 
124 #ifdef CONFIG_HAS_IOMEM
125 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
126 {
127 	/*
128 	 * Make sure the BAR is actually a memory resource, not an IO resource
129 	 */
130 	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
131 		WARN_ON(1);
132 		return NULL;
133 	}
134 	return ioremap_nocache(pci_resource_start(pdev, bar),
135 				     pci_resource_len(pdev, bar));
136 }
137 EXPORT_SYMBOL_GPL(pci_ioremap_bar);
138 #endif
139 
140 #define PCI_FIND_CAP_TTL	48
141 
142 static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
143 				   u8 pos, int cap, int *ttl)
144 {
145 	u8 id;
146 
147 	while ((*ttl)--) {
148 		pci_bus_read_config_byte(bus, devfn, pos, &pos);
149 		if (pos < 0x40)
150 			break;
151 		pos &= ~3;
152 		pci_bus_read_config_byte(bus, devfn, pos + PCI_CAP_LIST_ID,
153 					 &id);
154 		if (id == 0xff)
155 			break;
156 		if (id == cap)
157 			return pos;
158 		pos += PCI_CAP_LIST_NEXT;
159 	}
160 	return 0;
161 }
162 
163 static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
164 			       u8 pos, int cap)
165 {
166 	int ttl = PCI_FIND_CAP_TTL;
167 
168 	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
169 }
170 
171 int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
172 {
173 	return __pci_find_next_cap(dev->bus, dev->devfn,
174 				   pos + PCI_CAP_LIST_NEXT, cap);
175 }
176 EXPORT_SYMBOL_GPL(pci_find_next_capability);
177 
178 static int __pci_bus_find_cap_start(struct pci_bus *bus,
179 				    unsigned int devfn, u8 hdr_type)
180 {
181 	u16 status;
182 
183 	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
184 	if (!(status & PCI_STATUS_CAP_LIST))
185 		return 0;
186 
187 	switch (hdr_type) {
188 	case PCI_HEADER_TYPE_NORMAL:
189 	case PCI_HEADER_TYPE_BRIDGE:
190 		return PCI_CAPABILITY_LIST;
191 	case PCI_HEADER_TYPE_CARDBUS:
192 		return PCI_CB_CAPABILITY_LIST;
193 	default:
194 		return 0;
195 	}
196 
197 	return 0;
198 }
199 
200 /**
201  * pci_find_capability - query for devices' capabilities
202  * @dev: PCI device to query
203  * @cap: capability code
204  *
205  * Tell if a device supports a given PCI capability.
206  * Returns the address of the requested capability structure within the
207  * device's PCI configuration space or 0 in case the device does not
208  * support it.  Possible values for @cap:
209  *
210  *  %PCI_CAP_ID_PM           Power Management
211  *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
212  *  %PCI_CAP_ID_VPD          Vital Product Data
213  *  %PCI_CAP_ID_SLOTID       Slot Identification
214  *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
215  *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
216  *  %PCI_CAP_ID_PCIX         PCI-X
217  *  %PCI_CAP_ID_EXP          PCI Express
218  */
219 int pci_find_capability(struct pci_dev *dev, int cap)
220 {
221 	int pos;
222 
223 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
224 	if (pos)
225 		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
226 
227 	return pos;
228 }
229 EXPORT_SYMBOL(pci_find_capability);
230 
231 /**
232  * pci_bus_find_capability - query for devices' capabilities
233  * @bus:   the PCI bus to query
234  * @devfn: PCI device to query
235  * @cap:   capability code
236  *
237  * Like pci_find_capability() but works for pci devices that do not have a
238  * pci_dev structure set up yet.
239  *
240  * Returns the address of the requested capability structure within the
241  * device's PCI configuration space or 0 in case the device does not
242  * support it.
243  */
244 int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
245 {
246 	int pos;
247 	u8 hdr_type;
248 
249 	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
250 
251 	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f);
252 	if (pos)
253 		pos = __pci_find_next_cap(bus, devfn, pos, cap);
254 
255 	return pos;
256 }
257 EXPORT_SYMBOL(pci_bus_find_capability);
258 
259 /**
260  * pci_find_next_ext_capability - Find an extended capability
261  * @dev: PCI device to query
262  * @start: address at which to start looking (0 to start at beginning of list)
263  * @cap: capability code
264  *
265  * Returns the address of the next matching extended capability structure
266  * within the device's PCI configuration space or 0 if the device does
267  * not support it.  Some capabilities can occur several times, e.g., the
268  * vendor-specific capability, and this provides a way to find them all.
269  */
270 int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap)
271 {
272 	u32 header;
273 	int ttl;
274 	int pos = PCI_CFG_SPACE_SIZE;
275 
276 	/* minimum 8 bytes per capability */
277 	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
278 
279 	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
280 		return 0;
281 
282 	if (start)
283 		pos = start;
284 
285 	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
286 		return 0;
287 
288 	/*
289 	 * If we have no capabilities, this is indicated by cap ID,
290 	 * cap version and next pointer all being 0.
291 	 */
292 	if (header == 0)
293 		return 0;
294 
295 	while (ttl-- > 0) {
296 		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
297 			return pos;
298 
299 		pos = PCI_EXT_CAP_NEXT(header);
300 		if (pos < PCI_CFG_SPACE_SIZE)
301 			break;
302 
303 		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
304 			break;
305 	}
306 
307 	return 0;
308 }
309 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
310 
311 /**
312  * pci_find_ext_capability - Find an extended capability
313  * @dev: PCI device to query
314  * @cap: capability code
315  *
316  * Returns the address of the requested extended capability structure
317  * within the device's PCI configuration space or 0 if the device does
318  * not support it.  Possible values for @cap:
319  *
320  *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
321  *  %PCI_EXT_CAP_ID_VC		Virtual Channel
322  *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
323  *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
324  */
325 int pci_find_ext_capability(struct pci_dev *dev, int cap)
326 {
327 	return pci_find_next_ext_capability(dev, 0, cap);
328 }
329 EXPORT_SYMBOL_GPL(pci_find_ext_capability);
330 
331 static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap)
332 {
333 	int rc, ttl = PCI_FIND_CAP_TTL;
334 	u8 cap, mask;
335 
336 	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
337 		mask = HT_3BIT_CAP_MASK;
338 	else
339 		mask = HT_5BIT_CAP_MASK;
340 
341 	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
342 				      PCI_CAP_ID_HT, &ttl);
343 	while (pos) {
344 		rc = pci_read_config_byte(dev, pos + 3, &cap);
345 		if (rc != PCIBIOS_SUCCESSFUL)
346 			return 0;
347 
348 		if ((cap & mask) == ht_cap)
349 			return pos;
350 
351 		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
352 					      pos + PCI_CAP_LIST_NEXT,
353 					      PCI_CAP_ID_HT, &ttl);
354 	}
355 
356 	return 0;
357 }
358 /**
359  * pci_find_next_ht_capability - query a device's Hypertransport capabilities
360  * @dev: PCI device to query
361  * @pos: Position from which to continue searching
362  * @ht_cap: Hypertransport capability code
363  *
364  * To be used in conjunction with pci_find_ht_capability() to search for
365  * all capabilities matching @ht_cap. @pos should always be a value returned
366  * from pci_find_ht_capability().
367  *
368  * NB. To be 100% safe against broken PCI devices, the caller should take
369  * steps to avoid an infinite loop.
370  */
371 int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap)
372 {
373 	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
374 }
375 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
376 
377 /**
378  * pci_find_ht_capability - query a device's Hypertransport capabilities
379  * @dev: PCI device to query
380  * @ht_cap: Hypertransport capability code
381  *
382  * Tell if a device supports a given Hypertransport capability.
383  * Returns an address within the device's PCI configuration space
384  * or 0 in case the device does not support the request capability.
385  * The address points to the PCI capability, of type PCI_CAP_ID_HT,
386  * which has a Hypertransport capability matching @ht_cap.
387  */
388 int pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
389 {
390 	int pos;
391 
392 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
393 	if (pos)
394 		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
395 
396 	return pos;
397 }
398 EXPORT_SYMBOL_GPL(pci_find_ht_capability);
399 
400 /**
401  * pci_find_parent_resource - return resource region of parent bus of given region
402  * @dev: PCI device structure contains resources to be searched
403  * @res: child resource record for which parent is sought
404  *
405  *  For given resource region of given device, return the resource
406  *  region of parent bus the given region is contained in.
407  */
408 struct resource *pci_find_parent_resource(const struct pci_dev *dev,
409 					  struct resource *res)
410 {
411 	const struct pci_bus *bus = dev->bus;
412 	struct resource *r;
413 	int i;
414 
415 	pci_bus_for_each_resource(bus, r, i) {
416 		if (!r)
417 			continue;
418 		if (res->start && resource_contains(r, res)) {
419 
420 			/*
421 			 * If the window is prefetchable but the BAR is
422 			 * not, the allocator made a mistake.
423 			 */
424 			if (r->flags & IORESOURCE_PREFETCH &&
425 			    !(res->flags & IORESOURCE_PREFETCH))
426 				return NULL;
427 
428 			/*
429 			 * If we're below a transparent bridge, there may
430 			 * be both a positively-decoded aperture and a
431 			 * subtractively-decoded region that contain the BAR.
432 			 * We want the positively-decoded one, so this depends
433 			 * on pci_bus_for_each_resource() giving us those
434 			 * first.
435 			 */
436 			return r;
437 		}
438 	}
439 	return NULL;
440 }
441 EXPORT_SYMBOL(pci_find_parent_resource);
442 
443 /**
444  * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
445  * @dev: the PCI device to operate on
446  * @pos: config space offset of status word
447  * @mask: mask of bit(s) to care about in status word
448  *
449  * Return 1 when mask bit(s) in status word clear, 0 otherwise.
450  */
451 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
452 {
453 	int i;
454 
455 	/* Wait for Transaction Pending bit clean */
456 	for (i = 0; i < 4; i++) {
457 		u16 status;
458 		if (i)
459 			msleep((1 << (i - 1)) * 100);
460 
461 		pci_read_config_word(dev, pos, &status);
462 		if (!(status & mask))
463 			return 1;
464 	}
465 
466 	return 0;
467 }
468 
469 /**
470  * pci_restore_bars - restore a devices BAR values (e.g. after wake-up)
471  * @dev: PCI device to have its BARs restored
472  *
473  * Restore the BAR values for a given device, so as to make it
474  * accessible by its driver.
475  */
476 static void pci_restore_bars(struct pci_dev *dev)
477 {
478 	int i;
479 
480 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
481 		pci_update_resource(dev, i);
482 }
483 
484 static struct pci_platform_pm_ops *pci_platform_pm;
485 
486 int pci_set_platform_pm(struct pci_platform_pm_ops *ops)
487 {
488 	if (!ops->is_manageable || !ops->set_state || !ops->choose_state
489 	    || !ops->sleep_wake)
490 		return -EINVAL;
491 	pci_platform_pm = ops;
492 	return 0;
493 }
494 
495 static inline bool platform_pci_power_manageable(struct pci_dev *dev)
496 {
497 	return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false;
498 }
499 
500 static inline int platform_pci_set_power_state(struct pci_dev *dev,
501 					       pci_power_t t)
502 {
503 	return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
504 }
505 
506 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
507 {
508 	return pci_platform_pm ?
509 			pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR;
510 }
511 
512 static inline int platform_pci_sleep_wake(struct pci_dev *dev, bool enable)
513 {
514 	return pci_platform_pm ?
515 			pci_platform_pm->sleep_wake(dev, enable) : -ENODEV;
516 }
517 
518 static inline int platform_pci_run_wake(struct pci_dev *dev, bool enable)
519 {
520 	return pci_platform_pm ?
521 			pci_platform_pm->run_wake(dev, enable) : -ENODEV;
522 }
523 
524 /**
525  * pci_raw_set_power_state - Use PCI PM registers to set the power state of
526  *                           given PCI device
527  * @dev: PCI device to handle.
528  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
529  *
530  * RETURN VALUE:
531  * -EINVAL if the requested state is invalid.
532  * -EIO if device does not support PCI PM or its PM capabilities register has a
533  * wrong version, or device doesn't support the requested state.
534  * 0 if device already is in the requested state.
535  * 0 if device's power state has been successfully changed.
536  */
537 static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
538 {
539 	u16 pmcsr;
540 	bool need_restore = false;
541 
542 	/* Check if we're already there */
543 	if (dev->current_state == state)
544 		return 0;
545 
546 	if (!dev->pm_cap)
547 		return -EIO;
548 
549 	if (state < PCI_D0 || state > PCI_D3hot)
550 		return -EINVAL;
551 
552 	/* Validate current state:
553 	 * Can enter D0 from any state, but if we can only go deeper
554 	 * to sleep if we're already in a low power state
555 	 */
556 	if (state != PCI_D0 && dev->current_state <= PCI_D3cold
557 	    && dev->current_state > state) {
558 		dev_err(&dev->dev, "invalid power transition (from state %d to %d)\n",
559 			dev->current_state, state);
560 		return -EINVAL;
561 	}
562 
563 	/* check if this device supports the desired state */
564 	if ((state == PCI_D1 && !dev->d1_support)
565 	   || (state == PCI_D2 && !dev->d2_support))
566 		return -EIO;
567 
568 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
569 
570 	/* If we're (effectively) in D3, force entire word to 0.
571 	 * This doesn't affect PME_Status, disables PME_En, and
572 	 * sets PowerState to 0.
573 	 */
574 	switch (dev->current_state) {
575 	case PCI_D0:
576 	case PCI_D1:
577 	case PCI_D2:
578 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
579 		pmcsr |= state;
580 		break;
581 	case PCI_D3hot:
582 	case PCI_D3cold:
583 	case PCI_UNKNOWN: /* Boot-up */
584 		if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot
585 		 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET))
586 			need_restore = true;
587 		/* Fall-through: force to D0 */
588 	default:
589 		pmcsr = 0;
590 		break;
591 	}
592 
593 	/* enter specified state */
594 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
595 
596 	/* Mandatory power management transition delays */
597 	/* see PCI PM 1.1 5.6.1 table 18 */
598 	if (state == PCI_D3hot || dev->current_state == PCI_D3hot)
599 		pci_dev_d3_sleep(dev);
600 	else if (state == PCI_D2 || dev->current_state == PCI_D2)
601 		udelay(PCI_PM_D2_DELAY);
602 
603 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
604 	dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
605 	if (dev->current_state != state && printk_ratelimit())
606 		dev_info(&dev->dev, "Refused to change power state, currently in D%d\n",
607 			 dev->current_state);
608 
609 	/*
610 	 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
611 	 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
612 	 * from D3hot to D0 _may_ perform an internal reset, thereby
613 	 * going to "D0 Uninitialized" rather than "D0 Initialized".
614 	 * For example, at least some versions of the 3c905B and the
615 	 * 3c556B exhibit this behaviour.
616 	 *
617 	 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
618 	 * devices in a D3hot state at boot.  Consequently, we need to
619 	 * restore at least the BARs so that the device will be
620 	 * accessible to its driver.
621 	 */
622 	if (need_restore)
623 		pci_restore_bars(dev);
624 
625 	if (dev->bus->self)
626 		pcie_aspm_pm_state_change(dev->bus->self);
627 
628 	return 0;
629 }
630 
631 /**
632  * pci_update_current_state - Read PCI power state of given device from its
633  *                            PCI PM registers and cache it
634  * @dev: PCI device to handle.
635  * @state: State to cache in case the device doesn't have the PM capability
636  */
637 void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
638 {
639 	if (dev->pm_cap) {
640 		u16 pmcsr;
641 
642 		/*
643 		 * Configuration space is not accessible for device in
644 		 * D3cold, so just keep or set D3cold for safety
645 		 */
646 		if (dev->current_state == PCI_D3cold)
647 			return;
648 		if (state == PCI_D3cold) {
649 			dev->current_state = PCI_D3cold;
650 			return;
651 		}
652 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
653 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
654 	} else {
655 		dev->current_state = state;
656 	}
657 }
658 
659 /**
660  * pci_power_up - Put the given device into D0 forcibly
661  * @dev: PCI device to power up
662  */
663 void pci_power_up(struct pci_dev *dev)
664 {
665 	if (platform_pci_power_manageable(dev))
666 		platform_pci_set_power_state(dev, PCI_D0);
667 
668 	pci_raw_set_power_state(dev, PCI_D0);
669 	pci_update_current_state(dev, PCI_D0);
670 }
671 
672 /**
673  * pci_platform_power_transition - Use platform to change device power state
674  * @dev: PCI device to handle.
675  * @state: State to put the device into.
676  */
677 static int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
678 {
679 	int error;
680 
681 	if (platform_pci_power_manageable(dev)) {
682 		error = platform_pci_set_power_state(dev, state);
683 		if (!error)
684 			pci_update_current_state(dev, state);
685 	} else
686 		error = -ENODEV;
687 
688 	if (error && !dev->pm_cap) /* Fall back to PCI_D0 */
689 		dev->current_state = PCI_D0;
690 
691 	return error;
692 }
693 
694 /**
695  * pci_wakeup - Wake up a PCI device
696  * @pci_dev: Device to handle.
697  * @ign: ignored parameter
698  */
699 static int pci_wakeup(struct pci_dev *pci_dev, void *ign)
700 {
701 	pci_wakeup_event(pci_dev);
702 	pm_request_resume(&pci_dev->dev);
703 	return 0;
704 }
705 
706 /**
707  * pci_wakeup_bus - Walk given bus and wake up devices on it
708  * @bus: Top bus of the subtree to walk.
709  */
710 static void pci_wakeup_bus(struct pci_bus *bus)
711 {
712 	if (bus)
713 		pci_walk_bus(bus, pci_wakeup, NULL);
714 }
715 
716 /**
717  * __pci_start_power_transition - Start power transition of a PCI device
718  * @dev: PCI device to handle.
719  * @state: State to put the device into.
720  */
721 static void __pci_start_power_transition(struct pci_dev *dev, pci_power_t state)
722 {
723 	if (state == PCI_D0) {
724 		pci_platform_power_transition(dev, PCI_D0);
725 		/*
726 		 * Mandatory power management transition delays, see
727 		 * PCI Express Base Specification Revision 2.0 Section
728 		 * 6.6.1: Conventional Reset.  Do not delay for
729 		 * devices powered on/off by corresponding bridge,
730 		 * because have already delayed for the bridge.
731 		 */
732 		if (dev->runtime_d3cold) {
733 			msleep(dev->d3cold_delay);
734 			/*
735 			 * When powering on a bridge from D3cold, the
736 			 * whole hierarchy may be powered on into
737 			 * D0uninitialized state, resume them to give
738 			 * them a chance to suspend again
739 			 */
740 			pci_wakeup_bus(dev->subordinate);
741 		}
742 	}
743 }
744 
745 /**
746  * __pci_dev_set_current_state - Set current state of a PCI device
747  * @dev: Device to handle
748  * @data: pointer to state to be set
749  */
750 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
751 {
752 	pci_power_t state = *(pci_power_t *)data;
753 
754 	dev->current_state = state;
755 	return 0;
756 }
757 
758 /**
759  * __pci_bus_set_current_state - Walk given bus and set current state of devices
760  * @bus: Top bus of the subtree to walk.
761  * @state: state to be set
762  */
763 static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
764 {
765 	if (bus)
766 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
767 }
768 
769 /**
770  * __pci_complete_power_transition - Complete power transition of a PCI device
771  * @dev: PCI device to handle.
772  * @state: State to put the device into.
773  *
774  * This function should not be called directly by device drivers.
775  */
776 int __pci_complete_power_transition(struct pci_dev *dev, pci_power_t state)
777 {
778 	int ret;
779 
780 	if (state <= PCI_D0)
781 		return -EINVAL;
782 	ret = pci_platform_power_transition(dev, state);
783 	/* Power off the bridge may power off the whole hierarchy */
784 	if (!ret && state == PCI_D3cold)
785 		__pci_bus_set_current_state(dev->subordinate, PCI_D3cold);
786 	return ret;
787 }
788 EXPORT_SYMBOL_GPL(__pci_complete_power_transition);
789 
790 /**
791  * pci_set_power_state - Set the power state of a PCI device
792  * @dev: PCI device to handle.
793  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
794  *
795  * Transition a device to a new power state, using the platform firmware and/or
796  * the device's PCI PM registers.
797  *
798  * RETURN VALUE:
799  * -EINVAL if the requested state is invalid.
800  * -EIO if device does not support PCI PM or its PM capabilities register has a
801  * wrong version, or device doesn't support the requested state.
802  * 0 if device already is in the requested state.
803  * 0 if device's power state has been successfully changed.
804  */
805 int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
806 {
807 	int error;
808 
809 	/* bound the state we're entering */
810 	if (state > PCI_D3cold)
811 		state = PCI_D3cold;
812 	else if (state < PCI_D0)
813 		state = PCI_D0;
814 	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
815 		/*
816 		 * If the device or the parent bridge do not support PCI PM,
817 		 * ignore the request if we're doing anything other than putting
818 		 * it into D0 (which would only happen on boot).
819 		 */
820 		return 0;
821 
822 	/* Check if we're already there */
823 	if (dev->current_state == state)
824 		return 0;
825 
826 	__pci_start_power_transition(dev, state);
827 
828 	/* This device is quirked not to be put into D3, so
829 	   don't put it in D3 */
830 	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
831 		return 0;
832 
833 	/*
834 	 * To put device in D3cold, we put device into D3hot in native
835 	 * way, then put device into D3cold with platform ops
836 	 */
837 	error = pci_raw_set_power_state(dev, state > PCI_D3hot ?
838 					PCI_D3hot : state);
839 
840 	if (!__pci_complete_power_transition(dev, state))
841 		error = 0;
842 	/*
843 	 * When aspm_policy is "powersave" this call ensures
844 	 * that ASPM is configured.
845 	 */
846 	if (!error && dev->bus->self)
847 		pcie_aspm_powersave_config_link(dev->bus->self);
848 
849 	return error;
850 }
851 EXPORT_SYMBOL(pci_set_power_state);
852 
853 /**
854  * pci_choose_state - Choose the power state of a PCI device
855  * @dev: PCI device to be suspended
856  * @state: target sleep state for the whole system. This is the value
857  *	that is passed to suspend() function.
858  *
859  * Returns PCI power state suitable for given device and given system
860  * message.
861  */
862 
863 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
864 {
865 	pci_power_t ret;
866 
867 	if (!dev->pm_cap)
868 		return PCI_D0;
869 
870 	ret = platform_pci_choose_state(dev);
871 	if (ret != PCI_POWER_ERROR)
872 		return ret;
873 
874 	switch (state.event) {
875 	case PM_EVENT_ON:
876 		return PCI_D0;
877 	case PM_EVENT_FREEZE:
878 	case PM_EVENT_PRETHAW:
879 		/* REVISIT both freeze and pre-thaw "should" use D0 */
880 	case PM_EVENT_SUSPEND:
881 	case PM_EVENT_HIBERNATE:
882 		return PCI_D3hot;
883 	default:
884 		dev_info(&dev->dev, "unrecognized suspend event %d\n",
885 			 state.event);
886 		BUG();
887 	}
888 	return PCI_D0;
889 }
890 EXPORT_SYMBOL(pci_choose_state);
891 
892 #define PCI_EXP_SAVE_REGS	7
893 
894 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
895 						       u16 cap, bool extended)
896 {
897 	struct pci_cap_saved_state *tmp;
898 
899 	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
900 		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
901 			return tmp;
902 	}
903 	return NULL;
904 }
905 
906 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
907 {
908 	return _pci_find_saved_cap(dev, cap, false);
909 }
910 
911 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
912 {
913 	return _pci_find_saved_cap(dev, cap, true);
914 }
915 
916 static int pci_save_pcie_state(struct pci_dev *dev)
917 {
918 	int i = 0;
919 	struct pci_cap_saved_state *save_state;
920 	u16 *cap;
921 
922 	if (!pci_is_pcie(dev))
923 		return 0;
924 
925 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
926 	if (!save_state) {
927 		dev_err(&dev->dev, "buffer not found in %s\n", __func__);
928 		return -ENOMEM;
929 	}
930 
931 	cap = (u16 *)&save_state->cap.data[0];
932 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
933 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
934 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
935 	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
936 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
937 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
938 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
939 
940 	return 0;
941 }
942 
943 static void pci_restore_pcie_state(struct pci_dev *dev)
944 {
945 	int i = 0;
946 	struct pci_cap_saved_state *save_state;
947 	u16 *cap;
948 
949 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
950 	if (!save_state)
951 		return;
952 
953 	cap = (u16 *)&save_state->cap.data[0];
954 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
955 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
956 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
957 	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
958 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
959 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
960 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
961 }
962 
963 
964 static int pci_save_pcix_state(struct pci_dev *dev)
965 {
966 	int pos;
967 	struct pci_cap_saved_state *save_state;
968 
969 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
970 	if (pos <= 0)
971 		return 0;
972 
973 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
974 	if (!save_state) {
975 		dev_err(&dev->dev, "buffer not found in %s\n", __func__);
976 		return -ENOMEM;
977 	}
978 
979 	pci_read_config_word(dev, pos + PCI_X_CMD,
980 			     (u16 *)save_state->cap.data);
981 
982 	return 0;
983 }
984 
985 static void pci_restore_pcix_state(struct pci_dev *dev)
986 {
987 	int i = 0, pos;
988 	struct pci_cap_saved_state *save_state;
989 	u16 *cap;
990 
991 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
992 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
993 	if (!save_state || pos <= 0)
994 		return;
995 	cap = (u16 *)&save_state->cap.data[0];
996 
997 	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
998 }
999 
1000 
1001 /**
1002  * pci_save_state - save the PCI configuration space of a device before suspending
1003  * @dev: - PCI device that we're dealing with
1004  */
1005 int pci_save_state(struct pci_dev *dev)
1006 {
1007 	int i;
1008 	/* XXX: 100% dword access ok here? */
1009 	for (i = 0; i < 16; i++)
1010 		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1011 	dev->state_saved = true;
1012 	if ((i = pci_save_pcie_state(dev)) != 0)
1013 		return i;
1014 	if ((i = pci_save_pcix_state(dev)) != 0)
1015 		return i;
1016 	if ((i = pci_save_vc_state(dev)) != 0)
1017 		return i;
1018 	return 0;
1019 }
1020 EXPORT_SYMBOL(pci_save_state);
1021 
1022 static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1023 				     u32 saved_val, int retry)
1024 {
1025 	u32 val;
1026 
1027 	pci_read_config_dword(pdev, offset, &val);
1028 	if (val == saved_val)
1029 		return;
1030 
1031 	for (;;) {
1032 		dev_dbg(&pdev->dev, "restoring config space at offset %#x (was %#x, writing %#x)\n",
1033 			offset, val, saved_val);
1034 		pci_write_config_dword(pdev, offset, saved_val);
1035 		if (retry-- <= 0)
1036 			return;
1037 
1038 		pci_read_config_dword(pdev, offset, &val);
1039 		if (val == saved_val)
1040 			return;
1041 
1042 		mdelay(1);
1043 	}
1044 }
1045 
1046 static void pci_restore_config_space_range(struct pci_dev *pdev,
1047 					   int start, int end, int retry)
1048 {
1049 	int index;
1050 
1051 	for (index = end; index >= start; index--)
1052 		pci_restore_config_dword(pdev, 4 * index,
1053 					 pdev->saved_config_space[index],
1054 					 retry);
1055 }
1056 
1057 static void pci_restore_config_space(struct pci_dev *pdev)
1058 {
1059 	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1060 		pci_restore_config_space_range(pdev, 10, 15, 0);
1061 		/* Restore BARs before the command register. */
1062 		pci_restore_config_space_range(pdev, 4, 9, 10);
1063 		pci_restore_config_space_range(pdev, 0, 3, 0);
1064 	} else {
1065 		pci_restore_config_space_range(pdev, 0, 15, 0);
1066 	}
1067 }
1068 
1069 /**
1070  * pci_restore_state - Restore the saved state of a PCI device
1071  * @dev: - PCI device that we're dealing with
1072  */
1073 void pci_restore_state(struct pci_dev *dev)
1074 {
1075 	if (!dev->state_saved)
1076 		return;
1077 
1078 	/* PCI Express register must be restored first */
1079 	pci_restore_pcie_state(dev);
1080 	pci_restore_ats_state(dev);
1081 	pci_restore_vc_state(dev);
1082 
1083 	pci_restore_config_space(dev);
1084 
1085 	pci_restore_pcix_state(dev);
1086 	pci_restore_msi_state(dev);
1087 	pci_restore_iov_state(dev);
1088 
1089 	dev->state_saved = false;
1090 }
1091 EXPORT_SYMBOL(pci_restore_state);
1092 
1093 struct pci_saved_state {
1094 	u32 config_space[16];
1095 	struct pci_cap_saved_data cap[0];
1096 };
1097 
1098 /**
1099  * pci_store_saved_state - Allocate and return an opaque struct containing
1100  *			   the device saved state.
1101  * @dev: PCI device that we're dealing with
1102  *
1103  * Return NULL if no state or error.
1104  */
1105 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1106 {
1107 	struct pci_saved_state *state;
1108 	struct pci_cap_saved_state *tmp;
1109 	struct pci_cap_saved_data *cap;
1110 	size_t size;
1111 
1112 	if (!dev->state_saved)
1113 		return NULL;
1114 
1115 	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1116 
1117 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1118 		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1119 
1120 	state = kzalloc(size, GFP_KERNEL);
1121 	if (!state)
1122 		return NULL;
1123 
1124 	memcpy(state->config_space, dev->saved_config_space,
1125 	       sizeof(state->config_space));
1126 
1127 	cap = state->cap;
1128 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1129 		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1130 		memcpy(cap, &tmp->cap, len);
1131 		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1132 	}
1133 	/* Empty cap_save terminates list */
1134 
1135 	return state;
1136 }
1137 EXPORT_SYMBOL_GPL(pci_store_saved_state);
1138 
1139 /**
1140  * pci_load_saved_state - Reload the provided save state into struct pci_dev.
1141  * @dev: PCI device that we're dealing with
1142  * @state: Saved state returned from pci_store_saved_state()
1143  */
1144 static int pci_load_saved_state(struct pci_dev *dev,
1145 				struct pci_saved_state *state)
1146 {
1147 	struct pci_cap_saved_data *cap;
1148 
1149 	dev->state_saved = false;
1150 
1151 	if (!state)
1152 		return 0;
1153 
1154 	memcpy(dev->saved_config_space, state->config_space,
1155 	       sizeof(state->config_space));
1156 
1157 	cap = state->cap;
1158 	while (cap->size) {
1159 		struct pci_cap_saved_state *tmp;
1160 
1161 		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
1162 		if (!tmp || tmp->cap.size != cap->size)
1163 			return -EINVAL;
1164 
1165 		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
1166 		cap = (struct pci_cap_saved_data *)((u8 *)cap +
1167 		       sizeof(struct pci_cap_saved_data) + cap->size);
1168 	}
1169 
1170 	dev->state_saved = true;
1171 	return 0;
1172 }
1173 
1174 /**
1175  * pci_load_and_free_saved_state - Reload the save state pointed to by state,
1176  *				   and free the memory allocated for it.
1177  * @dev: PCI device that we're dealing with
1178  * @state: Pointer to saved state returned from pci_store_saved_state()
1179  */
1180 int pci_load_and_free_saved_state(struct pci_dev *dev,
1181 				  struct pci_saved_state **state)
1182 {
1183 	int ret = pci_load_saved_state(dev, *state);
1184 	kfree(*state);
1185 	*state = NULL;
1186 	return ret;
1187 }
1188 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
1189 
1190 int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
1191 {
1192 	return pci_enable_resources(dev, bars);
1193 }
1194 
1195 static int do_pci_enable_device(struct pci_dev *dev, int bars)
1196 {
1197 	int err;
1198 	u16 cmd;
1199 	u8 pin;
1200 
1201 	err = pci_set_power_state(dev, PCI_D0);
1202 	if (err < 0 && err != -EIO)
1203 		return err;
1204 	err = pcibios_enable_device(dev, bars);
1205 	if (err < 0)
1206 		return err;
1207 	pci_fixup_device(pci_fixup_enable, dev);
1208 
1209 	if (dev->msi_enabled || dev->msix_enabled)
1210 		return 0;
1211 
1212 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
1213 	if (pin) {
1214 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
1215 		if (cmd & PCI_COMMAND_INTX_DISABLE)
1216 			pci_write_config_word(dev, PCI_COMMAND,
1217 					      cmd & ~PCI_COMMAND_INTX_DISABLE);
1218 	}
1219 
1220 	return 0;
1221 }
1222 
1223 /**
1224  * pci_reenable_device - Resume abandoned device
1225  * @dev: PCI device to be resumed
1226  *
1227  *  Note this function is a backend of pci_default_resume and is not supposed
1228  *  to be called by normal code, write proper resume handler and use it instead.
1229  */
1230 int pci_reenable_device(struct pci_dev *dev)
1231 {
1232 	if (pci_is_enabled(dev))
1233 		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
1234 	return 0;
1235 }
1236 EXPORT_SYMBOL(pci_reenable_device);
1237 
1238 static void pci_enable_bridge(struct pci_dev *dev)
1239 {
1240 	struct pci_dev *bridge;
1241 	int retval;
1242 
1243 	bridge = pci_upstream_bridge(dev);
1244 	if (bridge)
1245 		pci_enable_bridge(bridge);
1246 
1247 	if (pci_is_enabled(dev)) {
1248 		if (!dev->is_busmaster)
1249 			pci_set_master(dev);
1250 		return;
1251 	}
1252 
1253 	retval = pci_enable_device(dev);
1254 	if (retval)
1255 		dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n",
1256 			retval);
1257 	pci_set_master(dev);
1258 }
1259 
1260 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
1261 {
1262 	struct pci_dev *bridge;
1263 	int err;
1264 	int i, bars = 0;
1265 
1266 	/*
1267 	 * Power state could be unknown at this point, either due to a fresh
1268 	 * boot or a device removal call.  So get the current power state
1269 	 * so that things like MSI message writing will behave as expected
1270 	 * (e.g. if the device really is in D0 at enable time).
1271 	 */
1272 	if (dev->pm_cap) {
1273 		u16 pmcsr;
1274 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1275 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
1276 	}
1277 
1278 	if (atomic_inc_return(&dev->enable_cnt) > 1)
1279 		return 0;		/* already enabled */
1280 
1281 	bridge = pci_upstream_bridge(dev);
1282 	if (bridge)
1283 		pci_enable_bridge(bridge);
1284 
1285 	/* only skip sriov related */
1286 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1287 		if (dev->resource[i].flags & flags)
1288 			bars |= (1 << i);
1289 	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
1290 		if (dev->resource[i].flags & flags)
1291 			bars |= (1 << i);
1292 
1293 	err = do_pci_enable_device(dev, bars);
1294 	if (err < 0)
1295 		atomic_dec(&dev->enable_cnt);
1296 	return err;
1297 }
1298 
1299 /**
1300  * pci_enable_device_io - Initialize a device for use with IO space
1301  * @dev: PCI device to be initialized
1302  *
1303  *  Initialize device before it's used by a driver. Ask low-level code
1304  *  to enable I/O resources. Wake up the device if it was suspended.
1305  *  Beware, this function can fail.
1306  */
1307 int pci_enable_device_io(struct pci_dev *dev)
1308 {
1309 	return pci_enable_device_flags(dev, IORESOURCE_IO);
1310 }
1311 EXPORT_SYMBOL(pci_enable_device_io);
1312 
1313 /**
1314  * pci_enable_device_mem - Initialize a device for use with Memory space
1315  * @dev: PCI device to be initialized
1316  *
1317  *  Initialize device before it's used by a driver. Ask low-level code
1318  *  to enable Memory resources. Wake up the device if it was suspended.
1319  *  Beware, this function can fail.
1320  */
1321 int pci_enable_device_mem(struct pci_dev *dev)
1322 {
1323 	return pci_enable_device_flags(dev, IORESOURCE_MEM);
1324 }
1325 EXPORT_SYMBOL(pci_enable_device_mem);
1326 
1327 /**
1328  * pci_enable_device - Initialize device before it's used by a driver.
1329  * @dev: PCI device to be initialized
1330  *
1331  *  Initialize device before it's used by a driver. Ask low-level code
1332  *  to enable I/O and memory. Wake up the device if it was suspended.
1333  *  Beware, this function can fail.
1334  *
1335  *  Note we don't actually enable the device many times if we call
1336  *  this function repeatedly (we just increment the count).
1337  */
1338 int pci_enable_device(struct pci_dev *dev)
1339 {
1340 	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
1341 }
1342 EXPORT_SYMBOL(pci_enable_device);
1343 
1344 /*
1345  * Managed PCI resources.  This manages device on/off, intx/msi/msix
1346  * on/off and BAR regions.  pci_dev itself records msi/msix status, so
1347  * there's no need to track it separately.  pci_devres is initialized
1348  * when a device is enabled using managed PCI device enable interface.
1349  */
1350 struct pci_devres {
1351 	unsigned int enabled:1;
1352 	unsigned int pinned:1;
1353 	unsigned int orig_intx:1;
1354 	unsigned int restore_intx:1;
1355 	u32 region_mask;
1356 };
1357 
1358 static void pcim_release(struct device *gendev, void *res)
1359 {
1360 	struct pci_dev *dev = container_of(gendev, struct pci_dev, dev);
1361 	struct pci_devres *this = res;
1362 	int i;
1363 
1364 	if (dev->msi_enabled)
1365 		pci_disable_msi(dev);
1366 	if (dev->msix_enabled)
1367 		pci_disable_msix(dev);
1368 
1369 	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
1370 		if (this->region_mask & (1 << i))
1371 			pci_release_region(dev, i);
1372 
1373 	if (this->restore_intx)
1374 		pci_intx(dev, this->orig_intx);
1375 
1376 	if (this->enabled && !this->pinned)
1377 		pci_disable_device(dev);
1378 }
1379 
1380 static struct pci_devres *get_pci_dr(struct pci_dev *pdev)
1381 {
1382 	struct pci_devres *dr, *new_dr;
1383 
1384 	dr = devres_find(&pdev->dev, pcim_release, NULL, NULL);
1385 	if (dr)
1386 		return dr;
1387 
1388 	new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL);
1389 	if (!new_dr)
1390 		return NULL;
1391 	return devres_get(&pdev->dev, new_dr, NULL, NULL);
1392 }
1393 
1394 static struct pci_devres *find_pci_dr(struct pci_dev *pdev)
1395 {
1396 	if (pci_is_managed(pdev))
1397 		return devres_find(&pdev->dev, pcim_release, NULL, NULL);
1398 	return NULL;
1399 }
1400 
1401 /**
1402  * pcim_enable_device - Managed pci_enable_device()
1403  * @pdev: PCI device to be initialized
1404  *
1405  * Managed pci_enable_device().
1406  */
1407 int pcim_enable_device(struct pci_dev *pdev)
1408 {
1409 	struct pci_devres *dr;
1410 	int rc;
1411 
1412 	dr = get_pci_dr(pdev);
1413 	if (unlikely(!dr))
1414 		return -ENOMEM;
1415 	if (dr->enabled)
1416 		return 0;
1417 
1418 	rc = pci_enable_device(pdev);
1419 	if (!rc) {
1420 		pdev->is_managed = 1;
1421 		dr->enabled = 1;
1422 	}
1423 	return rc;
1424 }
1425 EXPORT_SYMBOL(pcim_enable_device);
1426 
1427 /**
1428  * pcim_pin_device - Pin managed PCI device
1429  * @pdev: PCI device to pin
1430  *
1431  * Pin managed PCI device @pdev.  Pinned device won't be disabled on
1432  * driver detach.  @pdev must have been enabled with
1433  * pcim_enable_device().
1434  */
1435 void pcim_pin_device(struct pci_dev *pdev)
1436 {
1437 	struct pci_devres *dr;
1438 
1439 	dr = find_pci_dr(pdev);
1440 	WARN_ON(!dr || !dr->enabled);
1441 	if (dr)
1442 		dr->pinned = 1;
1443 }
1444 EXPORT_SYMBOL(pcim_pin_device);
1445 
1446 /*
1447  * pcibios_add_device - provide arch specific hooks when adding device dev
1448  * @dev: the PCI device being added
1449  *
1450  * Permits the platform to provide architecture specific functionality when
1451  * devices are added. This is the default implementation. Architecture
1452  * implementations can override this.
1453  */
1454 int __weak pcibios_add_device(struct pci_dev *dev)
1455 {
1456 	return 0;
1457 }
1458 
1459 /**
1460  * pcibios_release_device - provide arch specific hooks when releasing device dev
1461  * @dev: the PCI device being released
1462  *
1463  * Permits the platform to provide architecture specific functionality when
1464  * devices are released. This is the default implementation. Architecture
1465  * implementations can override this.
1466  */
1467 void __weak pcibios_release_device(struct pci_dev *dev) {}
1468 
1469 /**
1470  * pcibios_disable_device - disable arch specific PCI resources for device dev
1471  * @dev: the PCI device to disable
1472  *
1473  * Disables architecture specific PCI resources for the device. This
1474  * is the default implementation. Architecture implementations can
1475  * override this.
1476  */
1477 void __weak pcibios_disable_device (struct pci_dev *dev) {}
1478 
1479 /**
1480  * pcibios_penalize_isa_irq - penalize an ISA IRQ
1481  * @irq: ISA IRQ to penalize
1482  * @active: IRQ active or not
1483  *
1484  * Permits the platform to provide architecture-specific functionality when
1485  * penalizing ISA IRQs. This is the default implementation. Architecture
1486  * implementations can override this.
1487  */
1488 void __weak pcibios_penalize_isa_irq(int irq, int active) {}
1489 
1490 static void do_pci_disable_device(struct pci_dev *dev)
1491 {
1492 	u16 pci_command;
1493 
1494 	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
1495 	if (pci_command & PCI_COMMAND_MASTER) {
1496 		pci_command &= ~PCI_COMMAND_MASTER;
1497 		pci_write_config_word(dev, PCI_COMMAND, pci_command);
1498 	}
1499 
1500 	pcibios_disable_device(dev);
1501 }
1502 
1503 /**
1504  * pci_disable_enabled_device - Disable device without updating enable_cnt
1505  * @dev: PCI device to disable
1506  *
1507  * NOTE: This function is a backend of PCI power management routines and is
1508  * not supposed to be called drivers.
1509  */
1510 void pci_disable_enabled_device(struct pci_dev *dev)
1511 {
1512 	if (pci_is_enabled(dev))
1513 		do_pci_disable_device(dev);
1514 }
1515 
1516 /**
1517  * pci_disable_device - Disable PCI device after use
1518  * @dev: PCI device to be disabled
1519  *
1520  * Signal to the system that the PCI device is not in use by the system
1521  * anymore.  This only involves disabling PCI bus-mastering, if active.
1522  *
1523  * Note we don't actually disable the device until all callers of
1524  * pci_enable_device() have called pci_disable_device().
1525  */
1526 void pci_disable_device(struct pci_dev *dev)
1527 {
1528 	struct pci_devres *dr;
1529 
1530 	dr = find_pci_dr(dev);
1531 	if (dr)
1532 		dr->enabled = 0;
1533 
1534 	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
1535 		      "disabling already-disabled device");
1536 
1537 	if (atomic_dec_return(&dev->enable_cnt) != 0)
1538 		return;
1539 
1540 	do_pci_disable_device(dev);
1541 
1542 	dev->is_busmaster = 0;
1543 }
1544 EXPORT_SYMBOL(pci_disable_device);
1545 
1546 /**
1547  * pcibios_set_pcie_reset_state - set reset state for device dev
1548  * @dev: the PCIe device reset
1549  * @state: Reset state to enter into
1550  *
1551  *
1552  * Sets the PCIe reset state for the device. This is the default
1553  * implementation. Architecture implementations can override this.
1554  */
1555 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
1556 					enum pcie_reset_state state)
1557 {
1558 	return -EINVAL;
1559 }
1560 
1561 /**
1562  * pci_set_pcie_reset_state - set reset state for device dev
1563  * @dev: the PCIe device reset
1564  * @state: Reset state to enter into
1565  *
1566  *
1567  * Sets the PCI reset state for the device.
1568  */
1569 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
1570 {
1571 	return pcibios_set_pcie_reset_state(dev, state);
1572 }
1573 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
1574 
1575 /**
1576  * pci_check_pme_status - Check if given device has generated PME.
1577  * @dev: Device to check.
1578  *
1579  * Check the PME status of the device and if set, clear it and clear PME enable
1580  * (if set).  Return 'true' if PME status and PME enable were both set or
1581  * 'false' otherwise.
1582  */
1583 bool pci_check_pme_status(struct pci_dev *dev)
1584 {
1585 	int pmcsr_pos;
1586 	u16 pmcsr;
1587 	bool ret = false;
1588 
1589 	if (!dev->pm_cap)
1590 		return false;
1591 
1592 	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
1593 	pci_read_config_word(dev, pmcsr_pos, &pmcsr);
1594 	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
1595 		return false;
1596 
1597 	/* Clear PME status. */
1598 	pmcsr |= PCI_PM_CTRL_PME_STATUS;
1599 	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
1600 		/* Disable PME to avoid interrupt flood. */
1601 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
1602 		ret = true;
1603 	}
1604 
1605 	pci_write_config_word(dev, pmcsr_pos, pmcsr);
1606 
1607 	return ret;
1608 }
1609 
1610 /**
1611  * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
1612  * @dev: Device to handle.
1613  * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
1614  *
1615  * Check if @dev has generated PME and queue a resume request for it in that
1616  * case.
1617  */
1618 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
1619 {
1620 	if (pme_poll_reset && dev->pme_poll)
1621 		dev->pme_poll = false;
1622 
1623 	if (pci_check_pme_status(dev)) {
1624 		pci_wakeup_event(dev);
1625 		pm_request_resume(&dev->dev);
1626 	}
1627 	return 0;
1628 }
1629 
1630 /**
1631  * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
1632  * @bus: Top bus of the subtree to walk.
1633  */
1634 void pci_pme_wakeup_bus(struct pci_bus *bus)
1635 {
1636 	if (bus)
1637 		pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
1638 }
1639 
1640 
1641 /**
1642  * pci_pme_capable - check the capability of PCI device to generate PME#
1643  * @dev: PCI device to handle.
1644  * @state: PCI state from which device will issue PME#.
1645  */
1646 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
1647 {
1648 	if (!dev->pm_cap)
1649 		return false;
1650 
1651 	return !!(dev->pme_support & (1 << state));
1652 }
1653 EXPORT_SYMBOL(pci_pme_capable);
1654 
1655 static void pci_pme_list_scan(struct work_struct *work)
1656 {
1657 	struct pci_pme_device *pme_dev, *n;
1658 
1659 	mutex_lock(&pci_pme_list_mutex);
1660 	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
1661 		if (pme_dev->dev->pme_poll) {
1662 			struct pci_dev *bridge;
1663 
1664 			bridge = pme_dev->dev->bus->self;
1665 			/*
1666 			 * If bridge is in low power state, the
1667 			 * configuration space of subordinate devices
1668 			 * may be not accessible
1669 			 */
1670 			if (bridge && bridge->current_state != PCI_D0)
1671 				continue;
1672 			pci_pme_wakeup(pme_dev->dev, NULL);
1673 		} else {
1674 			list_del(&pme_dev->list);
1675 			kfree(pme_dev);
1676 		}
1677 	}
1678 	if (!list_empty(&pci_pme_list))
1679 		schedule_delayed_work(&pci_pme_work,
1680 				      msecs_to_jiffies(PME_TIMEOUT));
1681 	mutex_unlock(&pci_pme_list_mutex);
1682 }
1683 
1684 /**
1685  * pci_pme_active - enable or disable PCI device's PME# function
1686  * @dev: PCI device to handle.
1687  * @enable: 'true' to enable PME# generation; 'false' to disable it.
1688  *
1689  * The caller must verify that the device is capable of generating PME# before
1690  * calling this function with @enable equal to 'true'.
1691  */
1692 void pci_pme_active(struct pci_dev *dev, bool enable)
1693 {
1694 	u16 pmcsr;
1695 
1696 	if (!dev->pme_support)
1697 		return;
1698 
1699 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1700 	/* Clear PME_Status by writing 1 to it and enable PME# */
1701 	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
1702 	if (!enable)
1703 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
1704 
1705 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
1706 
1707 	/*
1708 	 * PCI (as opposed to PCIe) PME requires that the device have
1709 	 * its PME# line hooked up correctly. Not all hardware vendors
1710 	 * do this, so the PME never gets delivered and the device
1711 	 * remains asleep. The easiest way around this is to
1712 	 * periodically walk the list of suspended devices and check
1713 	 * whether any have their PME flag set. The assumption is that
1714 	 * we'll wake up often enough anyway that this won't be a huge
1715 	 * hit, and the power savings from the devices will still be a
1716 	 * win.
1717 	 *
1718 	 * Although PCIe uses in-band PME message instead of PME# line
1719 	 * to report PME, PME does not work for some PCIe devices in
1720 	 * reality.  For example, there are devices that set their PME
1721 	 * status bits, but don't really bother to send a PME message;
1722 	 * there are PCI Express Root Ports that don't bother to
1723 	 * trigger interrupts when they receive PME messages from the
1724 	 * devices below.  So PME poll is used for PCIe devices too.
1725 	 */
1726 
1727 	if (dev->pme_poll) {
1728 		struct pci_pme_device *pme_dev;
1729 		if (enable) {
1730 			pme_dev = kmalloc(sizeof(struct pci_pme_device),
1731 					  GFP_KERNEL);
1732 			if (!pme_dev) {
1733 				dev_warn(&dev->dev, "can't enable PME#\n");
1734 				return;
1735 			}
1736 			pme_dev->dev = dev;
1737 			mutex_lock(&pci_pme_list_mutex);
1738 			list_add(&pme_dev->list, &pci_pme_list);
1739 			if (list_is_singular(&pci_pme_list))
1740 				schedule_delayed_work(&pci_pme_work,
1741 						      msecs_to_jiffies(PME_TIMEOUT));
1742 			mutex_unlock(&pci_pme_list_mutex);
1743 		} else {
1744 			mutex_lock(&pci_pme_list_mutex);
1745 			list_for_each_entry(pme_dev, &pci_pme_list, list) {
1746 				if (pme_dev->dev == dev) {
1747 					list_del(&pme_dev->list);
1748 					kfree(pme_dev);
1749 					break;
1750 				}
1751 			}
1752 			mutex_unlock(&pci_pme_list_mutex);
1753 		}
1754 	}
1755 
1756 	dev_dbg(&dev->dev, "PME# %s\n", enable ? "enabled" : "disabled");
1757 }
1758 EXPORT_SYMBOL(pci_pme_active);
1759 
1760 /**
1761  * __pci_enable_wake - enable PCI device as wakeup event source
1762  * @dev: PCI device affected
1763  * @state: PCI state from which device will issue wakeup events
1764  * @runtime: True if the events are to be generated at run time
1765  * @enable: True to enable event generation; false to disable
1766  *
1767  * This enables the device as a wakeup event source, or disables it.
1768  * When such events involves platform-specific hooks, those hooks are
1769  * called automatically by this routine.
1770  *
1771  * Devices with legacy power management (no standard PCI PM capabilities)
1772  * always require such platform hooks.
1773  *
1774  * RETURN VALUE:
1775  * 0 is returned on success
1776  * -EINVAL is returned if device is not supposed to wake up the system
1777  * Error code depending on the platform is returned if both the platform and
1778  * the native mechanism fail to enable the generation of wake-up events
1779  */
1780 int __pci_enable_wake(struct pci_dev *dev, pci_power_t state,
1781 		      bool runtime, bool enable)
1782 {
1783 	int ret = 0;
1784 
1785 	if (enable && !runtime && !device_may_wakeup(&dev->dev))
1786 		return -EINVAL;
1787 
1788 	/* Don't do the same thing twice in a row for one device. */
1789 	if (!!enable == !!dev->wakeup_prepared)
1790 		return 0;
1791 
1792 	/*
1793 	 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
1794 	 * Anderson we should be doing PME# wake enable followed by ACPI wake
1795 	 * enable.  To disable wake-up we call the platform first, for symmetry.
1796 	 */
1797 
1798 	if (enable) {
1799 		int error;
1800 
1801 		if (pci_pme_capable(dev, state))
1802 			pci_pme_active(dev, true);
1803 		else
1804 			ret = 1;
1805 		error = runtime ? platform_pci_run_wake(dev, true) :
1806 					platform_pci_sleep_wake(dev, true);
1807 		if (ret)
1808 			ret = error;
1809 		if (!ret)
1810 			dev->wakeup_prepared = true;
1811 	} else {
1812 		if (runtime)
1813 			platform_pci_run_wake(dev, false);
1814 		else
1815 			platform_pci_sleep_wake(dev, false);
1816 		pci_pme_active(dev, false);
1817 		dev->wakeup_prepared = false;
1818 	}
1819 
1820 	return ret;
1821 }
1822 EXPORT_SYMBOL(__pci_enable_wake);
1823 
1824 /**
1825  * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
1826  * @dev: PCI device to prepare
1827  * @enable: True to enable wake-up event generation; false to disable
1828  *
1829  * Many drivers want the device to wake up the system from D3_hot or D3_cold
1830  * and this function allows them to set that up cleanly - pci_enable_wake()
1831  * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
1832  * ordering constraints.
1833  *
1834  * This function only returns error code if the device is not capable of
1835  * generating PME# from both D3_hot and D3_cold, and the platform is unable to
1836  * enable wake-up power for it.
1837  */
1838 int pci_wake_from_d3(struct pci_dev *dev, bool enable)
1839 {
1840 	return pci_pme_capable(dev, PCI_D3cold) ?
1841 			pci_enable_wake(dev, PCI_D3cold, enable) :
1842 			pci_enable_wake(dev, PCI_D3hot, enable);
1843 }
1844 EXPORT_SYMBOL(pci_wake_from_d3);
1845 
1846 /**
1847  * pci_target_state - find an appropriate low power state for a given PCI dev
1848  * @dev: PCI device
1849  *
1850  * Use underlying platform code to find a supported low power state for @dev.
1851  * If the platform can't manage @dev, return the deepest state from which it
1852  * can generate wake events, based on any available PME info.
1853  */
1854 static pci_power_t pci_target_state(struct pci_dev *dev)
1855 {
1856 	pci_power_t target_state = PCI_D3hot;
1857 
1858 	if (platform_pci_power_manageable(dev)) {
1859 		/*
1860 		 * Call the platform to choose the target state of the device
1861 		 * and enable wake-up from this state if supported.
1862 		 */
1863 		pci_power_t state = platform_pci_choose_state(dev);
1864 
1865 		switch (state) {
1866 		case PCI_POWER_ERROR:
1867 		case PCI_UNKNOWN:
1868 			break;
1869 		case PCI_D1:
1870 		case PCI_D2:
1871 			if (pci_no_d1d2(dev))
1872 				break;
1873 		default:
1874 			target_state = state;
1875 		}
1876 	} else if (!dev->pm_cap) {
1877 		target_state = PCI_D0;
1878 	} else if (device_may_wakeup(&dev->dev)) {
1879 		/*
1880 		 * Find the deepest state from which the device can generate
1881 		 * wake-up events, make it the target state and enable device
1882 		 * to generate PME#.
1883 		 */
1884 		if (dev->pme_support) {
1885 			while (target_state
1886 			      && !(dev->pme_support & (1 << target_state)))
1887 				target_state--;
1888 		}
1889 	}
1890 
1891 	return target_state;
1892 }
1893 
1894 /**
1895  * pci_prepare_to_sleep - prepare PCI device for system-wide transition into a sleep state
1896  * @dev: Device to handle.
1897  *
1898  * Choose the power state appropriate for the device depending on whether
1899  * it can wake up the system and/or is power manageable by the platform
1900  * (PCI_D3hot is the default) and put the device into that state.
1901  */
1902 int pci_prepare_to_sleep(struct pci_dev *dev)
1903 {
1904 	pci_power_t target_state = pci_target_state(dev);
1905 	int error;
1906 
1907 	if (target_state == PCI_POWER_ERROR)
1908 		return -EIO;
1909 
1910 	/* D3cold during system suspend/hibernate is not supported */
1911 	if (target_state > PCI_D3hot)
1912 		target_state = PCI_D3hot;
1913 
1914 	pci_enable_wake(dev, target_state, device_may_wakeup(&dev->dev));
1915 
1916 	error = pci_set_power_state(dev, target_state);
1917 
1918 	if (error)
1919 		pci_enable_wake(dev, target_state, false);
1920 
1921 	return error;
1922 }
1923 EXPORT_SYMBOL(pci_prepare_to_sleep);
1924 
1925 /**
1926  * pci_back_from_sleep - turn PCI device on during system-wide transition into working state
1927  * @dev: Device to handle.
1928  *
1929  * Disable device's system wake-up capability and put it into D0.
1930  */
1931 int pci_back_from_sleep(struct pci_dev *dev)
1932 {
1933 	pci_enable_wake(dev, PCI_D0, false);
1934 	return pci_set_power_state(dev, PCI_D0);
1935 }
1936 EXPORT_SYMBOL(pci_back_from_sleep);
1937 
1938 /**
1939  * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
1940  * @dev: PCI device being suspended.
1941  *
1942  * Prepare @dev to generate wake-up events at run time and put it into a low
1943  * power state.
1944  */
1945 int pci_finish_runtime_suspend(struct pci_dev *dev)
1946 {
1947 	pci_power_t target_state = pci_target_state(dev);
1948 	int error;
1949 
1950 	if (target_state == PCI_POWER_ERROR)
1951 		return -EIO;
1952 
1953 	dev->runtime_d3cold = target_state == PCI_D3cold;
1954 
1955 	__pci_enable_wake(dev, target_state, true, pci_dev_run_wake(dev));
1956 
1957 	error = pci_set_power_state(dev, target_state);
1958 
1959 	if (error) {
1960 		__pci_enable_wake(dev, target_state, true, false);
1961 		dev->runtime_d3cold = false;
1962 	}
1963 
1964 	return error;
1965 }
1966 
1967 /**
1968  * pci_dev_run_wake - Check if device can generate run-time wake-up events.
1969  * @dev: Device to check.
1970  *
1971  * Return true if the device itself is capable of generating wake-up events
1972  * (through the platform or using the native PCIe PME) or if the device supports
1973  * PME and one of its upstream bridges can generate wake-up events.
1974  */
1975 bool pci_dev_run_wake(struct pci_dev *dev)
1976 {
1977 	struct pci_bus *bus = dev->bus;
1978 
1979 	if (device_run_wake(&dev->dev))
1980 		return true;
1981 
1982 	if (!dev->pme_support)
1983 		return false;
1984 
1985 	while (bus->parent) {
1986 		struct pci_dev *bridge = bus->self;
1987 
1988 		if (device_run_wake(&bridge->dev))
1989 			return true;
1990 
1991 		bus = bus->parent;
1992 	}
1993 
1994 	/* We have reached the root bus. */
1995 	if (bus->bridge)
1996 		return device_run_wake(bus->bridge);
1997 
1998 	return false;
1999 }
2000 EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2001 
2002 void pci_config_pm_runtime_get(struct pci_dev *pdev)
2003 {
2004 	struct device *dev = &pdev->dev;
2005 	struct device *parent = dev->parent;
2006 
2007 	if (parent)
2008 		pm_runtime_get_sync(parent);
2009 	pm_runtime_get_noresume(dev);
2010 	/*
2011 	 * pdev->current_state is set to PCI_D3cold during suspending,
2012 	 * so wait until suspending completes
2013 	 */
2014 	pm_runtime_barrier(dev);
2015 	/*
2016 	 * Only need to resume devices in D3cold, because config
2017 	 * registers are still accessible for devices suspended but
2018 	 * not in D3cold.
2019 	 */
2020 	if (pdev->current_state == PCI_D3cold)
2021 		pm_runtime_resume(dev);
2022 }
2023 
2024 void pci_config_pm_runtime_put(struct pci_dev *pdev)
2025 {
2026 	struct device *dev = &pdev->dev;
2027 	struct device *parent = dev->parent;
2028 
2029 	pm_runtime_put(dev);
2030 	if (parent)
2031 		pm_runtime_put_sync(parent);
2032 }
2033 
2034 /**
2035  * pci_pm_init - Initialize PM functions of given PCI device
2036  * @dev: PCI device to handle.
2037  */
2038 void pci_pm_init(struct pci_dev *dev)
2039 {
2040 	int pm;
2041 	u16 pmc;
2042 
2043 	pm_runtime_forbid(&dev->dev);
2044 	pm_runtime_set_active(&dev->dev);
2045 	pm_runtime_enable(&dev->dev);
2046 	device_enable_async_suspend(&dev->dev);
2047 	dev->wakeup_prepared = false;
2048 
2049 	dev->pm_cap = 0;
2050 	dev->pme_support = 0;
2051 
2052 	/* find PCI PM capability in list */
2053 	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
2054 	if (!pm)
2055 		return;
2056 	/* Check device's ability to generate PME# */
2057 	pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
2058 
2059 	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
2060 		dev_err(&dev->dev, "unsupported PM cap regs version (%u)\n",
2061 			pmc & PCI_PM_CAP_VER_MASK);
2062 		return;
2063 	}
2064 
2065 	dev->pm_cap = pm;
2066 	dev->d3_delay = PCI_PM_D3_WAIT;
2067 	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
2068 	dev->d3cold_allowed = true;
2069 
2070 	dev->d1_support = false;
2071 	dev->d2_support = false;
2072 	if (!pci_no_d1d2(dev)) {
2073 		if (pmc & PCI_PM_CAP_D1)
2074 			dev->d1_support = true;
2075 		if (pmc & PCI_PM_CAP_D2)
2076 			dev->d2_support = true;
2077 
2078 		if (dev->d1_support || dev->d2_support)
2079 			dev_printk(KERN_DEBUG, &dev->dev, "supports%s%s\n",
2080 				   dev->d1_support ? " D1" : "",
2081 				   dev->d2_support ? " D2" : "");
2082 	}
2083 
2084 	pmc &= PCI_PM_CAP_PME_MASK;
2085 	if (pmc) {
2086 		dev_printk(KERN_DEBUG, &dev->dev,
2087 			 "PME# supported from%s%s%s%s%s\n",
2088 			 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
2089 			 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
2090 			 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
2091 			 (pmc & PCI_PM_CAP_PME_D3) ? " D3hot" : "",
2092 			 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
2093 		dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT;
2094 		dev->pme_poll = true;
2095 		/*
2096 		 * Make device's PM flags reflect the wake-up capability, but
2097 		 * let the user space enable it to wake up the system as needed.
2098 		 */
2099 		device_set_wakeup_capable(&dev->dev, true);
2100 		/* Disable the PME# generation functionality */
2101 		pci_pme_active(dev, false);
2102 	}
2103 }
2104 
2105 static void pci_add_saved_cap(struct pci_dev *pci_dev,
2106 	struct pci_cap_saved_state *new_cap)
2107 {
2108 	hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
2109 }
2110 
2111 /**
2112  * _pci_add_cap_save_buffer - allocate buffer for saving given
2113  *                            capability registers
2114  * @dev: the PCI device
2115  * @cap: the capability to allocate the buffer for
2116  * @extended: Standard or Extended capability ID
2117  * @size: requested size of the buffer
2118  */
2119 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
2120 				    bool extended, unsigned int size)
2121 {
2122 	int pos;
2123 	struct pci_cap_saved_state *save_state;
2124 
2125 	if (extended)
2126 		pos = pci_find_ext_capability(dev, cap);
2127 	else
2128 		pos = pci_find_capability(dev, cap);
2129 
2130 	if (pos <= 0)
2131 		return 0;
2132 
2133 	save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
2134 	if (!save_state)
2135 		return -ENOMEM;
2136 
2137 	save_state->cap.cap_nr = cap;
2138 	save_state->cap.cap_extended = extended;
2139 	save_state->cap.size = size;
2140 	pci_add_saved_cap(dev, save_state);
2141 
2142 	return 0;
2143 }
2144 
2145 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
2146 {
2147 	return _pci_add_cap_save_buffer(dev, cap, false, size);
2148 }
2149 
2150 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
2151 {
2152 	return _pci_add_cap_save_buffer(dev, cap, true, size);
2153 }
2154 
2155 /**
2156  * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
2157  * @dev: the PCI device
2158  */
2159 void pci_allocate_cap_save_buffers(struct pci_dev *dev)
2160 {
2161 	int error;
2162 
2163 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
2164 					PCI_EXP_SAVE_REGS * sizeof(u16));
2165 	if (error)
2166 		dev_err(&dev->dev,
2167 			"unable to preallocate PCI Express save buffer\n");
2168 
2169 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
2170 	if (error)
2171 		dev_err(&dev->dev,
2172 			"unable to preallocate PCI-X save buffer\n");
2173 
2174 	pci_allocate_vc_save_buffers(dev);
2175 }
2176 
2177 void pci_free_cap_save_buffers(struct pci_dev *dev)
2178 {
2179 	struct pci_cap_saved_state *tmp;
2180 	struct hlist_node *n;
2181 
2182 	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
2183 		kfree(tmp);
2184 }
2185 
2186 /**
2187  * pci_configure_ari - enable or disable ARI forwarding
2188  * @dev: the PCI device
2189  *
2190  * If @dev and its upstream bridge both support ARI, enable ARI in the
2191  * bridge.  Otherwise, disable ARI in the bridge.
2192  */
2193 void pci_configure_ari(struct pci_dev *dev)
2194 {
2195 	u32 cap;
2196 	struct pci_dev *bridge;
2197 
2198 	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
2199 		return;
2200 
2201 	bridge = dev->bus->self;
2202 	if (!bridge)
2203 		return;
2204 
2205 	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
2206 	if (!(cap & PCI_EXP_DEVCAP2_ARI))
2207 		return;
2208 
2209 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
2210 		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
2211 					 PCI_EXP_DEVCTL2_ARI);
2212 		bridge->ari_enabled = 1;
2213 	} else {
2214 		pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
2215 					   PCI_EXP_DEVCTL2_ARI);
2216 		bridge->ari_enabled = 0;
2217 	}
2218 }
2219 
2220 static int pci_acs_enable;
2221 
2222 /**
2223  * pci_request_acs - ask for ACS to be enabled if supported
2224  */
2225 void pci_request_acs(void)
2226 {
2227 	pci_acs_enable = 1;
2228 }
2229 
2230 /**
2231  * pci_std_enable_acs - enable ACS on devices using standard ACS capabilites
2232  * @dev: the PCI device
2233  */
2234 static int pci_std_enable_acs(struct pci_dev *dev)
2235 {
2236 	int pos;
2237 	u16 cap;
2238 	u16 ctrl;
2239 
2240 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
2241 	if (!pos)
2242 		return -ENODEV;
2243 
2244 	pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
2245 	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
2246 
2247 	/* Source Validation */
2248 	ctrl |= (cap & PCI_ACS_SV);
2249 
2250 	/* P2P Request Redirect */
2251 	ctrl |= (cap & PCI_ACS_RR);
2252 
2253 	/* P2P Completion Redirect */
2254 	ctrl |= (cap & PCI_ACS_CR);
2255 
2256 	/* Upstream Forwarding */
2257 	ctrl |= (cap & PCI_ACS_UF);
2258 
2259 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
2260 
2261 	return 0;
2262 }
2263 
2264 /**
2265  * pci_enable_acs - enable ACS if hardware support it
2266  * @dev: the PCI device
2267  */
2268 void pci_enable_acs(struct pci_dev *dev)
2269 {
2270 	if (!pci_acs_enable)
2271 		return;
2272 
2273 	if (!pci_std_enable_acs(dev))
2274 		return;
2275 
2276 	pci_dev_specific_enable_acs(dev);
2277 }
2278 
2279 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
2280 {
2281 	int pos;
2282 	u16 cap, ctrl;
2283 
2284 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ACS);
2285 	if (!pos)
2286 		return false;
2287 
2288 	/*
2289 	 * Except for egress control, capabilities are either required
2290 	 * or only required if controllable.  Features missing from the
2291 	 * capability field can therefore be assumed as hard-wired enabled.
2292 	 */
2293 	pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
2294 	acs_flags &= (cap | PCI_ACS_EC);
2295 
2296 	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
2297 	return (ctrl & acs_flags) == acs_flags;
2298 }
2299 
2300 /**
2301  * pci_acs_enabled - test ACS against required flags for a given device
2302  * @pdev: device to test
2303  * @acs_flags: required PCI ACS flags
2304  *
2305  * Return true if the device supports the provided flags.  Automatically
2306  * filters out flags that are not implemented on multifunction devices.
2307  *
2308  * Note that this interface checks the effective ACS capabilities of the
2309  * device rather than the actual capabilities.  For instance, most single
2310  * function endpoints are not required to support ACS because they have no
2311  * opportunity for peer-to-peer access.  We therefore return 'true'
2312  * regardless of whether the device exposes an ACS capability.  This makes
2313  * it much easier for callers of this function to ignore the actual type
2314  * or topology of the device when testing ACS support.
2315  */
2316 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
2317 {
2318 	int ret;
2319 
2320 	ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
2321 	if (ret >= 0)
2322 		return ret > 0;
2323 
2324 	/*
2325 	 * Conventional PCI and PCI-X devices never support ACS, either
2326 	 * effectively or actually.  The shared bus topology implies that
2327 	 * any device on the bus can receive or snoop DMA.
2328 	 */
2329 	if (!pci_is_pcie(pdev))
2330 		return false;
2331 
2332 	switch (pci_pcie_type(pdev)) {
2333 	/*
2334 	 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
2335 	 * but since their primary interface is PCI/X, we conservatively
2336 	 * handle them as we would a non-PCIe device.
2337 	 */
2338 	case PCI_EXP_TYPE_PCIE_BRIDGE:
2339 	/*
2340 	 * PCIe 3.0, 6.12.1 excludes ACS on these devices.  "ACS is never
2341 	 * applicable... must never implement an ACS Extended Capability...".
2342 	 * This seems arbitrary, but we take a conservative interpretation
2343 	 * of this statement.
2344 	 */
2345 	case PCI_EXP_TYPE_PCI_BRIDGE:
2346 	case PCI_EXP_TYPE_RC_EC:
2347 		return false;
2348 	/*
2349 	 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
2350 	 * implement ACS in order to indicate their peer-to-peer capabilities,
2351 	 * regardless of whether they are single- or multi-function devices.
2352 	 */
2353 	case PCI_EXP_TYPE_DOWNSTREAM:
2354 	case PCI_EXP_TYPE_ROOT_PORT:
2355 		return pci_acs_flags_enabled(pdev, acs_flags);
2356 	/*
2357 	 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
2358 	 * implemented by the remaining PCIe types to indicate peer-to-peer
2359 	 * capabilities, but only when they are part of a multifunction
2360 	 * device.  The footnote for section 6.12 indicates the specific
2361 	 * PCIe types included here.
2362 	 */
2363 	case PCI_EXP_TYPE_ENDPOINT:
2364 	case PCI_EXP_TYPE_UPSTREAM:
2365 	case PCI_EXP_TYPE_LEG_END:
2366 	case PCI_EXP_TYPE_RC_END:
2367 		if (!pdev->multifunction)
2368 			break;
2369 
2370 		return pci_acs_flags_enabled(pdev, acs_flags);
2371 	}
2372 
2373 	/*
2374 	 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
2375 	 * to single function devices with the exception of downstream ports.
2376 	 */
2377 	return true;
2378 }
2379 
2380 /**
2381  * pci_acs_path_enable - test ACS flags from start to end in a hierarchy
2382  * @start: starting downstream device
2383  * @end: ending upstream device or NULL to search to the root bus
2384  * @acs_flags: required flags
2385  *
2386  * Walk up a device tree from start to end testing PCI ACS support.  If
2387  * any step along the way does not support the required flags, return false.
2388  */
2389 bool pci_acs_path_enabled(struct pci_dev *start,
2390 			  struct pci_dev *end, u16 acs_flags)
2391 {
2392 	struct pci_dev *pdev, *parent = start;
2393 
2394 	do {
2395 		pdev = parent;
2396 
2397 		if (!pci_acs_enabled(pdev, acs_flags))
2398 			return false;
2399 
2400 		if (pci_is_root_bus(pdev->bus))
2401 			return (end == NULL);
2402 
2403 		parent = pdev->bus->self;
2404 	} while (pdev != end);
2405 
2406 	return true;
2407 }
2408 
2409 /**
2410  * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge
2411  * @dev: the PCI device
2412  * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD)
2413  *
2414  * Perform INTx swizzling for a device behind one level of bridge.  This is
2415  * required by section 9.1 of the PCI-to-PCI bridge specification for devices
2416  * behind bridges on add-in cards.  For devices with ARI enabled, the slot
2417  * number is always 0 (see the Implementation Note in section 2.2.8.1 of
2418  * the PCI Express Base Specification, Revision 2.1)
2419  */
2420 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin)
2421 {
2422 	int slot;
2423 
2424 	if (pci_ari_enabled(dev->bus))
2425 		slot = 0;
2426 	else
2427 		slot = PCI_SLOT(dev->devfn);
2428 
2429 	return (((pin - 1) + slot) % 4) + 1;
2430 }
2431 
2432 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge)
2433 {
2434 	u8 pin;
2435 
2436 	pin = dev->pin;
2437 	if (!pin)
2438 		return -1;
2439 
2440 	while (!pci_is_root_bus(dev->bus)) {
2441 		pin = pci_swizzle_interrupt_pin(dev, pin);
2442 		dev = dev->bus->self;
2443 	}
2444 	*bridge = dev;
2445 	return pin;
2446 }
2447 
2448 /**
2449  * pci_common_swizzle - swizzle INTx all the way to root bridge
2450  * @dev: the PCI device
2451  * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD)
2452  *
2453  * Perform INTx swizzling for a device.  This traverses through all PCI-to-PCI
2454  * bridges all the way up to a PCI root bus.
2455  */
2456 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp)
2457 {
2458 	u8 pin = *pinp;
2459 
2460 	while (!pci_is_root_bus(dev->bus)) {
2461 		pin = pci_swizzle_interrupt_pin(dev, pin);
2462 		dev = dev->bus->self;
2463 	}
2464 	*pinp = pin;
2465 	return PCI_SLOT(dev->devfn);
2466 }
2467 
2468 /**
2469  *	pci_release_region - Release a PCI bar
2470  *	@pdev: PCI device whose resources were previously reserved by pci_request_region
2471  *	@bar: BAR to release
2472  *
2473  *	Releases the PCI I/O and memory resources previously reserved by a
2474  *	successful call to pci_request_region.  Call this function only
2475  *	after all use of the PCI regions has ceased.
2476  */
2477 void pci_release_region(struct pci_dev *pdev, int bar)
2478 {
2479 	struct pci_devres *dr;
2480 
2481 	if (pci_resource_len(pdev, bar) == 0)
2482 		return;
2483 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
2484 		release_region(pci_resource_start(pdev, bar),
2485 				pci_resource_len(pdev, bar));
2486 	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
2487 		release_mem_region(pci_resource_start(pdev, bar),
2488 				pci_resource_len(pdev, bar));
2489 
2490 	dr = find_pci_dr(pdev);
2491 	if (dr)
2492 		dr->region_mask &= ~(1 << bar);
2493 }
2494 EXPORT_SYMBOL(pci_release_region);
2495 
2496 /**
2497  *	__pci_request_region - Reserved PCI I/O and memory resource
2498  *	@pdev: PCI device whose resources are to be reserved
2499  *	@bar: BAR to be reserved
2500  *	@res_name: Name to be associated with resource.
2501  *	@exclusive: whether the region access is exclusive or not
2502  *
2503  *	Mark the PCI region associated with PCI device @pdev BR @bar as
2504  *	being reserved by owner @res_name.  Do not access any
2505  *	address inside the PCI regions unless this call returns
2506  *	successfully.
2507  *
2508  *	If @exclusive is set, then the region is marked so that userspace
2509  *	is explicitly not allowed to map the resource via /dev/mem or
2510  *	sysfs MMIO access.
2511  *
2512  *	Returns 0 on success, or %EBUSY on error.  A warning
2513  *	message is also printed on failure.
2514  */
2515 static int __pci_request_region(struct pci_dev *pdev, int bar,
2516 				const char *res_name, int exclusive)
2517 {
2518 	struct pci_devres *dr;
2519 
2520 	if (pci_resource_len(pdev, bar) == 0)
2521 		return 0;
2522 
2523 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
2524 		if (!request_region(pci_resource_start(pdev, bar),
2525 			    pci_resource_len(pdev, bar), res_name))
2526 			goto err_out;
2527 	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
2528 		if (!__request_mem_region(pci_resource_start(pdev, bar),
2529 					pci_resource_len(pdev, bar), res_name,
2530 					exclusive))
2531 			goto err_out;
2532 	}
2533 
2534 	dr = find_pci_dr(pdev);
2535 	if (dr)
2536 		dr->region_mask |= 1 << bar;
2537 
2538 	return 0;
2539 
2540 err_out:
2541 	dev_warn(&pdev->dev, "BAR %d: can't reserve %pR\n", bar,
2542 		 &pdev->resource[bar]);
2543 	return -EBUSY;
2544 }
2545 
2546 /**
2547  *	pci_request_region - Reserve PCI I/O and memory resource
2548  *	@pdev: PCI device whose resources are to be reserved
2549  *	@bar: BAR to be reserved
2550  *	@res_name: Name to be associated with resource
2551  *
2552  *	Mark the PCI region associated with PCI device @pdev BAR @bar as
2553  *	being reserved by owner @res_name.  Do not access any
2554  *	address inside the PCI regions unless this call returns
2555  *	successfully.
2556  *
2557  *	Returns 0 on success, or %EBUSY on error.  A warning
2558  *	message is also printed on failure.
2559  */
2560 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
2561 {
2562 	return __pci_request_region(pdev, bar, res_name, 0);
2563 }
2564 EXPORT_SYMBOL(pci_request_region);
2565 
2566 /**
2567  *	pci_request_region_exclusive - Reserved PCI I/O and memory resource
2568  *	@pdev: PCI device whose resources are to be reserved
2569  *	@bar: BAR to be reserved
2570  *	@res_name: Name to be associated with resource.
2571  *
2572  *	Mark the PCI region associated with PCI device @pdev BR @bar as
2573  *	being reserved by owner @res_name.  Do not access any
2574  *	address inside the PCI regions unless this call returns
2575  *	successfully.
2576  *
2577  *	Returns 0 on success, or %EBUSY on error.  A warning
2578  *	message is also printed on failure.
2579  *
2580  *	The key difference that _exclusive makes it that userspace is
2581  *	explicitly not allowed to map the resource via /dev/mem or
2582  *	sysfs.
2583  */
2584 int pci_request_region_exclusive(struct pci_dev *pdev, int bar,
2585 				 const char *res_name)
2586 {
2587 	return __pci_request_region(pdev, bar, res_name, IORESOURCE_EXCLUSIVE);
2588 }
2589 EXPORT_SYMBOL(pci_request_region_exclusive);
2590 
2591 /**
2592  * pci_release_selected_regions - Release selected PCI I/O and memory resources
2593  * @pdev: PCI device whose resources were previously reserved
2594  * @bars: Bitmask of BARs to be released
2595  *
2596  * Release selected PCI I/O and memory resources previously reserved.
2597  * Call this function only after all use of the PCI regions has ceased.
2598  */
2599 void pci_release_selected_regions(struct pci_dev *pdev, int bars)
2600 {
2601 	int i;
2602 
2603 	for (i = 0; i < 6; i++)
2604 		if (bars & (1 << i))
2605 			pci_release_region(pdev, i);
2606 }
2607 EXPORT_SYMBOL(pci_release_selected_regions);
2608 
2609 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
2610 					  const char *res_name, int excl)
2611 {
2612 	int i;
2613 
2614 	for (i = 0; i < 6; i++)
2615 		if (bars & (1 << i))
2616 			if (__pci_request_region(pdev, i, res_name, excl))
2617 				goto err_out;
2618 	return 0;
2619 
2620 err_out:
2621 	while (--i >= 0)
2622 		if (bars & (1 << i))
2623 			pci_release_region(pdev, i);
2624 
2625 	return -EBUSY;
2626 }
2627 
2628 
2629 /**
2630  * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
2631  * @pdev: PCI device whose resources are to be reserved
2632  * @bars: Bitmask of BARs to be requested
2633  * @res_name: Name to be associated with resource
2634  */
2635 int pci_request_selected_regions(struct pci_dev *pdev, int bars,
2636 				 const char *res_name)
2637 {
2638 	return __pci_request_selected_regions(pdev, bars, res_name, 0);
2639 }
2640 EXPORT_SYMBOL(pci_request_selected_regions);
2641 
2642 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
2643 					   const char *res_name)
2644 {
2645 	return __pci_request_selected_regions(pdev, bars, res_name,
2646 			IORESOURCE_EXCLUSIVE);
2647 }
2648 EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
2649 
2650 /**
2651  *	pci_release_regions - Release reserved PCI I/O and memory resources
2652  *	@pdev: PCI device whose resources were previously reserved by pci_request_regions
2653  *
2654  *	Releases all PCI I/O and memory resources previously reserved by a
2655  *	successful call to pci_request_regions.  Call this function only
2656  *	after all use of the PCI regions has ceased.
2657  */
2658 
2659 void pci_release_regions(struct pci_dev *pdev)
2660 {
2661 	pci_release_selected_regions(pdev, (1 << 6) - 1);
2662 }
2663 EXPORT_SYMBOL(pci_release_regions);
2664 
2665 /**
2666  *	pci_request_regions - Reserved PCI I/O and memory resources
2667  *	@pdev: PCI device whose resources are to be reserved
2668  *	@res_name: Name to be associated with resource.
2669  *
2670  *	Mark all PCI regions associated with PCI device @pdev as
2671  *	being reserved by owner @res_name.  Do not access any
2672  *	address inside the PCI regions unless this call returns
2673  *	successfully.
2674  *
2675  *	Returns 0 on success, or %EBUSY on error.  A warning
2676  *	message is also printed on failure.
2677  */
2678 int pci_request_regions(struct pci_dev *pdev, const char *res_name)
2679 {
2680 	return pci_request_selected_regions(pdev, ((1 << 6) - 1), res_name);
2681 }
2682 EXPORT_SYMBOL(pci_request_regions);
2683 
2684 /**
2685  *	pci_request_regions_exclusive - Reserved PCI I/O and memory resources
2686  *	@pdev: PCI device whose resources are to be reserved
2687  *	@res_name: Name to be associated with resource.
2688  *
2689  *	Mark all PCI regions associated with PCI device @pdev as
2690  *	being reserved by owner @res_name.  Do not access any
2691  *	address inside the PCI regions unless this call returns
2692  *	successfully.
2693  *
2694  *	pci_request_regions_exclusive() will mark the region so that
2695  *	/dev/mem and the sysfs MMIO access will not be allowed.
2696  *
2697  *	Returns 0 on success, or %EBUSY on error.  A warning
2698  *	message is also printed on failure.
2699  */
2700 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
2701 {
2702 	return pci_request_selected_regions_exclusive(pdev,
2703 					((1 << 6) - 1), res_name);
2704 }
2705 EXPORT_SYMBOL(pci_request_regions_exclusive);
2706 
2707 static void __pci_set_master(struct pci_dev *dev, bool enable)
2708 {
2709 	u16 old_cmd, cmd;
2710 
2711 	pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
2712 	if (enable)
2713 		cmd = old_cmd | PCI_COMMAND_MASTER;
2714 	else
2715 		cmd = old_cmd & ~PCI_COMMAND_MASTER;
2716 	if (cmd != old_cmd) {
2717 		dev_dbg(&dev->dev, "%s bus mastering\n",
2718 			enable ? "enabling" : "disabling");
2719 		pci_write_config_word(dev, PCI_COMMAND, cmd);
2720 	}
2721 	dev->is_busmaster = enable;
2722 }
2723 
2724 /**
2725  * pcibios_setup - process "pci=" kernel boot arguments
2726  * @str: string used to pass in "pci=" kernel boot arguments
2727  *
2728  * Process kernel boot arguments.  This is the default implementation.
2729  * Architecture specific implementations can override this as necessary.
2730  */
2731 char * __weak __init pcibios_setup(char *str)
2732 {
2733 	return str;
2734 }
2735 
2736 /**
2737  * pcibios_set_master - enable PCI bus-mastering for device dev
2738  * @dev: the PCI device to enable
2739  *
2740  * Enables PCI bus-mastering for the device.  This is the default
2741  * implementation.  Architecture specific implementations can override
2742  * this if necessary.
2743  */
2744 void __weak pcibios_set_master(struct pci_dev *dev)
2745 {
2746 	u8 lat;
2747 
2748 	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
2749 	if (pci_is_pcie(dev))
2750 		return;
2751 
2752 	pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
2753 	if (lat < 16)
2754 		lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
2755 	else if (lat > pcibios_max_latency)
2756 		lat = pcibios_max_latency;
2757 	else
2758 		return;
2759 
2760 	pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
2761 }
2762 
2763 /**
2764  * pci_set_master - enables bus-mastering for device dev
2765  * @dev: the PCI device to enable
2766  *
2767  * Enables bus-mastering on the device and calls pcibios_set_master()
2768  * to do the needed arch specific settings.
2769  */
2770 void pci_set_master(struct pci_dev *dev)
2771 {
2772 	__pci_set_master(dev, true);
2773 	pcibios_set_master(dev);
2774 }
2775 EXPORT_SYMBOL(pci_set_master);
2776 
2777 /**
2778  * pci_clear_master - disables bus-mastering for device dev
2779  * @dev: the PCI device to disable
2780  */
2781 void pci_clear_master(struct pci_dev *dev)
2782 {
2783 	__pci_set_master(dev, false);
2784 }
2785 EXPORT_SYMBOL(pci_clear_master);
2786 
2787 /**
2788  * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
2789  * @dev: the PCI device for which MWI is to be enabled
2790  *
2791  * Helper function for pci_set_mwi.
2792  * Originally copied from drivers/net/acenic.c.
2793  * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
2794  *
2795  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
2796  */
2797 int pci_set_cacheline_size(struct pci_dev *dev)
2798 {
2799 	u8 cacheline_size;
2800 
2801 	if (!pci_cache_line_size)
2802 		return -EINVAL;
2803 
2804 	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
2805 	   equal to or multiple of the right value. */
2806 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
2807 	if (cacheline_size >= pci_cache_line_size &&
2808 	    (cacheline_size % pci_cache_line_size) == 0)
2809 		return 0;
2810 
2811 	/* Write the correct value. */
2812 	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
2813 	/* Read it back. */
2814 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
2815 	if (cacheline_size == pci_cache_line_size)
2816 		return 0;
2817 
2818 	dev_printk(KERN_DEBUG, &dev->dev, "cache line size of %d is not supported\n",
2819 		   pci_cache_line_size << 2);
2820 
2821 	return -EINVAL;
2822 }
2823 EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
2824 
2825 /**
2826  * pci_set_mwi - enables memory-write-invalidate PCI transaction
2827  * @dev: the PCI device for which MWI is enabled
2828  *
2829  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
2830  *
2831  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
2832  */
2833 int pci_set_mwi(struct pci_dev *dev)
2834 {
2835 #ifdef PCI_DISABLE_MWI
2836 	return 0;
2837 #else
2838 	int rc;
2839 	u16 cmd;
2840 
2841 	rc = pci_set_cacheline_size(dev);
2842 	if (rc)
2843 		return rc;
2844 
2845 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
2846 	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
2847 		dev_dbg(&dev->dev, "enabling Mem-Wr-Inval\n");
2848 		cmd |= PCI_COMMAND_INVALIDATE;
2849 		pci_write_config_word(dev, PCI_COMMAND, cmd);
2850 	}
2851 	return 0;
2852 #endif
2853 }
2854 EXPORT_SYMBOL(pci_set_mwi);
2855 
2856 /**
2857  * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
2858  * @dev: the PCI device for which MWI is enabled
2859  *
2860  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
2861  * Callers are not required to check the return value.
2862  *
2863  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
2864  */
2865 int pci_try_set_mwi(struct pci_dev *dev)
2866 {
2867 #ifdef PCI_DISABLE_MWI
2868 	return 0;
2869 #else
2870 	return pci_set_mwi(dev);
2871 #endif
2872 }
2873 EXPORT_SYMBOL(pci_try_set_mwi);
2874 
2875 /**
2876  * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
2877  * @dev: the PCI device to disable
2878  *
2879  * Disables PCI Memory-Write-Invalidate transaction on the device
2880  */
2881 void pci_clear_mwi(struct pci_dev *dev)
2882 {
2883 #ifndef PCI_DISABLE_MWI
2884 	u16 cmd;
2885 
2886 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
2887 	if (cmd & PCI_COMMAND_INVALIDATE) {
2888 		cmd &= ~PCI_COMMAND_INVALIDATE;
2889 		pci_write_config_word(dev, PCI_COMMAND, cmd);
2890 	}
2891 #endif
2892 }
2893 EXPORT_SYMBOL(pci_clear_mwi);
2894 
2895 /**
2896  * pci_intx - enables/disables PCI INTx for device dev
2897  * @pdev: the PCI device to operate on
2898  * @enable: boolean: whether to enable or disable PCI INTx
2899  *
2900  * Enables/disables PCI INTx for device dev
2901  */
2902 void pci_intx(struct pci_dev *pdev, int enable)
2903 {
2904 	u16 pci_command, new;
2905 
2906 	pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
2907 
2908 	if (enable)
2909 		new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
2910 	else
2911 		new = pci_command | PCI_COMMAND_INTX_DISABLE;
2912 
2913 	if (new != pci_command) {
2914 		struct pci_devres *dr;
2915 
2916 		pci_write_config_word(pdev, PCI_COMMAND, new);
2917 
2918 		dr = find_pci_dr(pdev);
2919 		if (dr && !dr->restore_intx) {
2920 			dr->restore_intx = 1;
2921 			dr->orig_intx = !enable;
2922 		}
2923 	}
2924 }
2925 EXPORT_SYMBOL_GPL(pci_intx);
2926 
2927 /**
2928  * pci_intx_mask_supported - probe for INTx masking support
2929  * @dev: the PCI device to operate on
2930  *
2931  * Check if the device dev support INTx masking via the config space
2932  * command word.
2933  */
2934 bool pci_intx_mask_supported(struct pci_dev *dev)
2935 {
2936 	bool mask_supported = false;
2937 	u16 orig, new;
2938 
2939 	if (dev->broken_intx_masking)
2940 		return false;
2941 
2942 	pci_cfg_access_lock(dev);
2943 
2944 	pci_read_config_word(dev, PCI_COMMAND, &orig);
2945 	pci_write_config_word(dev, PCI_COMMAND,
2946 			      orig ^ PCI_COMMAND_INTX_DISABLE);
2947 	pci_read_config_word(dev, PCI_COMMAND, &new);
2948 
2949 	/*
2950 	 * There's no way to protect against hardware bugs or detect them
2951 	 * reliably, but as long as we know what the value should be, let's
2952 	 * go ahead and check it.
2953 	 */
2954 	if ((new ^ orig) & ~PCI_COMMAND_INTX_DISABLE) {
2955 		dev_err(&dev->dev, "Command register changed from 0x%x to 0x%x: driver or hardware bug?\n",
2956 			orig, new);
2957 	} else if ((new ^ orig) & PCI_COMMAND_INTX_DISABLE) {
2958 		mask_supported = true;
2959 		pci_write_config_word(dev, PCI_COMMAND, orig);
2960 	}
2961 
2962 	pci_cfg_access_unlock(dev);
2963 	return mask_supported;
2964 }
2965 EXPORT_SYMBOL_GPL(pci_intx_mask_supported);
2966 
2967 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask)
2968 {
2969 	struct pci_bus *bus = dev->bus;
2970 	bool mask_updated = true;
2971 	u32 cmd_status_dword;
2972 	u16 origcmd, newcmd;
2973 	unsigned long flags;
2974 	bool irq_pending;
2975 
2976 	/*
2977 	 * We do a single dword read to retrieve both command and status.
2978 	 * Document assumptions that make this possible.
2979 	 */
2980 	BUILD_BUG_ON(PCI_COMMAND % 4);
2981 	BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS);
2982 
2983 	raw_spin_lock_irqsave(&pci_lock, flags);
2984 
2985 	bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword);
2986 
2987 	irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT;
2988 
2989 	/*
2990 	 * Check interrupt status register to see whether our device
2991 	 * triggered the interrupt (when masking) or the next IRQ is
2992 	 * already pending (when unmasking).
2993 	 */
2994 	if (mask != irq_pending) {
2995 		mask_updated = false;
2996 		goto done;
2997 	}
2998 
2999 	origcmd = cmd_status_dword;
3000 	newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE;
3001 	if (mask)
3002 		newcmd |= PCI_COMMAND_INTX_DISABLE;
3003 	if (newcmd != origcmd)
3004 		bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd);
3005 
3006 done:
3007 	raw_spin_unlock_irqrestore(&pci_lock, flags);
3008 
3009 	return mask_updated;
3010 }
3011 
3012 /**
3013  * pci_check_and_mask_intx - mask INTx on pending interrupt
3014  * @dev: the PCI device to operate on
3015  *
3016  * Check if the device dev has its INTx line asserted, mask it and
3017  * return true in that case. False is returned if not interrupt was
3018  * pending.
3019  */
3020 bool pci_check_and_mask_intx(struct pci_dev *dev)
3021 {
3022 	return pci_check_and_set_intx_mask(dev, true);
3023 }
3024 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx);
3025 
3026 /**
3027  * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending
3028  * @dev: the PCI device to operate on
3029  *
3030  * Check if the device dev has its INTx line asserted, unmask it if not
3031  * and return true. False is returned and the mask remains active if
3032  * there was still an interrupt pending.
3033  */
3034 bool pci_check_and_unmask_intx(struct pci_dev *dev)
3035 {
3036 	return pci_check_and_set_intx_mask(dev, false);
3037 }
3038 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx);
3039 
3040 /**
3041  * pci_msi_off - disables any MSI or MSI-X capabilities
3042  * @dev: the PCI device to operate on
3043  *
3044  * If you want to use MSI, see pci_enable_msi() and friends.
3045  * This is a lower-level primitive that allows us to disable
3046  * MSI operation at the device level.
3047  */
3048 void pci_msi_off(struct pci_dev *dev)
3049 {
3050 	int pos;
3051 	u16 control;
3052 
3053 	/*
3054 	 * This looks like it could go in msi.c, but we need it even when
3055 	 * CONFIG_PCI_MSI=n.  For the same reason, we can't use
3056 	 * dev->msi_cap or dev->msix_cap here.
3057 	 */
3058 	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
3059 	if (pos) {
3060 		pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control);
3061 		control &= ~PCI_MSI_FLAGS_ENABLE;
3062 		pci_write_config_word(dev, pos + PCI_MSI_FLAGS, control);
3063 	}
3064 	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
3065 	if (pos) {
3066 		pci_read_config_word(dev, pos + PCI_MSIX_FLAGS, &control);
3067 		control &= ~PCI_MSIX_FLAGS_ENABLE;
3068 		pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control);
3069 	}
3070 }
3071 EXPORT_SYMBOL_GPL(pci_msi_off);
3072 
3073 int pci_set_dma_max_seg_size(struct pci_dev *dev, unsigned int size)
3074 {
3075 	return dma_set_max_seg_size(&dev->dev, size);
3076 }
3077 EXPORT_SYMBOL(pci_set_dma_max_seg_size);
3078 
3079 int pci_set_dma_seg_boundary(struct pci_dev *dev, unsigned long mask)
3080 {
3081 	return dma_set_seg_boundary(&dev->dev, mask);
3082 }
3083 EXPORT_SYMBOL(pci_set_dma_seg_boundary);
3084 
3085 /**
3086  * pci_wait_for_pending_transaction - waits for pending transaction
3087  * @dev: the PCI device to operate on
3088  *
3089  * Return 0 if transaction is pending 1 otherwise.
3090  */
3091 int pci_wait_for_pending_transaction(struct pci_dev *dev)
3092 {
3093 	if (!pci_is_pcie(dev))
3094 		return 1;
3095 
3096 	return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
3097 				    PCI_EXP_DEVSTA_TRPND);
3098 }
3099 EXPORT_SYMBOL(pci_wait_for_pending_transaction);
3100 
3101 static int pcie_flr(struct pci_dev *dev, int probe)
3102 {
3103 	u32 cap;
3104 
3105 	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
3106 	if (!(cap & PCI_EXP_DEVCAP_FLR))
3107 		return -ENOTTY;
3108 
3109 	if (probe)
3110 		return 0;
3111 
3112 	if (!pci_wait_for_pending_transaction(dev))
3113 		dev_err(&dev->dev, "transaction is not cleared; proceeding with reset anyway\n");
3114 
3115 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
3116 
3117 	msleep(100);
3118 
3119 	return 0;
3120 }
3121 
3122 static int pci_af_flr(struct pci_dev *dev, int probe)
3123 {
3124 	int pos;
3125 	u8 cap;
3126 
3127 	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
3128 	if (!pos)
3129 		return -ENOTTY;
3130 
3131 	pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
3132 	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
3133 		return -ENOTTY;
3134 
3135 	if (probe)
3136 		return 0;
3137 
3138 	/* Wait for Transaction Pending bit clean */
3139 	if (pci_wait_for_pending(dev, pos + PCI_AF_STATUS, PCI_AF_STATUS_TP))
3140 		goto clear;
3141 
3142 	dev_err(&dev->dev, "transaction is not cleared; proceeding with reset anyway\n");
3143 
3144 clear:
3145 	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
3146 	msleep(100);
3147 
3148 	return 0;
3149 }
3150 
3151 /**
3152  * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
3153  * @dev: Device to reset.
3154  * @probe: If set, only check if the device can be reset this way.
3155  *
3156  * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
3157  * unset, it will be reinitialized internally when going from PCI_D3hot to
3158  * PCI_D0.  If that's the case and the device is not in a low-power state
3159  * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
3160  *
3161  * NOTE: This causes the caller to sleep for twice the device power transition
3162  * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
3163  * by default (i.e. unless the @dev's d3_delay field has a different value).
3164  * Moreover, only devices in D0 can be reset by this function.
3165  */
3166 static int pci_pm_reset(struct pci_dev *dev, int probe)
3167 {
3168 	u16 csr;
3169 
3170 	if (!dev->pm_cap)
3171 		return -ENOTTY;
3172 
3173 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
3174 	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
3175 		return -ENOTTY;
3176 
3177 	if (probe)
3178 		return 0;
3179 
3180 	if (dev->current_state != PCI_D0)
3181 		return -EINVAL;
3182 
3183 	csr &= ~PCI_PM_CTRL_STATE_MASK;
3184 	csr |= PCI_D3hot;
3185 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
3186 	pci_dev_d3_sleep(dev);
3187 
3188 	csr &= ~PCI_PM_CTRL_STATE_MASK;
3189 	csr |= PCI_D0;
3190 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
3191 	pci_dev_d3_sleep(dev);
3192 
3193 	return 0;
3194 }
3195 
3196 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
3197 {
3198 	u16 ctrl;
3199 
3200 	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
3201 	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
3202 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
3203 	/*
3204 	 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms.  Double
3205 	 * this to 2ms to ensure that we meet the minimum requirement.
3206 	 */
3207 	msleep(2);
3208 
3209 	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
3210 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
3211 
3212 	/*
3213 	 * Trhfa for conventional PCI is 2^25 clock cycles.
3214 	 * Assuming a minimum 33MHz clock this results in a 1s
3215 	 * delay before we can consider subordinate devices to
3216 	 * be re-initialized.  PCIe has some ways to shorten this,
3217 	 * but we don't make use of them yet.
3218 	 */
3219 	ssleep(1);
3220 }
3221 
3222 /**
3223  * pci_reset_bridge_secondary_bus - Reset the secondary bus on a PCI bridge.
3224  * @dev: Bridge device
3225  *
3226  * Use the bridge control register to assert reset on the secondary bus.
3227  * Devices on the secondary bus are left in power-on state.
3228  */
3229 void pci_reset_bridge_secondary_bus(struct pci_dev *dev)
3230 {
3231 	pcibios_reset_secondary_bus(dev);
3232 }
3233 EXPORT_SYMBOL_GPL(pci_reset_bridge_secondary_bus);
3234 
3235 static int pci_parent_bus_reset(struct pci_dev *dev, int probe)
3236 {
3237 	struct pci_dev *pdev;
3238 
3239 	if (pci_is_root_bus(dev->bus) || dev->subordinate || !dev->bus->self)
3240 		return -ENOTTY;
3241 
3242 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
3243 		if (pdev != dev)
3244 			return -ENOTTY;
3245 
3246 	if (probe)
3247 		return 0;
3248 
3249 	pci_reset_bridge_secondary_bus(dev->bus->self);
3250 
3251 	return 0;
3252 }
3253 
3254 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe)
3255 {
3256 	int rc = -ENOTTY;
3257 
3258 	if (!hotplug || !try_module_get(hotplug->ops->owner))
3259 		return rc;
3260 
3261 	if (hotplug->ops->reset_slot)
3262 		rc = hotplug->ops->reset_slot(hotplug, probe);
3263 
3264 	module_put(hotplug->ops->owner);
3265 
3266 	return rc;
3267 }
3268 
3269 static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe)
3270 {
3271 	struct pci_dev *pdev;
3272 
3273 	if (dev->subordinate || !dev->slot)
3274 		return -ENOTTY;
3275 
3276 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
3277 		if (pdev != dev && pdev->slot == dev->slot)
3278 			return -ENOTTY;
3279 
3280 	return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
3281 }
3282 
3283 static int __pci_dev_reset(struct pci_dev *dev, int probe)
3284 {
3285 	int rc;
3286 
3287 	might_sleep();
3288 
3289 	rc = pci_dev_specific_reset(dev, probe);
3290 	if (rc != -ENOTTY)
3291 		goto done;
3292 
3293 	rc = pcie_flr(dev, probe);
3294 	if (rc != -ENOTTY)
3295 		goto done;
3296 
3297 	rc = pci_af_flr(dev, probe);
3298 	if (rc != -ENOTTY)
3299 		goto done;
3300 
3301 	rc = pci_pm_reset(dev, probe);
3302 	if (rc != -ENOTTY)
3303 		goto done;
3304 
3305 	rc = pci_dev_reset_slot_function(dev, probe);
3306 	if (rc != -ENOTTY)
3307 		goto done;
3308 
3309 	rc = pci_parent_bus_reset(dev, probe);
3310 done:
3311 	return rc;
3312 }
3313 
3314 static void pci_dev_lock(struct pci_dev *dev)
3315 {
3316 	pci_cfg_access_lock(dev);
3317 	/* block PM suspend, driver probe, etc. */
3318 	device_lock(&dev->dev);
3319 }
3320 
3321 /* Return 1 on successful lock, 0 on contention */
3322 static int pci_dev_trylock(struct pci_dev *dev)
3323 {
3324 	if (pci_cfg_access_trylock(dev)) {
3325 		if (device_trylock(&dev->dev))
3326 			return 1;
3327 		pci_cfg_access_unlock(dev);
3328 	}
3329 
3330 	return 0;
3331 }
3332 
3333 static void pci_dev_unlock(struct pci_dev *dev)
3334 {
3335 	device_unlock(&dev->dev);
3336 	pci_cfg_access_unlock(dev);
3337 }
3338 
3339 /**
3340  * pci_reset_notify - notify device driver of reset
3341  * @dev: device to be notified of reset
3342  * @prepare: 'true' if device is about to be reset; 'false' if reset attempt
3343  *           completed
3344  *
3345  * Must be called prior to device access being disabled and after device
3346  * access is restored.
3347  */
3348 static void pci_reset_notify(struct pci_dev *dev, bool prepare)
3349 {
3350 	const struct pci_error_handlers *err_handler =
3351 			dev->driver ? dev->driver->err_handler : NULL;
3352 	if (err_handler && err_handler->reset_notify)
3353 		err_handler->reset_notify(dev, prepare);
3354 }
3355 
3356 static void pci_dev_save_and_disable(struct pci_dev *dev)
3357 {
3358 	pci_reset_notify(dev, true);
3359 
3360 	/*
3361 	 * Wake-up device prior to save.  PM registers default to D0 after
3362 	 * reset and a simple register restore doesn't reliably return
3363 	 * to a non-D0 state anyway.
3364 	 */
3365 	pci_set_power_state(dev, PCI_D0);
3366 
3367 	pci_save_state(dev);
3368 	/*
3369 	 * Disable the device by clearing the Command register, except for
3370 	 * INTx-disable which is set.  This not only disables MMIO and I/O port
3371 	 * BARs, but also prevents the device from being Bus Master, preventing
3372 	 * DMA from the device including MSI/MSI-X interrupts.  For PCI 2.3
3373 	 * compliant devices, INTx-disable prevents legacy interrupts.
3374 	 */
3375 	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
3376 }
3377 
3378 static void pci_dev_restore(struct pci_dev *dev)
3379 {
3380 	pci_restore_state(dev);
3381 	pci_reset_notify(dev, false);
3382 }
3383 
3384 static int pci_dev_reset(struct pci_dev *dev, int probe)
3385 {
3386 	int rc;
3387 
3388 	if (!probe)
3389 		pci_dev_lock(dev);
3390 
3391 	rc = __pci_dev_reset(dev, probe);
3392 
3393 	if (!probe)
3394 		pci_dev_unlock(dev);
3395 
3396 	return rc;
3397 }
3398 
3399 /**
3400  * __pci_reset_function - reset a PCI device function
3401  * @dev: PCI device to reset
3402  *
3403  * Some devices allow an individual function to be reset without affecting
3404  * other functions in the same device.  The PCI device must be responsive
3405  * to PCI config space in order to use this function.
3406  *
3407  * The device function is presumed to be unused when this function is called.
3408  * Resetting the device will make the contents of PCI configuration space
3409  * random, so any caller of this must be prepared to reinitialise the
3410  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
3411  * etc.
3412  *
3413  * Returns 0 if the device function was successfully reset or negative if the
3414  * device doesn't support resetting a single function.
3415  */
3416 int __pci_reset_function(struct pci_dev *dev)
3417 {
3418 	return pci_dev_reset(dev, 0);
3419 }
3420 EXPORT_SYMBOL_GPL(__pci_reset_function);
3421 
3422 /**
3423  * __pci_reset_function_locked - reset a PCI device function while holding
3424  * the @dev mutex lock.
3425  * @dev: PCI device to reset
3426  *
3427  * Some devices allow an individual function to be reset without affecting
3428  * other functions in the same device.  The PCI device must be responsive
3429  * to PCI config space in order to use this function.
3430  *
3431  * The device function is presumed to be unused and the caller is holding
3432  * the device mutex lock when this function is called.
3433  * Resetting the device will make the contents of PCI configuration space
3434  * random, so any caller of this must be prepared to reinitialise the
3435  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
3436  * etc.
3437  *
3438  * Returns 0 if the device function was successfully reset or negative if the
3439  * device doesn't support resetting a single function.
3440  */
3441 int __pci_reset_function_locked(struct pci_dev *dev)
3442 {
3443 	return __pci_dev_reset(dev, 0);
3444 }
3445 EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
3446 
3447 /**
3448  * pci_probe_reset_function - check whether the device can be safely reset
3449  * @dev: PCI device to reset
3450  *
3451  * Some devices allow an individual function to be reset without affecting
3452  * other functions in the same device.  The PCI device must be responsive
3453  * to PCI config space in order to use this function.
3454  *
3455  * Returns 0 if the device function can be reset or negative if the
3456  * device doesn't support resetting a single function.
3457  */
3458 int pci_probe_reset_function(struct pci_dev *dev)
3459 {
3460 	return pci_dev_reset(dev, 1);
3461 }
3462 
3463 /**
3464  * pci_reset_function - quiesce and reset a PCI device function
3465  * @dev: PCI device to reset
3466  *
3467  * Some devices allow an individual function to be reset without affecting
3468  * other functions in the same device.  The PCI device must be responsive
3469  * to PCI config space in order to use this function.
3470  *
3471  * This function does not just reset the PCI portion of a device, but
3472  * clears all the state associated with the device.  This function differs
3473  * from __pci_reset_function in that it saves and restores device state
3474  * over the reset.
3475  *
3476  * Returns 0 if the device function was successfully reset or negative if the
3477  * device doesn't support resetting a single function.
3478  */
3479 int pci_reset_function(struct pci_dev *dev)
3480 {
3481 	int rc;
3482 
3483 	rc = pci_dev_reset(dev, 1);
3484 	if (rc)
3485 		return rc;
3486 
3487 	pci_dev_save_and_disable(dev);
3488 
3489 	rc = pci_dev_reset(dev, 0);
3490 
3491 	pci_dev_restore(dev);
3492 
3493 	return rc;
3494 }
3495 EXPORT_SYMBOL_GPL(pci_reset_function);
3496 
3497 /**
3498  * pci_try_reset_function - quiesce and reset a PCI device function
3499  * @dev: PCI device to reset
3500  *
3501  * Same as above, except return -EAGAIN if unable to lock device.
3502  */
3503 int pci_try_reset_function(struct pci_dev *dev)
3504 {
3505 	int rc;
3506 
3507 	rc = pci_dev_reset(dev, 1);
3508 	if (rc)
3509 		return rc;
3510 
3511 	pci_dev_save_and_disable(dev);
3512 
3513 	if (pci_dev_trylock(dev)) {
3514 		rc = __pci_dev_reset(dev, 0);
3515 		pci_dev_unlock(dev);
3516 	} else
3517 		rc = -EAGAIN;
3518 
3519 	pci_dev_restore(dev);
3520 
3521 	return rc;
3522 }
3523 EXPORT_SYMBOL_GPL(pci_try_reset_function);
3524 
3525 /* Lock devices from the top of the tree down */
3526 static void pci_bus_lock(struct pci_bus *bus)
3527 {
3528 	struct pci_dev *dev;
3529 
3530 	list_for_each_entry(dev, &bus->devices, bus_list) {
3531 		pci_dev_lock(dev);
3532 		if (dev->subordinate)
3533 			pci_bus_lock(dev->subordinate);
3534 	}
3535 }
3536 
3537 /* Unlock devices from the bottom of the tree up */
3538 static void pci_bus_unlock(struct pci_bus *bus)
3539 {
3540 	struct pci_dev *dev;
3541 
3542 	list_for_each_entry(dev, &bus->devices, bus_list) {
3543 		if (dev->subordinate)
3544 			pci_bus_unlock(dev->subordinate);
3545 		pci_dev_unlock(dev);
3546 	}
3547 }
3548 
3549 /* Return 1 on successful lock, 0 on contention */
3550 static int pci_bus_trylock(struct pci_bus *bus)
3551 {
3552 	struct pci_dev *dev;
3553 
3554 	list_for_each_entry(dev, &bus->devices, bus_list) {
3555 		if (!pci_dev_trylock(dev))
3556 			goto unlock;
3557 		if (dev->subordinate) {
3558 			if (!pci_bus_trylock(dev->subordinate)) {
3559 				pci_dev_unlock(dev);
3560 				goto unlock;
3561 			}
3562 		}
3563 	}
3564 	return 1;
3565 
3566 unlock:
3567 	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
3568 		if (dev->subordinate)
3569 			pci_bus_unlock(dev->subordinate);
3570 		pci_dev_unlock(dev);
3571 	}
3572 	return 0;
3573 }
3574 
3575 /* Lock devices from the top of the tree down */
3576 static void pci_slot_lock(struct pci_slot *slot)
3577 {
3578 	struct pci_dev *dev;
3579 
3580 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
3581 		if (!dev->slot || dev->slot != slot)
3582 			continue;
3583 		pci_dev_lock(dev);
3584 		if (dev->subordinate)
3585 			pci_bus_lock(dev->subordinate);
3586 	}
3587 }
3588 
3589 /* Unlock devices from the bottom of the tree up */
3590 static void pci_slot_unlock(struct pci_slot *slot)
3591 {
3592 	struct pci_dev *dev;
3593 
3594 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
3595 		if (!dev->slot || dev->slot != slot)
3596 			continue;
3597 		if (dev->subordinate)
3598 			pci_bus_unlock(dev->subordinate);
3599 		pci_dev_unlock(dev);
3600 	}
3601 }
3602 
3603 /* Return 1 on successful lock, 0 on contention */
3604 static int pci_slot_trylock(struct pci_slot *slot)
3605 {
3606 	struct pci_dev *dev;
3607 
3608 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
3609 		if (!dev->slot || dev->slot != slot)
3610 			continue;
3611 		if (!pci_dev_trylock(dev))
3612 			goto unlock;
3613 		if (dev->subordinate) {
3614 			if (!pci_bus_trylock(dev->subordinate)) {
3615 				pci_dev_unlock(dev);
3616 				goto unlock;
3617 			}
3618 		}
3619 	}
3620 	return 1;
3621 
3622 unlock:
3623 	list_for_each_entry_continue_reverse(dev,
3624 					     &slot->bus->devices, bus_list) {
3625 		if (!dev->slot || dev->slot != slot)
3626 			continue;
3627 		if (dev->subordinate)
3628 			pci_bus_unlock(dev->subordinate);
3629 		pci_dev_unlock(dev);
3630 	}
3631 	return 0;
3632 }
3633 
3634 /* Save and disable devices from the top of the tree down */
3635 static void pci_bus_save_and_disable(struct pci_bus *bus)
3636 {
3637 	struct pci_dev *dev;
3638 
3639 	list_for_each_entry(dev, &bus->devices, bus_list) {
3640 		pci_dev_save_and_disable(dev);
3641 		if (dev->subordinate)
3642 			pci_bus_save_and_disable(dev->subordinate);
3643 	}
3644 }
3645 
3646 /*
3647  * Restore devices from top of the tree down - parent bridges need to be
3648  * restored before we can get to subordinate devices.
3649  */
3650 static void pci_bus_restore(struct pci_bus *bus)
3651 {
3652 	struct pci_dev *dev;
3653 
3654 	list_for_each_entry(dev, &bus->devices, bus_list) {
3655 		pci_dev_restore(dev);
3656 		if (dev->subordinate)
3657 			pci_bus_restore(dev->subordinate);
3658 	}
3659 }
3660 
3661 /* Save and disable devices from the top of the tree down */
3662 static void pci_slot_save_and_disable(struct pci_slot *slot)
3663 {
3664 	struct pci_dev *dev;
3665 
3666 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
3667 		if (!dev->slot || dev->slot != slot)
3668 			continue;
3669 		pci_dev_save_and_disable(dev);
3670 		if (dev->subordinate)
3671 			pci_bus_save_and_disable(dev->subordinate);
3672 	}
3673 }
3674 
3675 /*
3676  * Restore devices from top of the tree down - parent bridges need to be
3677  * restored before we can get to subordinate devices.
3678  */
3679 static void pci_slot_restore(struct pci_slot *slot)
3680 {
3681 	struct pci_dev *dev;
3682 
3683 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
3684 		if (!dev->slot || dev->slot != slot)
3685 			continue;
3686 		pci_dev_restore(dev);
3687 		if (dev->subordinate)
3688 			pci_bus_restore(dev->subordinate);
3689 	}
3690 }
3691 
3692 static int pci_slot_reset(struct pci_slot *slot, int probe)
3693 {
3694 	int rc;
3695 
3696 	if (!slot)
3697 		return -ENOTTY;
3698 
3699 	if (!probe)
3700 		pci_slot_lock(slot);
3701 
3702 	might_sleep();
3703 
3704 	rc = pci_reset_hotplug_slot(slot->hotplug, probe);
3705 
3706 	if (!probe)
3707 		pci_slot_unlock(slot);
3708 
3709 	return rc;
3710 }
3711 
3712 /**
3713  * pci_probe_reset_slot - probe whether a PCI slot can be reset
3714  * @slot: PCI slot to probe
3715  *
3716  * Return 0 if slot can be reset, negative if a slot reset is not supported.
3717  */
3718 int pci_probe_reset_slot(struct pci_slot *slot)
3719 {
3720 	return pci_slot_reset(slot, 1);
3721 }
3722 EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
3723 
3724 /**
3725  * pci_reset_slot - reset a PCI slot
3726  * @slot: PCI slot to reset
3727  *
3728  * A PCI bus may host multiple slots, each slot may support a reset mechanism
3729  * independent of other slots.  For instance, some slots may support slot power
3730  * control.  In the case of a 1:1 bus to slot architecture, this function may
3731  * wrap the bus reset to avoid spurious slot related events such as hotplug.
3732  * Generally a slot reset should be attempted before a bus reset.  All of the
3733  * function of the slot and any subordinate buses behind the slot are reset
3734  * through this function.  PCI config space of all devices in the slot and
3735  * behind the slot is saved before and restored after reset.
3736  *
3737  * Return 0 on success, non-zero on error.
3738  */
3739 int pci_reset_slot(struct pci_slot *slot)
3740 {
3741 	int rc;
3742 
3743 	rc = pci_slot_reset(slot, 1);
3744 	if (rc)
3745 		return rc;
3746 
3747 	pci_slot_save_and_disable(slot);
3748 
3749 	rc = pci_slot_reset(slot, 0);
3750 
3751 	pci_slot_restore(slot);
3752 
3753 	return rc;
3754 }
3755 EXPORT_SYMBOL_GPL(pci_reset_slot);
3756 
3757 /**
3758  * pci_try_reset_slot - Try to reset a PCI slot
3759  * @slot: PCI slot to reset
3760  *
3761  * Same as above except return -EAGAIN if the slot cannot be locked
3762  */
3763 int pci_try_reset_slot(struct pci_slot *slot)
3764 {
3765 	int rc;
3766 
3767 	rc = pci_slot_reset(slot, 1);
3768 	if (rc)
3769 		return rc;
3770 
3771 	pci_slot_save_and_disable(slot);
3772 
3773 	if (pci_slot_trylock(slot)) {
3774 		might_sleep();
3775 		rc = pci_reset_hotplug_slot(slot->hotplug, 0);
3776 		pci_slot_unlock(slot);
3777 	} else
3778 		rc = -EAGAIN;
3779 
3780 	pci_slot_restore(slot);
3781 
3782 	return rc;
3783 }
3784 EXPORT_SYMBOL_GPL(pci_try_reset_slot);
3785 
3786 static int pci_bus_reset(struct pci_bus *bus, int probe)
3787 {
3788 	if (!bus->self)
3789 		return -ENOTTY;
3790 
3791 	if (probe)
3792 		return 0;
3793 
3794 	pci_bus_lock(bus);
3795 
3796 	might_sleep();
3797 
3798 	pci_reset_bridge_secondary_bus(bus->self);
3799 
3800 	pci_bus_unlock(bus);
3801 
3802 	return 0;
3803 }
3804 
3805 /**
3806  * pci_probe_reset_bus - probe whether a PCI bus can be reset
3807  * @bus: PCI bus to probe
3808  *
3809  * Return 0 if bus can be reset, negative if a bus reset is not supported.
3810  */
3811 int pci_probe_reset_bus(struct pci_bus *bus)
3812 {
3813 	return pci_bus_reset(bus, 1);
3814 }
3815 EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
3816 
3817 /**
3818  * pci_reset_bus - reset a PCI bus
3819  * @bus: top level PCI bus to reset
3820  *
3821  * Do a bus reset on the given bus and any subordinate buses, saving
3822  * and restoring state of all devices.
3823  *
3824  * Return 0 on success, non-zero on error.
3825  */
3826 int pci_reset_bus(struct pci_bus *bus)
3827 {
3828 	int rc;
3829 
3830 	rc = pci_bus_reset(bus, 1);
3831 	if (rc)
3832 		return rc;
3833 
3834 	pci_bus_save_and_disable(bus);
3835 
3836 	rc = pci_bus_reset(bus, 0);
3837 
3838 	pci_bus_restore(bus);
3839 
3840 	return rc;
3841 }
3842 EXPORT_SYMBOL_GPL(pci_reset_bus);
3843 
3844 /**
3845  * pci_try_reset_bus - Try to reset a PCI bus
3846  * @bus: top level PCI bus to reset
3847  *
3848  * Same as above except return -EAGAIN if the bus cannot be locked
3849  */
3850 int pci_try_reset_bus(struct pci_bus *bus)
3851 {
3852 	int rc;
3853 
3854 	rc = pci_bus_reset(bus, 1);
3855 	if (rc)
3856 		return rc;
3857 
3858 	pci_bus_save_and_disable(bus);
3859 
3860 	if (pci_bus_trylock(bus)) {
3861 		might_sleep();
3862 		pci_reset_bridge_secondary_bus(bus->self);
3863 		pci_bus_unlock(bus);
3864 	} else
3865 		rc = -EAGAIN;
3866 
3867 	pci_bus_restore(bus);
3868 
3869 	return rc;
3870 }
3871 EXPORT_SYMBOL_GPL(pci_try_reset_bus);
3872 
3873 /**
3874  * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
3875  * @dev: PCI device to query
3876  *
3877  * Returns mmrbc: maximum designed memory read count in bytes
3878  *    or appropriate error value.
3879  */
3880 int pcix_get_max_mmrbc(struct pci_dev *dev)
3881 {
3882 	int cap;
3883 	u32 stat;
3884 
3885 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
3886 	if (!cap)
3887 		return -EINVAL;
3888 
3889 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
3890 		return -EINVAL;
3891 
3892 	return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21);
3893 }
3894 EXPORT_SYMBOL(pcix_get_max_mmrbc);
3895 
3896 /**
3897  * pcix_get_mmrbc - get PCI-X maximum memory read byte count
3898  * @dev: PCI device to query
3899  *
3900  * Returns mmrbc: maximum memory read count in bytes
3901  *    or appropriate error value.
3902  */
3903 int pcix_get_mmrbc(struct pci_dev *dev)
3904 {
3905 	int cap;
3906 	u16 cmd;
3907 
3908 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
3909 	if (!cap)
3910 		return -EINVAL;
3911 
3912 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
3913 		return -EINVAL;
3914 
3915 	return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2);
3916 }
3917 EXPORT_SYMBOL(pcix_get_mmrbc);
3918 
3919 /**
3920  * pcix_set_mmrbc - set PCI-X maximum memory read byte count
3921  * @dev: PCI device to query
3922  * @mmrbc: maximum memory read count in bytes
3923  *    valid values are 512, 1024, 2048, 4096
3924  *
3925  * If possible sets maximum memory read byte count, some bridges have erratas
3926  * that prevent this.
3927  */
3928 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
3929 {
3930 	int cap;
3931 	u32 stat, v, o;
3932 	u16 cmd;
3933 
3934 	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
3935 		return -EINVAL;
3936 
3937 	v = ffs(mmrbc) - 10;
3938 
3939 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
3940 	if (!cap)
3941 		return -EINVAL;
3942 
3943 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
3944 		return -EINVAL;
3945 
3946 	if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21)
3947 		return -E2BIG;
3948 
3949 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
3950 		return -EINVAL;
3951 
3952 	o = (cmd & PCI_X_CMD_MAX_READ) >> 2;
3953 	if (o != v) {
3954 		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
3955 			return -EIO;
3956 
3957 		cmd &= ~PCI_X_CMD_MAX_READ;
3958 		cmd |= v << 2;
3959 		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
3960 			return -EIO;
3961 	}
3962 	return 0;
3963 }
3964 EXPORT_SYMBOL(pcix_set_mmrbc);
3965 
3966 /**
3967  * pcie_get_readrq - get PCI Express read request size
3968  * @dev: PCI device to query
3969  *
3970  * Returns maximum memory read request in bytes
3971  *    or appropriate error value.
3972  */
3973 int pcie_get_readrq(struct pci_dev *dev)
3974 {
3975 	u16 ctl;
3976 
3977 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
3978 
3979 	return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12);
3980 }
3981 EXPORT_SYMBOL(pcie_get_readrq);
3982 
3983 /**
3984  * pcie_set_readrq - set PCI Express maximum memory read request
3985  * @dev: PCI device to query
3986  * @rq: maximum memory read count in bytes
3987  *    valid values are 128, 256, 512, 1024, 2048, 4096
3988  *
3989  * If possible sets maximum memory read request in bytes
3990  */
3991 int pcie_set_readrq(struct pci_dev *dev, int rq)
3992 {
3993 	u16 v;
3994 
3995 	if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
3996 		return -EINVAL;
3997 
3998 	/*
3999 	 * If using the "performance" PCIe config, we clamp the
4000 	 * read rq size to the max packet size to prevent the
4001 	 * host bridge generating requests larger than we can
4002 	 * cope with
4003 	 */
4004 	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
4005 		int mps = pcie_get_mps(dev);
4006 
4007 		if (mps < rq)
4008 			rq = mps;
4009 	}
4010 
4011 	v = (ffs(rq) - 8) << 12;
4012 
4013 	return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
4014 						  PCI_EXP_DEVCTL_READRQ, v);
4015 }
4016 EXPORT_SYMBOL(pcie_set_readrq);
4017 
4018 /**
4019  * pcie_get_mps - get PCI Express maximum payload size
4020  * @dev: PCI device to query
4021  *
4022  * Returns maximum payload size in bytes
4023  */
4024 int pcie_get_mps(struct pci_dev *dev)
4025 {
4026 	u16 ctl;
4027 
4028 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
4029 
4030 	return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5);
4031 }
4032 EXPORT_SYMBOL(pcie_get_mps);
4033 
4034 /**
4035  * pcie_set_mps - set PCI Express maximum payload size
4036  * @dev: PCI device to query
4037  * @mps: maximum payload size in bytes
4038  *    valid values are 128, 256, 512, 1024, 2048, 4096
4039  *
4040  * If possible sets maximum payload size
4041  */
4042 int pcie_set_mps(struct pci_dev *dev, int mps)
4043 {
4044 	u16 v;
4045 
4046 	if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
4047 		return -EINVAL;
4048 
4049 	v = ffs(mps) - 8;
4050 	if (v > dev->pcie_mpss)
4051 		return -EINVAL;
4052 	v <<= 5;
4053 
4054 	return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
4055 						  PCI_EXP_DEVCTL_PAYLOAD, v);
4056 }
4057 EXPORT_SYMBOL(pcie_set_mps);
4058 
4059 /**
4060  * pcie_get_minimum_link - determine minimum link settings of a PCI device
4061  * @dev: PCI device to query
4062  * @speed: storage for minimum speed
4063  * @width: storage for minimum width
4064  *
4065  * This function will walk up the PCI device chain and determine the minimum
4066  * link width and speed of the device.
4067  */
4068 int pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed,
4069 			  enum pcie_link_width *width)
4070 {
4071 	int ret;
4072 
4073 	*speed = PCI_SPEED_UNKNOWN;
4074 	*width = PCIE_LNK_WIDTH_UNKNOWN;
4075 
4076 	while (dev) {
4077 		u16 lnksta;
4078 		enum pci_bus_speed next_speed;
4079 		enum pcie_link_width next_width;
4080 
4081 		ret = pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
4082 		if (ret)
4083 			return ret;
4084 
4085 		next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS];
4086 		next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >>
4087 			PCI_EXP_LNKSTA_NLW_SHIFT;
4088 
4089 		if (next_speed < *speed)
4090 			*speed = next_speed;
4091 
4092 		if (next_width < *width)
4093 			*width = next_width;
4094 
4095 		dev = dev->bus->self;
4096 	}
4097 
4098 	return 0;
4099 }
4100 EXPORT_SYMBOL(pcie_get_minimum_link);
4101 
4102 /**
4103  * pci_select_bars - Make BAR mask from the type of resource
4104  * @dev: the PCI device for which BAR mask is made
4105  * @flags: resource type mask to be selected
4106  *
4107  * This helper routine makes bar mask from the type of resource.
4108  */
4109 int pci_select_bars(struct pci_dev *dev, unsigned long flags)
4110 {
4111 	int i, bars = 0;
4112 	for (i = 0; i < PCI_NUM_RESOURCES; i++)
4113 		if (pci_resource_flags(dev, i) & flags)
4114 			bars |= (1 << i);
4115 	return bars;
4116 }
4117 EXPORT_SYMBOL(pci_select_bars);
4118 
4119 /**
4120  * pci_resource_bar - get position of the BAR associated with a resource
4121  * @dev: the PCI device
4122  * @resno: the resource number
4123  * @type: the BAR type to be filled in
4124  *
4125  * Returns BAR position in config space, or 0 if the BAR is invalid.
4126  */
4127 int pci_resource_bar(struct pci_dev *dev, int resno, enum pci_bar_type *type)
4128 {
4129 	int reg;
4130 
4131 	if (resno < PCI_ROM_RESOURCE) {
4132 		*type = pci_bar_unknown;
4133 		return PCI_BASE_ADDRESS_0 + 4 * resno;
4134 	} else if (resno == PCI_ROM_RESOURCE) {
4135 		*type = pci_bar_mem32;
4136 		return dev->rom_base_reg;
4137 	} else if (resno < PCI_BRIDGE_RESOURCES) {
4138 		/* device specific resource */
4139 		reg = pci_iov_resource_bar(dev, resno, type);
4140 		if (reg)
4141 			return reg;
4142 	}
4143 
4144 	dev_err(&dev->dev, "BAR %d: invalid resource\n", resno);
4145 	return 0;
4146 }
4147 
4148 /* Some architectures require additional programming to enable VGA */
4149 static arch_set_vga_state_t arch_set_vga_state;
4150 
4151 void __init pci_register_set_vga_state(arch_set_vga_state_t func)
4152 {
4153 	arch_set_vga_state = func;	/* NULL disables */
4154 }
4155 
4156 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
4157 				  unsigned int command_bits, u32 flags)
4158 {
4159 	if (arch_set_vga_state)
4160 		return arch_set_vga_state(dev, decode, command_bits,
4161 						flags);
4162 	return 0;
4163 }
4164 
4165 /**
4166  * pci_set_vga_state - set VGA decode state on device and parents if requested
4167  * @dev: the PCI device
4168  * @decode: true = enable decoding, false = disable decoding
4169  * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
4170  * @flags: traverse ancestors and change bridges
4171  * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
4172  */
4173 int pci_set_vga_state(struct pci_dev *dev, bool decode,
4174 		      unsigned int command_bits, u32 flags)
4175 {
4176 	struct pci_bus *bus;
4177 	struct pci_dev *bridge;
4178 	u16 cmd;
4179 	int rc;
4180 
4181 	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
4182 
4183 	/* ARCH specific VGA enables */
4184 	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
4185 	if (rc)
4186 		return rc;
4187 
4188 	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
4189 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
4190 		if (decode == true)
4191 			cmd |= command_bits;
4192 		else
4193 			cmd &= ~command_bits;
4194 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4195 	}
4196 
4197 	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
4198 		return 0;
4199 
4200 	bus = dev->bus;
4201 	while (bus) {
4202 		bridge = bus->self;
4203 		if (bridge) {
4204 			pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
4205 					     &cmd);
4206 			if (decode == true)
4207 				cmd |= PCI_BRIDGE_CTL_VGA;
4208 			else
4209 				cmd &= ~PCI_BRIDGE_CTL_VGA;
4210 			pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
4211 					      cmd);
4212 		}
4213 		bus = bus->parent;
4214 	}
4215 	return 0;
4216 }
4217 
4218 bool pci_device_is_present(struct pci_dev *pdev)
4219 {
4220 	u32 v;
4221 
4222 	return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
4223 }
4224 EXPORT_SYMBOL_GPL(pci_device_is_present);
4225 
4226 #define RESOURCE_ALIGNMENT_PARAM_SIZE COMMAND_LINE_SIZE
4227 static char resource_alignment_param[RESOURCE_ALIGNMENT_PARAM_SIZE] = {0};
4228 static DEFINE_SPINLOCK(resource_alignment_lock);
4229 
4230 /**
4231  * pci_specified_resource_alignment - get resource alignment specified by user.
4232  * @dev: the PCI device to get
4233  *
4234  * RETURNS: Resource alignment if it is specified.
4235  *          Zero if it is not specified.
4236  */
4237 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev)
4238 {
4239 	int seg, bus, slot, func, align_order, count;
4240 	resource_size_t align = 0;
4241 	char *p;
4242 
4243 	spin_lock(&resource_alignment_lock);
4244 	p = resource_alignment_param;
4245 	while (*p) {
4246 		count = 0;
4247 		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
4248 							p[count] == '@') {
4249 			p += count + 1;
4250 		} else {
4251 			align_order = -1;
4252 		}
4253 		if (sscanf(p, "%x:%x:%x.%x%n",
4254 			&seg, &bus, &slot, &func, &count) != 4) {
4255 			seg = 0;
4256 			if (sscanf(p, "%x:%x.%x%n",
4257 					&bus, &slot, &func, &count) != 3) {
4258 				/* Invalid format */
4259 				printk(KERN_ERR "PCI: Can't parse resource_alignment parameter: %s\n",
4260 					p);
4261 				break;
4262 			}
4263 		}
4264 		p += count;
4265 		if (seg == pci_domain_nr(dev->bus) &&
4266 			bus == dev->bus->number &&
4267 			slot == PCI_SLOT(dev->devfn) &&
4268 			func == PCI_FUNC(dev->devfn)) {
4269 			if (align_order == -1)
4270 				align = PAGE_SIZE;
4271 			else
4272 				align = 1 << align_order;
4273 			/* Found */
4274 			break;
4275 		}
4276 		if (*p != ';' && *p != ',') {
4277 			/* End of param or invalid format */
4278 			break;
4279 		}
4280 		p++;
4281 	}
4282 	spin_unlock(&resource_alignment_lock);
4283 	return align;
4284 }
4285 
4286 /*
4287  * This function disables memory decoding and releases memory resources
4288  * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
4289  * It also rounds up size to specified alignment.
4290  * Later on, the kernel will assign page-aligned memory resource back
4291  * to the device.
4292  */
4293 void pci_reassigndev_resource_alignment(struct pci_dev *dev)
4294 {
4295 	int i;
4296 	struct resource *r;
4297 	resource_size_t align, size;
4298 	u16 command;
4299 
4300 	/* check if specified PCI is target device to reassign */
4301 	align = pci_specified_resource_alignment(dev);
4302 	if (!align)
4303 		return;
4304 
4305 	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
4306 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
4307 		dev_warn(&dev->dev,
4308 			"Can't reassign resources to host bridge.\n");
4309 		return;
4310 	}
4311 
4312 	dev_info(&dev->dev,
4313 		"Disabling memory decoding and releasing memory resources.\n");
4314 	pci_read_config_word(dev, PCI_COMMAND, &command);
4315 	command &= ~PCI_COMMAND_MEMORY;
4316 	pci_write_config_word(dev, PCI_COMMAND, command);
4317 
4318 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) {
4319 		r = &dev->resource[i];
4320 		if (!(r->flags & IORESOURCE_MEM))
4321 			continue;
4322 		size = resource_size(r);
4323 		if (size < align) {
4324 			size = align;
4325 			dev_info(&dev->dev,
4326 				"Rounding up size of resource #%d to %#llx.\n",
4327 				i, (unsigned long long)size);
4328 		}
4329 		r->flags |= IORESOURCE_UNSET;
4330 		r->end = size - 1;
4331 		r->start = 0;
4332 	}
4333 	/* Need to disable bridge's resource window,
4334 	 * to enable the kernel to reassign new resource
4335 	 * window later on.
4336 	 */
4337 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE &&
4338 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
4339 		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
4340 			r = &dev->resource[i];
4341 			if (!(r->flags & IORESOURCE_MEM))
4342 				continue;
4343 			r->flags |= IORESOURCE_UNSET;
4344 			r->end = resource_size(r) - 1;
4345 			r->start = 0;
4346 		}
4347 		pci_disable_bridge_window(dev);
4348 	}
4349 }
4350 
4351 static ssize_t pci_set_resource_alignment_param(const char *buf, size_t count)
4352 {
4353 	if (count > RESOURCE_ALIGNMENT_PARAM_SIZE - 1)
4354 		count = RESOURCE_ALIGNMENT_PARAM_SIZE - 1;
4355 	spin_lock(&resource_alignment_lock);
4356 	strncpy(resource_alignment_param, buf, count);
4357 	resource_alignment_param[count] = '\0';
4358 	spin_unlock(&resource_alignment_lock);
4359 	return count;
4360 }
4361 
4362 static ssize_t pci_get_resource_alignment_param(char *buf, size_t size)
4363 {
4364 	size_t count;
4365 	spin_lock(&resource_alignment_lock);
4366 	count = snprintf(buf, size, "%s", resource_alignment_param);
4367 	spin_unlock(&resource_alignment_lock);
4368 	return count;
4369 }
4370 
4371 static ssize_t pci_resource_alignment_show(struct bus_type *bus, char *buf)
4372 {
4373 	return pci_get_resource_alignment_param(buf, PAGE_SIZE);
4374 }
4375 
4376 static ssize_t pci_resource_alignment_store(struct bus_type *bus,
4377 					const char *buf, size_t count)
4378 {
4379 	return pci_set_resource_alignment_param(buf, count);
4380 }
4381 
4382 BUS_ATTR(resource_alignment, 0644, pci_resource_alignment_show,
4383 					pci_resource_alignment_store);
4384 
4385 static int __init pci_resource_alignment_sysfs_init(void)
4386 {
4387 	return bus_create_file(&pci_bus_type,
4388 					&bus_attr_resource_alignment);
4389 }
4390 late_initcall(pci_resource_alignment_sysfs_init);
4391 
4392 static void pci_no_domains(void)
4393 {
4394 #ifdef CONFIG_PCI_DOMAINS
4395 	pci_domains_supported = 0;
4396 #endif
4397 }
4398 
4399 /**
4400  * pci_ext_cfg_avail - can we access extended PCI config space?
4401  *
4402  * Returns 1 if we can access PCI extended config space (offsets
4403  * greater than 0xff). This is the default implementation. Architecture
4404  * implementations can override this.
4405  */
4406 int __weak pci_ext_cfg_avail(void)
4407 {
4408 	return 1;
4409 }
4410 
4411 void __weak pci_fixup_cardbus(struct pci_bus *bus)
4412 {
4413 }
4414 EXPORT_SYMBOL(pci_fixup_cardbus);
4415 
4416 static int __init pci_setup(char *str)
4417 {
4418 	while (str) {
4419 		char *k = strchr(str, ',');
4420 		if (k)
4421 			*k++ = 0;
4422 		if (*str && (str = pcibios_setup(str)) && *str) {
4423 			if (!strcmp(str, "nomsi")) {
4424 				pci_no_msi();
4425 			} else if (!strcmp(str, "noaer")) {
4426 				pci_no_aer();
4427 			} else if (!strncmp(str, "realloc=", 8)) {
4428 				pci_realloc_get_opt(str + 8);
4429 			} else if (!strncmp(str, "realloc", 7)) {
4430 				pci_realloc_get_opt("on");
4431 			} else if (!strcmp(str, "nodomains")) {
4432 				pci_no_domains();
4433 			} else if (!strncmp(str, "noari", 5)) {
4434 				pcie_ari_disabled = true;
4435 			} else if (!strncmp(str, "cbiosize=", 9)) {
4436 				pci_cardbus_io_size = memparse(str + 9, &str);
4437 			} else if (!strncmp(str, "cbmemsize=", 10)) {
4438 				pci_cardbus_mem_size = memparse(str + 10, &str);
4439 			} else if (!strncmp(str, "resource_alignment=", 19)) {
4440 				pci_set_resource_alignment_param(str + 19,
4441 							strlen(str + 19));
4442 			} else if (!strncmp(str, "ecrc=", 5)) {
4443 				pcie_ecrc_get_policy(str + 5);
4444 			} else if (!strncmp(str, "hpiosize=", 9)) {
4445 				pci_hotplug_io_size = memparse(str + 9, &str);
4446 			} else if (!strncmp(str, "hpmemsize=", 10)) {
4447 				pci_hotplug_mem_size = memparse(str + 10, &str);
4448 			} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
4449 				pcie_bus_config = PCIE_BUS_TUNE_OFF;
4450 			} else if (!strncmp(str, "pcie_bus_safe", 13)) {
4451 				pcie_bus_config = PCIE_BUS_SAFE;
4452 			} else if (!strncmp(str, "pcie_bus_perf", 13)) {
4453 				pcie_bus_config = PCIE_BUS_PERFORMANCE;
4454 			} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
4455 				pcie_bus_config = PCIE_BUS_PEER2PEER;
4456 			} else if (!strncmp(str, "pcie_scan_all", 13)) {
4457 				pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
4458 			} else {
4459 				printk(KERN_ERR "PCI: Unknown option `%s'\n",
4460 						str);
4461 			}
4462 		}
4463 		str = k;
4464 	}
4465 	return 0;
4466 }
4467 early_param("pci", pci_setup);
4468