xref: /openbmc/linux/drivers/pci/pci.c (revision c8dbaa22)
1 /*
2  *	PCI Bus Services, see include/linux/pci.h for further explanation.
3  *
4  *	Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
5  *	David Mosberger-Tang
6  *
7  *	Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
8  */
9 
10 #include <linux/acpi.h>
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/dmi.h>
14 #include <linux/init.h>
15 #include <linux/of.h>
16 #include <linux/of_pci.h>
17 #include <linux/pci.h>
18 #include <linux/pm.h>
19 #include <linux/slab.h>
20 #include <linux/module.h>
21 #include <linux/spinlock.h>
22 #include <linux/string.h>
23 #include <linux/log2.h>
24 #include <linux/pci-aspm.h>
25 #include <linux/pm_wakeup.h>
26 #include <linux/interrupt.h>
27 #include <linux/device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/pci_hotplug.h>
30 #include <linux/vmalloc.h>
31 #include <linux/pci-ats.h>
32 #include <asm/setup.h>
33 #include <asm/dma.h>
34 #include <linux/aer.h>
35 #include "pci.h"
36 
37 const char *pci_power_names[] = {
38 	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
39 };
40 EXPORT_SYMBOL_GPL(pci_power_names);
41 
42 int isa_dma_bridge_buggy;
43 EXPORT_SYMBOL(isa_dma_bridge_buggy);
44 
45 int pci_pci_problems;
46 EXPORT_SYMBOL(pci_pci_problems);
47 
48 unsigned int pci_pm_d3_delay;
49 
50 static void pci_pme_list_scan(struct work_struct *work);
51 
52 static LIST_HEAD(pci_pme_list);
53 static DEFINE_MUTEX(pci_pme_list_mutex);
54 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
55 
56 struct pci_pme_device {
57 	struct list_head list;
58 	struct pci_dev *dev;
59 };
60 
61 #define PME_TIMEOUT 1000 /* How long between PME checks */
62 
63 static void pci_dev_d3_sleep(struct pci_dev *dev)
64 {
65 	unsigned int delay = dev->d3_delay;
66 
67 	if (delay < pci_pm_d3_delay)
68 		delay = pci_pm_d3_delay;
69 
70 	if (delay)
71 		msleep(delay);
72 }
73 
74 #ifdef CONFIG_PCI_DOMAINS
75 int pci_domains_supported = 1;
76 #endif
77 
78 #define DEFAULT_CARDBUS_IO_SIZE		(256)
79 #define DEFAULT_CARDBUS_MEM_SIZE	(64*1024*1024)
80 /* pci=cbmemsize=nnM,cbiosize=nn can override this */
81 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
82 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
83 
84 #define DEFAULT_HOTPLUG_IO_SIZE		(256)
85 #define DEFAULT_HOTPLUG_MEM_SIZE	(2*1024*1024)
86 /* pci=hpmemsize=nnM,hpiosize=nn can override this */
87 unsigned long pci_hotplug_io_size  = DEFAULT_HOTPLUG_IO_SIZE;
88 unsigned long pci_hotplug_mem_size = DEFAULT_HOTPLUG_MEM_SIZE;
89 
90 #define DEFAULT_HOTPLUG_BUS_SIZE	1
91 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
92 
93 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
94 
95 /*
96  * The default CLS is used if arch didn't set CLS explicitly and not
97  * all pci devices agree on the same value.  Arch can override either
98  * the dfl or actual value as it sees fit.  Don't forget this is
99  * measured in 32-bit words, not bytes.
100  */
101 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
102 u8 pci_cache_line_size;
103 
104 /*
105  * If we set up a device for bus mastering, we need to check the latency
106  * timer as certain BIOSes forget to set it properly.
107  */
108 unsigned int pcibios_max_latency = 255;
109 
110 /* If set, the PCIe ARI capability will not be used. */
111 static bool pcie_ari_disabled;
112 
113 /* Disable bridge_d3 for all PCIe ports */
114 static bool pci_bridge_d3_disable;
115 /* Force bridge_d3 for all PCIe ports */
116 static bool pci_bridge_d3_force;
117 
118 static int __init pcie_port_pm_setup(char *str)
119 {
120 	if (!strcmp(str, "off"))
121 		pci_bridge_d3_disable = true;
122 	else if (!strcmp(str, "force"))
123 		pci_bridge_d3_force = true;
124 	return 1;
125 }
126 __setup("pcie_port_pm=", pcie_port_pm_setup);
127 
128 /**
129  * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
130  * @bus: pointer to PCI bus structure to search
131  *
132  * Given a PCI bus, returns the highest PCI bus number present in the set
133  * including the given PCI bus and its list of child PCI buses.
134  */
135 unsigned char pci_bus_max_busnr(struct pci_bus *bus)
136 {
137 	struct pci_bus *tmp;
138 	unsigned char max, n;
139 
140 	max = bus->busn_res.end;
141 	list_for_each_entry(tmp, &bus->children, node) {
142 		n = pci_bus_max_busnr(tmp);
143 		if (n > max)
144 			max = n;
145 	}
146 	return max;
147 }
148 EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
149 
150 #ifdef CONFIG_HAS_IOMEM
151 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
152 {
153 	struct resource *res = &pdev->resource[bar];
154 
155 	/*
156 	 * Make sure the BAR is actually a memory resource, not an IO resource
157 	 */
158 	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
159 		dev_warn(&pdev->dev, "can't ioremap BAR %d: %pR\n", bar, res);
160 		return NULL;
161 	}
162 	return ioremap_nocache(res->start, resource_size(res));
163 }
164 EXPORT_SYMBOL_GPL(pci_ioremap_bar);
165 
166 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
167 {
168 	/*
169 	 * Make sure the BAR is actually a memory resource, not an IO resource
170 	 */
171 	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) {
172 		WARN_ON(1);
173 		return NULL;
174 	}
175 	return ioremap_wc(pci_resource_start(pdev, bar),
176 			  pci_resource_len(pdev, bar));
177 }
178 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
179 #endif
180 
181 
182 static int __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
183 				   u8 pos, int cap, int *ttl)
184 {
185 	u8 id;
186 	u16 ent;
187 
188 	pci_bus_read_config_byte(bus, devfn, pos, &pos);
189 
190 	while ((*ttl)--) {
191 		if (pos < 0x40)
192 			break;
193 		pos &= ~3;
194 		pci_bus_read_config_word(bus, devfn, pos, &ent);
195 
196 		id = ent & 0xff;
197 		if (id == 0xff)
198 			break;
199 		if (id == cap)
200 			return pos;
201 		pos = (ent >> 8);
202 	}
203 	return 0;
204 }
205 
206 static int __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
207 			       u8 pos, int cap)
208 {
209 	int ttl = PCI_FIND_CAP_TTL;
210 
211 	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl);
212 }
213 
214 int pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
215 {
216 	return __pci_find_next_cap(dev->bus, dev->devfn,
217 				   pos + PCI_CAP_LIST_NEXT, cap);
218 }
219 EXPORT_SYMBOL_GPL(pci_find_next_capability);
220 
221 static int __pci_bus_find_cap_start(struct pci_bus *bus,
222 				    unsigned int devfn, u8 hdr_type)
223 {
224 	u16 status;
225 
226 	pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status);
227 	if (!(status & PCI_STATUS_CAP_LIST))
228 		return 0;
229 
230 	switch (hdr_type) {
231 	case PCI_HEADER_TYPE_NORMAL:
232 	case PCI_HEADER_TYPE_BRIDGE:
233 		return PCI_CAPABILITY_LIST;
234 	case PCI_HEADER_TYPE_CARDBUS:
235 		return PCI_CB_CAPABILITY_LIST;
236 	}
237 
238 	return 0;
239 }
240 
241 /**
242  * pci_find_capability - query for devices' capabilities
243  * @dev: PCI device to query
244  * @cap: capability code
245  *
246  * Tell if a device supports a given PCI capability.
247  * Returns the address of the requested capability structure within the
248  * device's PCI configuration space or 0 in case the device does not
249  * support it.  Possible values for @cap:
250  *
251  *  %PCI_CAP_ID_PM           Power Management
252  *  %PCI_CAP_ID_AGP          Accelerated Graphics Port
253  *  %PCI_CAP_ID_VPD          Vital Product Data
254  *  %PCI_CAP_ID_SLOTID       Slot Identification
255  *  %PCI_CAP_ID_MSI          Message Signalled Interrupts
256  *  %PCI_CAP_ID_CHSWP        CompactPCI HotSwap
257  *  %PCI_CAP_ID_PCIX         PCI-X
258  *  %PCI_CAP_ID_EXP          PCI Express
259  */
260 int pci_find_capability(struct pci_dev *dev, int cap)
261 {
262 	int pos;
263 
264 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
265 	if (pos)
266 		pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap);
267 
268 	return pos;
269 }
270 EXPORT_SYMBOL(pci_find_capability);
271 
272 /**
273  * pci_bus_find_capability - query for devices' capabilities
274  * @bus:   the PCI bus to query
275  * @devfn: PCI device to query
276  * @cap:   capability code
277  *
278  * Like pci_find_capability() but works for pci devices that do not have a
279  * pci_dev structure set up yet.
280  *
281  * Returns the address of the requested capability structure within the
282  * device's PCI configuration space or 0 in case the device does not
283  * support it.
284  */
285 int pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
286 {
287 	int pos;
288 	u8 hdr_type;
289 
290 	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type);
291 
292 	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f);
293 	if (pos)
294 		pos = __pci_find_next_cap(bus, devfn, pos, cap);
295 
296 	return pos;
297 }
298 EXPORT_SYMBOL(pci_bus_find_capability);
299 
300 /**
301  * pci_find_next_ext_capability - Find an extended capability
302  * @dev: PCI device to query
303  * @start: address at which to start looking (0 to start at beginning of list)
304  * @cap: capability code
305  *
306  * Returns the address of the next matching extended capability structure
307  * within the device's PCI configuration space or 0 if the device does
308  * not support it.  Some capabilities can occur several times, e.g., the
309  * vendor-specific capability, and this provides a way to find them all.
310  */
311 int pci_find_next_ext_capability(struct pci_dev *dev, int start, int cap)
312 {
313 	u32 header;
314 	int ttl;
315 	int pos = PCI_CFG_SPACE_SIZE;
316 
317 	/* minimum 8 bytes per capability */
318 	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
319 
320 	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
321 		return 0;
322 
323 	if (start)
324 		pos = start;
325 
326 	if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
327 		return 0;
328 
329 	/*
330 	 * If we have no capabilities, this is indicated by cap ID,
331 	 * cap version and next pointer all being 0.
332 	 */
333 	if (header == 0)
334 		return 0;
335 
336 	while (ttl-- > 0) {
337 		if (PCI_EXT_CAP_ID(header) == cap && pos != start)
338 			return pos;
339 
340 		pos = PCI_EXT_CAP_NEXT(header);
341 		if (pos < PCI_CFG_SPACE_SIZE)
342 			break;
343 
344 		if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL)
345 			break;
346 	}
347 
348 	return 0;
349 }
350 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
351 
352 /**
353  * pci_find_ext_capability - Find an extended capability
354  * @dev: PCI device to query
355  * @cap: capability code
356  *
357  * Returns the address of the requested extended capability structure
358  * within the device's PCI configuration space or 0 if the device does
359  * not support it.  Possible values for @cap:
360  *
361  *  %PCI_EXT_CAP_ID_ERR		Advanced Error Reporting
362  *  %PCI_EXT_CAP_ID_VC		Virtual Channel
363  *  %PCI_EXT_CAP_ID_DSN		Device Serial Number
364  *  %PCI_EXT_CAP_ID_PWR		Power Budgeting
365  */
366 int pci_find_ext_capability(struct pci_dev *dev, int cap)
367 {
368 	return pci_find_next_ext_capability(dev, 0, cap);
369 }
370 EXPORT_SYMBOL_GPL(pci_find_ext_capability);
371 
372 static int __pci_find_next_ht_cap(struct pci_dev *dev, int pos, int ht_cap)
373 {
374 	int rc, ttl = PCI_FIND_CAP_TTL;
375 	u8 cap, mask;
376 
377 	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
378 		mask = HT_3BIT_CAP_MASK;
379 	else
380 		mask = HT_5BIT_CAP_MASK;
381 
382 	pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos,
383 				      PCI_CAP_ID_HT, &ttl);
384 	while (pos) {
385 		rc = pci_read_config_byte(dev, pos + 3, &cap);
386 		if (rc != PCIBIOS_SUCCESSFUL)
387 			return 0;
388 
389 		if ((cap & mask) == ht_cap)
390 			return pos;
391 
392 		pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn,
393 					      pos + PCI_CAP_LIST_NEXT,
394 					      PCI_CAP_ID_HT, &ttl);
395 	}
396 
397 	return 0;
398 }
399 /**
400  * pci_find_next_ht_capability - query a device's Hypertransport capabilities
401  * @dev: PCI device to query
402  * @pos: Position from which to continue searching
403  * @ht_cap: Hypertransport capability code
404  *
405  * To be used in conjunction with pci_find_ht_capability() to search for
406  * all capabilities matching @ht_cap. @pos should always be a value returned
407  * from pci_find_ht_capability().
408  *
409  * NB. To be 100% safe against broken PCI devices, the caller should take
410  * steps to avoid an infinite loop.
411  */
412 int pci_find_next_ht_capability(struct pci_dev *dev, int pos, int ht_cap)
413 {
414 	return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap);
415 }
416 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
417 
418 /**
419  * pci_find_ht_capability - query a device's Hypertransport capabilities
420  * @dev: PCI device to query
421  * @ht_cap: Hypertransport capability code
422  *
423  * Tell if a device supports a given Hypertransport capability.
424  * Returns an address within the device's PCI configuration space
425  * or 0 in case the device does not support the request capability.
426  * The address points to the PCI capability, of type PCI_CAP_ID_HT,
427  * which has a Hypertransport capability matching @ht_cap.
428  */
429 int pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
430 {
431 	int pos;
432 
433 	pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type);
434 	if (pos)
435 		pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
436 
437 	return pos;
438 }
439 EXPORT_SYMBOL_GPL(pci_find_ht_capability);
440 
441 /**
442  * pci_find_parent_resource - return resource region of parent bus of given region
443  * @dev: PCI device structure contains resources to be searched
444  * @res: child resource record for which parent is sought
445  *
446  *  For given resource region of given device, return the resource
447  *  region of parent bus the given region is contained in.
448  */
449 struct resource *pci_find_parent_resource(const struct pci_dev *dev,
450 					  struct resource *res)
451 {
452 	const struct pci_bus *bus = dev->bus;
453 	struct resource *r;
454 	int i;
455 
456 	pci_bus_for_each_resource(bus, r, i) {
457 		if (!r)
458 			continue;
459 		if (resource_contains(r, res)) {
460 
461 			/*
462 			 * If the window is prefetchable but the BAR is
463 			 * not, the allocator made a mistake.
464 			 */
465 			if (r->flags & IORESOURCE_PREFETCH &&
466 			    !(res->flags & IORESOURCE_PREFETCH))
467 				return NULL;
468 
469 			/*
470 			 * If we're below a transparent bridge, there may
471 			 * be both a positively-decoded aperture and a
472 			 * subtractively-decoded region that contain the BAR.
473 			 * We want the positively-decoded one, so this depends
474 			 * on pci_bus_for_each_resource() giving us those
475 			 * first.
476 			 */
477 			return r;
478 		}
479 	}
480 	return NULL;
481 }
482 EXPORT_SYMBOL(pci_find_parent_resource);
483 
484 /**
485  * pci_find_resource - Return matching PCI device resource
486  * @dev: PCI device to query
487  * @res: Resource to look for
488  *
489  * Goes over standard PCI resources (BARs) and checks if the given resource
490  * is partially or fully contained in any of them. In that case the
491  * matching resource is returned, %NULL otherwise.
492  */
493 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
494 {
495 	int i;
496 
497 	for (i = 0; i < PCI_ROM_RESOURCE; i++) {
498 		struct resource *r = &dev->resource[i];
499 
500 		if (r->start && resource_contains(r, res))
501 			return r;
502 	}
503 
504 	return NULL;
505 }
506 EXPORT_SYMBOL(pci_find_resource);
507 
508 /**
509  * pci_find_pcie_root_port - return PCIe Root Port
510  * @dev: PCI device to query
511  *
512  * Traverse up the parent chain and return the PCIe Root Port PCI Device
513  * for a given PCI Device.
514  */
515 struct pci_dev *pci_find_pcie_root_port(struct pci_dev *dev)
516 {
517 	struct pci_dev *bridge, *highest_pcie_bridge = NULL;
518 
519 	bridge = pci_upstream_bridge(dev);
520 	while (bridge && pci_is_pcie(bridge)) {
521 		highest_pcie_bridge = bridge;
522 		bridge = pci_upstream_bridge(bridge);
523 	}
524 
525 	if (highest_pcie_bridge &&
526 	    pci_pcie_type(highest_pcie_bridge) == PCI_EXP_TYPE_ROOT_PORT)
527 		return highest_pcie_bridge;
528 
529 	return NULL;
530 }
531 EXPORT_SYMBOL(pci_find_pcie_root_port);
532 
533 /**
534  * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
535  * @dev: the PCI device to operate on
536  * @pos: config space offset of status word
537  * @mask: mask of bit(s) to care about in status word
538  *
539  * Return 1 when mask bit(s) in status word clear, 0 otherwise.
540  */
541 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
542 {
543 	int i;
544 
545 	/* Wait for Transaction Pending bit clean */
546 	for (i = 0; i < 4; i++) {
547 		u16 status;
548 		if (i)
549 			msleep((1 << (i - 1)) * 100);
550 
551 		pci_read_config_word(dev, pos, &status);
552 		if (!(status & mask))
553 			return 1;
554 	}
555 
556 	return 0;
557 }
558 
559 /**
560  * pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
561  * @dev: PCI device to have its BARs restored
562  *
563  * Restore the BAR values for a given device, so as to make it
564  * accessible by its driver.
565  */
566 static void pci_restore_bars(struct pci_dev *dev)
567 {
568 	int i;
569 
570 	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
571 		pci_update_resource(dev, i);
572 }
573 
574 static const struct pci_platform_pm_ops *pci_platform_pm;
575 
576 int pci_set_platform_pm(const struct pci_platform_pm_ops *ops)
577 {
578 	if (!ops->is_manageable || !ops->set_state  || !ops->get_state ||
579 	    !ops->choose_state  || !ops->set_wakeup || !ops->need_resume)
580 		return -EINVAL;
581 	pci_platform_pm = ops;
582 	return 0;
583 }
584 
585 static inline bool platform_pci_power_manageable(struct pci_dev *dev)
586 {
587 	return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false;
588 }
589 
590 static inline int platform_pci_set_power_state(struct pci_dev *dev,
591 					       pci_power_t t)
592 {
593 	return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS;
594 }
595 
596 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
597 {
598 	return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN;
599 }
600 
601 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
602 {
603 	return pci_platform_pm ?
604 			pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR;
605 }
606 
607 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
608 {
609 	return pci_platform_pm ?
610 			pci_platform_pm->set_wakeup(dev, enable) : -ENODEV;
611 }
612 
613 static inline bool platform_pci_need_resume(struct pci_dev *dev)
614 {
615 	return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false;
616 }
617 
618 /**
619  * pci_raw_set_power_state - Use PCI PM registers to set the power state of
620  *                           given PCI device
621  * @dev: PCI device to handle.
622  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
623  *
624  * RETURN VALUE:
625  * -EINVAL if the requested state is invalid.
626  * -EIO if device does not support PCI PM or its PM capabilities register has a
627  * wrong version, or device doesn't support the requested state.
628  * 0 if device already is in the requested state.
629  * 0 if device's power state has been successfully changed.
630  */
631 static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state)
632 {
633 	u16 pmcsr;
634 	bool need_restore = false;
635 
636 	/* Check if we're already there */
637 	if (dev->current_state == state)
638 		return 0;
639 
640 	if (!dev->pm_cap)
641 		return -EIO;
642 
643 	if (state < PCI_D0 || state > PCI_D3hot)
644 		return -EINVAL;
645 
646 	/* Validate current state:
647 	 * Can enter D0 from any state, but if we can only go deeper
648 	 * to sleep if we're already in a low power state
649 	 */
650 	if (state != PCI_D0 && dev->current_state <= PCI_D3cold
651 	    && dev->current_state > state) {
652 		dev_err(&dev->dev, "invalid power transition (from state %d to %d)\n",
653 			dev->current_state, state);
654 		return -EINVAL;
655 	}
656 
657 	/* check if this device supports the desired state */
658 	if ((state == PCI_D1 && !dev->d1_support)
659 	   || (state == PCI_D2 && !dev->d2_support))
660 		return -EIO;
661 
662 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
663 
664 	/* If we're (effectively) in D3, force entire word to 0.
665 	 * This doesn't affect PME_Status, disables PME_En, and
666 	 * sets PowerState to 0.
667 	 */
668 	switch (dev->current_state) {
669 	case PCI_D0:
670 	case PCI_D1:
671 	case PCI_D2:
672 		pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
673 		pmcsr |= state;
674 		break;
675 	case PCI_D3hot:
676 	case PCI_D3cold:
677 	case PCI_UNKNOWN: /* Boot-up */
678 		if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot
679 		 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET))
680 			need_restore = true;
681 		/* Fall-through: force to D0 */
682 	default:
683 		pmcsr = 0;
684 		break;
685 	}
686 
687 	/* enter specified state */
688 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
689 
690 	/* Mandatory power management transition delays */
691 	/* see PCI PM 1.1 5.6.1 table 18 */
692 	if (state == PCI_D3hot || dev->current_state == PCI_D3hot)
693 		pci_dev_d3_sleep(dev);
694 	else if (state == PCI_D2 || dev->current_state == PCI_D2)
695 		udelay(PCI_PM_D2_DELAY);
696 
697 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
698 	dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
699 	if (dev->current_state != state && printk_ratelimit())
700 		dev_info(&dev->dev, "Refused to change power state, currently in D%d\n",
701 			 dev->current_state);
702 
703 	/*
704 	 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
705 	 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
706 	 * from D3hot to D0 _may_ perform an internal reset, thereby
707 	 * going to "D0 Uninitialized" rather than "D0 Initialized".
708 	 * For example, at least some versions of the 3c905B and the
709 	 * 3c556B exhibit this behaviour.
710 	 *
711 	 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave
712 	 * devices in a D3hot state at boot.  Consequently, we need to
713 	 * restore at least the BARs so that the device will be
714 	 * accessible to its driver.
715 	 */
716 	if (need_restore)
717 		pci_restore_bars(dev);
718 
719 	if (dev->bus->self)
720 		pcie_aspm_pm_state_change(dev->bus->self);
721 
722 	return 0;
723 }
724 
725 /**
726  * pci_update_current_state - Read power state of given device and cache it
727  * @dev: PCI device to handle.
728  * @state: State to cache in case the device doesn't have the PM capability
729  *
730  * The power state is read from the PMCSR register, which however is
731  * inaccessible in D3cold.  The platform firmware is therefore queried first
732  * to detect accessibility of the register.  In case the platform firmware
733  * reports an incorrect state or the device isn't power manageable by the
734  * platform at all, we try to detect D3cold by testing accessibility of the
735  * vendor ID in config space.
736  */
737 void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
738 {
739 	if (platform_pci_get_power_state(dev) == PCI_D3cold ||
740 	    !pci_device_is_present(dev)) {
741 		dev->current_state = PCI_D3cold;
742 	} else if (dev->pm_cap) {
743 		u16 pmcsr;
744 
745 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
746 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
747 	} else {
748 		dev->current_state = state;
749 	}
750 }
751 
752 /**
753  * pci_power_up - Put the given device into D0 forcibly
754  * @dev: PCI device to power up
755  */
756 void pci_power_up(struct pci_dev *dev)
757 {
758 	if (platform_pci_power_manageable(dev))
759 		platform_pci_set_power_state(dev, PCI_D0);
760 
761 	pci_raw_set_power_state(dev, PCI_D0);
762 	pci_update_current_state(dev, PCI_D0);
763 }
764 
765 /**
766  * pci_platform_power_transition - Use platform to change device power state
767  * @dev: PCI device to handle.
768  * @state: State to put the device into.
769  */
770 static int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
771 {
772 	int error;
773 
774 	if (platform_pci_power_manageable(dev)) {
775 		error = platform_pci_set_power_state(dev, state);
776 		if (!error)
777 			pci_update_current_state(dev, state);
778 	} else
779 		error = -ENODEV;
780 
781 	if (error && !dev->pm_cap) /* Fall back to PCI_D0 */
782 		dev->current_state = PCI_D0;
783 
784 	return error;
785 }
786 
787 /**
788  * pci_wakeup - Wake up a PCI device
789  * @pci_dev: Device to handle.
790  * @ign: ignored parameter
791  */
792 static int pci_wakeup(struct pci_dev *pci_dev, void *ign)
793 {
794 	pci_wakeup_event(pci_dev);
795 	pm_request_resume(&pci_dev->dev);
796 	return 0;
797 }
798 
799 /**
800  * pci_wakeup_bus - Walk given bus and wake up devices on it
801  * @bus: Top bus of the subtree to walk.
802  */
803 static void pci_wakeup_bus(struct pci_bus *bus)
804 {
805 	if (bus)
806 		pci_walk_bus(bus, pci_wakeup, NULL);
807 }
808 
809 /**
810  * __pci_start_power_transition - Start power transition of a PCI device
811  * @dev: PCI device to handle.
812  * @state: State to put the device into.
813  */
814 static void __pci_start_power_transition(struct pci_dev *dev, pci_power_t state)
815 {
816 	if (state == PCI_D0) {
817 		pci_platform_power_transition(dev, PCI_D0);
818 		/*
819 		 * Mandatory power management transition delays, see
820 		 * PCI Express Base Specification Revision 2.0 Section
821 		 * 6.6.1: Conventional Reset.  Do not delay for
822 		 * devices powered on/off by corresponding bridge,
823 		 * because have already delayed for the bridge.
824 		 */
825 		if (dev->runtime_d3cold) {
826 			if (dev->d3cold_delay)
827 				msleep(dev->d3cold_delay);
828 			/*
829 			 * When powering on a bridge from D3cold, the
830 			 * whole hierarchy may be powered on into
831 			 * D0uninitialized state, resume them to give
832 			 * them a chance to suspend again
833 			 */
834 			pci_wakeup_bus(dev->subordinate);
835 		}
836 	}
837 }
838 
839 /**
840  * __pci_dev_set_current_state - Set current state of a PCI device
841  * @dev: Device to handle
842  * @data: pointer to state to be set
843  */
844 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
845 {
846 	pci_power_t state = *(pci_power_t *)data;
847 
848 	dev->current_state = state;
849 	return 0;
850 }
851 
852 /**
853  * __pci_bus_set_current_state - Walk given bus and set current state of devices
854  * @bus: Top bus of the subtree to walk.
855  * @state: state to be set
856  */
857 static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
858 {
859 	if (bus)
860 		pci_walk_bus(bus, __pci_dev_set_current_state, &state);
861 }
862 
863 /**
864  * __pci_complete_power_transition - Complete power transition of a PCI device
865  * @dev: PCI device to handle.
866  * @state: State to put the device into.
867  *
868  * This function should not be called directly by device drivers.
869  */
870 int __pci_complete_power_transition(struct pci_dev *dev, pci_power_t state)
871 {
872 	int ret;
873 
874 	if (state <= PCI_D0)
875 		return -EINVAL;
876 	ret = pci_platform_power_transition(dev, state);
877 	/* Power off the bridge may power off the whole hierarchy */
878 	if (!ret && state == PCI_D3cold)
879 		__pci_bus_set_current_state(dev->subordinate, PCI_D3cold);
880 	return ret;
881 }
882 EXPORT_SYMBOL_GPL(__pci_complete_power_transition);
883 
884 /**
885  * pci_set_power_state - Set the power state of a PCI device
886  * @dev: PCI device to handle.
887  * @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
888  *
889  * Transition a device to a new power state, using the platform firmware and/or
890  * the device's PCI PM registers.
891  *
892  * RETURN VALUE:
893  * -EINVAL if the requested state is invalid.
894  * -EIO if device does not support PCI PM or its PM capabilities register has a
895  * wrong version, or device doesn't support the requested state.
896  * 0 if device already is in the requested state.
897  * 0 if device's power state has been successfully changed.
898  */
899 int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
900 {
901 	int error;
902 
903 	/* bound the state we're entering */
904 	if (state > PCI_D3cold)
905 		state = PCI_D3cold;
906 	else if (state < PCI_D0)
907 		state = PCI_D0;
908 	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
909 		/*
910 		 * If the device or the parent bridge do not support PCI PM,
911 		 * ignore the request if we're doing anything other than putting
912 		 * it into D0 (which would only happen on boot).
913 		 */
914 		return 0;
915 
916 	/* Check if we're already there */
917 	if (dev->current_state == state)
918 		return 0;
919 
920 	__pci_start_power_transition(dev, state);
921 
922 	/* This device is quirked not to be put into D3, so
923 	   don't put it in D3 */
924 	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
925 		return 0;
926 
927 	/*
928 	 * To put device in D3cold, we put device into D3hot in native
929 	 * way, then put device into D3cold with platform ops
930 	 */
931 	error = pci_raw_set_power_state(dev, state > PCI_D3hot ?
932 					PCI_D3hot : state);
933 
934 	if (!__pci_complete_power_transition(dev, state))
935 		error = 0;
936 
937 	return error;
938 }
939 EXPORT_SYMBOL(pci_set_power_state);
940 
941 /**
942  * pci_choose_state - Choose the power state of a PCI device
943  * @dev: PCI device to be suspended
944  * @state: target sleep state for the whole system. This is the value
945  *	that is passed to suspend() function.
946  *
947  * Returns PCI power state suitable for given device and given system
948  * message.
949  */
950 
951 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
952 {
953 	pci_power_t ret;
954 
955 	if (!dev->pm_cap)
956 		return PCI_D0;
957 
958 	ret = platform_pci_choose_state(dev);
959 	if (ret != PCI_POWER_ERROR)
960 		return ret;
961 
962 	switch (state.event) {
963 	case PM_EVENT_ON:
964 		return PCI_D0;
965 	case PM_EVENT_FREEZE:
966 	case PM_EVENT_PRETHAW:
967 		/* REVISIT both freeze and pre-thaw "should" use D0 */
968 	case PM_EVENT_SUSPEND:
969 	case PM_EVENT_HIBERNATE:
970 		return PCI_D3hot;
971 	default:
972 		dev_info(&dev->dev, "unrecognized suspend event %d\n",
973 			 state.event);
974 		BUG();
975 	}
976 	return PCI_D0;
977 }
978 EXPORT_SYMBOL(pci_choose_state);
979 
980 #define PCI_EXP_SAVE_REGS	7
981 
982 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
983 						       u16 cap, bool extended)
984 {
985 	struct pci_cap_saved_state *tmp;
986 
987 	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
988 		if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
989 			return tmp;
990 	}
991 	return NULL;
992 }
993 
994 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
995 {
996 	return _pci_find_saved_cap(dev, cap, false);
997 }
998 
999 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1000 {
1001 	return _pci_find_saved_cap(dev, cap, true);
1002 }
1003 
1004 static int pci_save_pcie_state(struct pci_dev *dev)
1005 {
1006 	int i = 0;
1007 	struct pci_cap_saved_state *save_state;
1008 	u16 *cap;
1009 
1010 	if (!pci_is_pcie(dev))
1011 		return 0;
1012 
1013 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1014 	if (!save_state) {
1015 		dev_err(&dev->dev, "buffer not found in %s\n", __func__);
1016 		return -ENOMEM;
1017 	}
1018 
1019 	cap = (u16 *)&save_state->cap.data[0];
1020 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]);
1021 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]);
1022 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]);
1023 	pcie_capability_read_word(dev, PCI_EXP_RTCTL,  &cap[i++]);
1024 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]);
1025 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]);
1026 	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]);
1027 
1028 	return 0;
1029 }
1030 
1031 static void pci_restore_pcie_state(struct pci_dev *dev)
1032 {
1033 	int i = 0;
1034 	struct pci_cap_saved_state *save_state;
1035 	u16 *cap;
1036 
1037 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1038 	if (!save_state)
1039 		return;
1040 
1041 	cap = (u16 *)&save_state->cap.data[0];
1042 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]);
1043 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]);
1044 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]);
1045 	pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]);
1046 	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]);
1047 	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]);
1048 	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]);
1049 }
1050 
1051 
1052 static int pci_save_pcix_state(struct pci_dev *dev)
1053 {
1054 	int pos;
1055 	struct pci_cap_saved_state *save_state;
1056 
1057 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1058 	if (!pos)
1059 		return 0;
1060 
1061 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1062 	if (!save_state) {
1063 		dev_err(&dev->dev, "buffer not found in %s\n", __func__);
1064 		return -ENOMEM;
1065 	}
1066 
1067 	pci_read_config_word(dev, pos + PCI_X_CMD,
1068 			     (u16 *)save_state->cap.data);
1069 
1070 	return 0;
1071 }
1072 
1073 static void pci_restore_pcix_state(struct pci_dev *dev)
1074 {
1075 	int i = 0, pos;
1076 	struct pci_cap_saved_state *save_state;
1077 	u16 *cap;
1078 
1079 	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1080 	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1081 	if (!save_state || !pos)
1082 		return;
1083 	cap = (u16 *)&save_state->cap.data[0];
1084 
1085 	pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]);
1086 }
1087 
1088 
1089 /**
1090  * pci_save_state - save the PCI configuration space of a device before suspending
1091  * @dev: - PCI device that we're dealing with
1092  */
1093 int pci_save_state(struct pci_dev *dev)
1094 {
1095 	int i;
1096 	/* XXX: 100% dword access ok here? */
1097 	for (i = 0; i < 16; i++)
1098 		pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]);
1099 	dev->state_saved = true;
1100 
1101 	i = pci_save_pcie_state(dev);
1102 	if (i != 0)
1103 		return i;
1104 
1105 	i = pci_save_pcix_state(dev);
1106 	if (i != 0)
1107 		return i;
1108 
1109 	return pci_save_vc_state(dev);
1110 }
1111 EXPORT_SYMBOL(pci_save_state);
1112 
1113 static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1114 				     u32 saved_val, int retry)
1115 {
1116 	u32 val;
1117 
1118 	pci_read_config_dword(pdev, offset, &val);
1119 	if (val == saved_val)
1120 		return;
1121 
1122 	for (;;) {
1123 		dev_dbg(&pdev->dev, "restoring config space at offset %#x (was %#x, writing %#x)\n",
1124 			offset, val, saved_val);
1125 		pci_write_config_dword(pdev, offset, saved_val);
1126 		if (retry-- <= 0)
1127 			return;
1128 
1129 		pci_read_config_dword(pdev, offset, &val);
1130 		if (val == saved_val)
1131 			return;
1132 
1133 		mdelay(1);
1134 	}
1135 }
1136 
1137 static void pci_restore_config_space_range(struct pci_dev *pdev,
1138 					   int start, int end, int retry)
1139 {
1140 	int index;
1141 
1142 	for (index = end; index >= start; index--)
1143 		pci_restore_config_dword(pdev, 4 * index,
1144 					 pdev->saved_config_space[index],
1145 					 retry);
1146 }
1147 
1148 static void pci_restore_config_space(struct pci_dev *pdev)
1149 {
1150 	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1151 		pci_restore_config_space_range(pdev, 10, 15, 0);
1152 		/* Restore BARs before the command register. */
1153 		pci_restore_config_space_range(pdev, 4, 9, 10);
1154 		pci_restore_config_space_range(pdev, 0, 3, 0);
1155 	} else {
1156 		pci_restore_config_space_range(pdev, 0, 15, 0);
1157 	}
1158 }
1159 
1160 /**
1161  * pci_restore_state - Restore the saved state of a PCI device
1162  * @dev: - PCI device that we're dealing with
1163  */
1164 void pci_restore_state(struct pci_dev *dev)
1165 {
1166 	if (!dev->state_saved)
1167 		return;
1168 
1169 	/* PCI Express register must be restored first */
1170 	pci_restore_pcie_state(dev);
1171 	pci_restore_pasid_state(dev);
1172 	pci_restore_pri_state(dev);
1173 	pci_restore_ats_state(dev);
1174 	pci_restore_vc_state(dev);
1175 
1176 	pci_cleanup_aer_error_status_regs(dev);
1177 
1178 	pci_restore_config_space(dev);
1179 
1180 	pci_restore_pcix_state(dev);
1181 	pci_restore_msi_state(dev);
1182 
1183 	/* Restore ACS and IOV configuration state */
1184 	pci_enable_acs(dev);
1185 	pci_restore_iov_state(dev);
1186 
1187 	dev->state_saved = false;
1188 }
1189 EXPORT_SYMBOL(pci_restore_state);
1190 
1191 struct pci_saved_state {
1192 	u32 config_space[16];
1193 	struct pci_cap_saved_data cap[0];
1194 };
1195 
1196 /**
1197  * pci_store_saved_state - Allocate and return an opaque struct containing
1198  *			   the device saved state.
1199  * @dev: PCI device that we're dealing with
1200  *
1201  * Return NULL if no state or error.
1202  */
1203 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1204 {
1205 	struct pci_saved_state *state;
1206 	struct pci_cap_saved_state *tmp;
1207 	struct pci_cap_saved_data *cap;
1208 	size_t size;
1209 
1210 	if (!dev->state_saved)
1211 		return NULL;
1212 
1213 	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1214 
1215 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1216 		size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1217 
1218 	state = kzalloc(size, GFP_KERNEL);
1219 	if (!state)
1220 		return NULL;
1221 
1222 	memcpy(state->config_space, dev->saved_config_space,
1223 	       sizeof(state->config_space));
1224 
1225 	cap = state->cap;
1226 	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1227 		size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1228 		memcpy(cap, &tmp->cap, len);
1229 		cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1230 	}
1231 	/* Empty cap_save terminates list */
1232 
1233 	return state;
1234 }
1235 EXPORT_SYMBOL_GPL(pci_store_saved_state);
1236 
1237 /**
1238  * pci_load_saved_state - Reload the provided save state into struct pci_dev.
1239  * @dev: PCI device that we're dealing with
1240  * @state: Saved state returned from pci_store_saved_state()
1241  */
1242 int pci_load_saved_state(struct pci_dev *dev,
1243 			 struct pci_saved_state *state)
1244 {
1245 	struct pci_cap_saved_data *cap;
1246 
1247 	dev->state_saved = false;
1248 
1249 	if (!state)
1250 		return 0;
1251 
1252 	memcpy(dev->saved_config_space, state->config_space,
1253 	       sizeof(state->config_space));
1254 
1255 	cap = state->cap;
1256 	while (cap->size) {
1257 		struct pci_cap_saved_state *tmp;
1258 
1259 		tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended);
1260 		if (!tmp || tmp->cap.size != cap->size)
1261 			return -EINVAL;
1262 
1263 		memcpy(tmp->cap.data, cap->data, tmp->cap.size);
1264 		cap = (struct pci_cap_saved_data *)((u8 *)cap +
1265 		       sizeof(struct pci_cap_saved_data) + cap->size);
1266 	}
1267 
1268 	dev->state_saved = true;
1269 	return 0;
1270 }
1271 EXPORT_SYMBOL_GPL(pci_load_saved_state);
1272 
1273 /**
1274  * pci_load_and_free_saved_state - Reload the save state pointed to by state,
1275  *				   and free the memory allocated for it.
1276  * @dev: PCI device that we're dealing with
1277  * @state: Pointer to saved state returned from pci_store_saved_state()
1278  */
1279 int pci_load_and_free_saved_state(struct pci_dev *dev,
1280 				  struct pci_saved_state **state)
1281 {
1282 	int ret = pci_load_saved_state(dev, *state);
1283 	kfree(*state);
1284 	*state = NULL;
1285 	return ret;
1286 }
1287 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
1288 
1289 int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
1290 {
1291 	return pci_enable_resources(dev, bars);
1292 }
1293 
1294 static int do_pci_enable_device(struct pci_dev *dev, int bars)
1295 {
1296 	int err;
1297 	struct pci_dev *bridge;
1298 	u16 cmd;
1299 	u8 pin;
1300 
1301 	err = pci_set_power_state(dev, PCI_D0);
1302 	if (err < 0 && err != -EIO)
1303 		return err;
1304 
1305 	bridge = pci_upstream_bridge(dev);
1306 	if (bridge)
1307 		pcie_aspm_powersave_config_link(bridge);
1308 
1309 	err = pcibios_enable_device(dev, bars);
1310 	if (err < 0)
1311 		return err;
1312 	pci_fixup_device(pci_fixup_enable, dev);
1313 
1314 	if (dev->msi_enabled || dev->msix_enabled)
1315 		return 0;
1316 
1317 	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin);
1318 	if (pin) {
1319 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
1320 		if (cmd & PCI_COMMAND_INTX_DISABLE)
1321 			pci_write_config_word(dev, PCI_COMMAND,
1322 					      cmd & ~PCI_COMMAND_INTX_DISABLE);
1323 	}
1324 
1325 	return 0;
1326 }
1327 
1328 /**
1329  * pci_reenable_device - Resume abandoned device
1330  * @dev: PCI device to be resumed
1331  *
1332  *  Note this function is a backend of pci_default_resume and is not supposed
1333  *  to be called by normal code, write proper resume handler and use it instead.
1334  */
1335 int pci_reenable_device(struct pci_dev *dev)
1336 {
1337 	if (pci_is_enabled(dev))
1338 		return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1);
1339 	return 0;
1340 }
1341 EXPORT_SYMBOL(pci_reenable_device);
1342 
1343 static void pci_enable_bridge(struct pci_dev *dev)
1344 {
1345 	struct pci_dev *bridge;
1346 	int retval;
1347 
1348 	bridge = pci_upstream_bridge(dev);
1349 	if (bridge)
1350 		pci_enable_bridge(bridge);
1351 
1352 	if (pci_is_enabled(dev)) {
1353 		if (!dev->is_busmaster)
1354 			pci_set_master(dev);
1355 		return;
1356 	}
1357 
1358 	retval = pci_enable_device(dev);
1359 	if (retval)
1360 		dev_err(&dev->dev, "Error enabling bridge (%d), continuing\n",
1361 			retval);
1362 	pci_set_master(dev);
1363 }
1364 
1365 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
1366 {
1367 	struct pci_dev *bridge;
1368 	int err;
1369 	int i, bars = 0;
1370 
1371 	/*
1372 	 * Power state could be unknown at this point, either due to a fresh
1373 	 * boot or a device removal call.  So get the current power state
1374 	 * so that things like MSI message writing will behave as expected
1375 	 * (e.g. if the device really is in D0 at enable time).
1376 	 */
1377 	if (dev->pm_cap) {
1378 		u16 pmcsr;
1379 		pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1380 		dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK);
1381 	}
1382 
1383 	if (atomic_inc_return(&dev->enable_cnt) > 1)
1384 		return 0;		/* already enabled */
1385 
1386 	bridge = pci_upstream_bridge(dev);
1387 	if (bridge)
1388 		pci_enable_bridge(bridge);
1389 
1390 	/* only skip sriov related */
1391 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1392 		if (dev->resource[i].flags & flags)
1393 			bars |= (1 << i);
1394 	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
1395 		if (dev->resource[i].flags & flags)
1396 			bars |= (1 << i);
1397 
1398 	err = do_pci_enable_device(dev, bars);
1399 	if (err < 0)
1400 		atomic_dec(&dev->enable_cnt);
1401 	return err;
1402 }
1403 
1404 /**
1405  * pci_enable_device_io - Initialize a device for use with IO space
1406  * @dev: PCI device to be initialized
1407  *
1408  *  Initialize device before it's used by a driver. Ask low-level code
1409  *  to enable I/O resources. Wake up the device if it was suspended.
1410  *  Beware, this function can fail.
1411  */
1412 int pci_enable_device_io(struct pci_dev *dev)
1413 {
1414 	return pci_enable_device_flags(dev, IORESOURCE_IO);
1415 }
1416 EXPORT_SYMBOL(pci_enable_device_io);
1417 
1418 /**
1419  * pci_enable_device_mem - Initialize a device for use with Memory space
1420  * @dev: PCI device to be initialized
1421  *
1422  *  Initialize device before it's used by a driver. Ask low-level code
1423  *  to enable Memory resources. Wake up the device if it was suspended.
1424  *  Beware, this function can fail.
1425  */
1426 int pci_enable_device_mem(struct pci_dev *dev)
1427 {
1428 	return pci_enable_device_flags(dev, IORESOURCE_MEM);
1429 }
1430 EXPORT_SYMBOL(pci_enable_device_mem);
1431 
1432 /**
1433  * pci_enable_device - Initialize device before it's used by a driver.
1434  * @dev: PCI device to be initialized
1435  *
1436  *  Initialize device before it's used by a driver. Ask low-level code
1437  *  to enable I/O and memory. Wake up the device if it was suspended.
1438  *  Beware, this function can fail.
1439  *
1440  *  Note we don't actually enable the device many times if we call
1441  *  this function repeatedly (we just increment the count).
1442  */
1443 int pci_enable_device(struct pci_dev *dev)
1444 {
1445 	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
1446 }
1447 EXPORT_SYMBOL(pci_enable_device);
1448 
1449 /*
1450  * Managed PCI resources.  This manages device on/off, intx/msi/msix
1451  * on/off and BAR regions.  pci_dev itself records msi/msix status, so
1452  * there's no need to track it separately.  pci_devres is initialized
1453  * when a device is enabled using managed PCI device enable interface.
1454  */
1455 struct pci_devres {
1456 	unsigned int enabled:1;
1457 	unsigned int pinned:1;
1458 	unsigned int orig_intx:1;
1459 	unsigned int restore_intx:1;
1460 	u32 region_mask;
1461 };
1462 
1463 static void pcim_release(struct device *gendev, void *res)
1464 {
1465 	struct pci_dev *dev = to_pci_dev(gendev);
1466 	struct pci_devres *this = res;
1467 	int i;
1468 
1469 	if (dev->msi_enabled)
1470 		pci_disable_msi(dev);
1471 	if (dev->msix_enabled)
1472 		pci_disable_msix(dev);
1473 
1474 	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
1475 		if (this->region_mask & (1 << i))
1476 			pci_release_region(dev, i);
1477 
1478 	if (this->restore_intx)
1479 		pci_intx(dev, this->orig_intx);
1480 
1481 	if (this->enabled && !this->pinned)
1482 		pci_disable_device(dev);
1483 }
1484 
1485 static struct pci_devres *get_pci_dr(struct pci_dev *pdev)
1486 {
1487 	struct pci_devres *dr, *new_dr;
1488 
1489 	dr = devres_find(&pdev->dev, pcim_release, NULL, NULL);
1490 	if (dr)
1491 		return dr;
1492 
1493 	new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL);
1494 	if (!new_dr)
1495 		return NULL;
1496 	return devres_get(&pdev->dev, new_dr, NULL, NULL);
1497 }
1498 
1499 static struct pci_devres *find_pci_dr(struct pci_dev *pdev)
1500 {
1501 	if (pci_is_managed(pdev))
1502 		return devres_find(&pdev->dev, pcim_release, NULL, NULL);
1503 	return NULL;
1504 }
1505 
1506 /**
1507  * pcim_enable_device - Managed pci_enable_device()
1508  * @pdev: PCI device to be initialized
1509  *
1510  * Managed pci_enable_device().
1511  */
1512 int pcim_enable_device(struct pci_dev *pdev)
1513 {
1514 	struct pci_devres *dr;
1515 	int rc;
1516 
1517 	dr = get_pci_dr(pdev);
1518 	if (unlikely(!dr))
1519 		return -ENOMEM;
1520 	if (dr->enabled)
1521 		return 0;
1522 
1523 	rc = pci_enable_device(pdev);
1524 	if (!rc) {
1525 		pdev->is_managed = 1;
1526 		dr->enabled = 1;
1527 	}
1528 	return rc;
1529 }
1530 EXPORT_SYMBOL(pcim_enable_device);
1531 
1532 /**
1533  * pcim_pin_device - Pin managed PCI device
1534  * @pdev: PCI device to pin
1535  *
1536  * Pin managed PCI device @pdev.  Pinned device won't be disabled on
1537  * driver detach.  @pdev must have been enabled with
1538  * pcim_enable_device().
1539  */
1540 void pcim_pin_device(struct pci_dev *pdev)
1541 {
1542 	struct pci_devres *dr;
1543 
1544 	dr = find_pci_dr(pdev);
1545 	WARN_ON(!dr || !dr->enabled);
1546 	if (dr)
1547 		dr->pinned = 1;
1548 }
1549 EXPORT_SYMBOL(pcim_pin_device);
1550 
1551 /*
1552  * pcibios_add_device - provide arch specific hooks when adding device dev
1553  * @dev: the PCI device being added
1554  *
1555  * Permits the platform to provide architecture specific functionality when
1556  * devices are added. This is the default implementation. Architecture
1557  * implementations can override this.
1558  */
1559 int __weak pcibios_add_device(struct pci_dev *dev)
1560 {
1561 	return 0;
1562 }
1563 
1564 /**
1565  * pcibios_release_device - provide arch specific hooks when releasing device dev
1566  * @dev: the PCI device being released
1567  *
1568  * Permits the platform to provide architecture specific functionality when
1569  * devices are released. This is the default implementation. Architecture
1570  * implementations can override this.
1571  */
1572 void __weak pcibios_release_device(struct pci_dev *dev) {}
1573 
1574 /**
1575  * pcibios_disable_device - disable arch specific PCI resources for device dev
1576  * @dev: the PCI device to disable
1577  *
1578  * Disables architecture specific PCI resources for the device. This
1579  * is the default implementation. Architecture implementations can
1580  * override this.
1581  */
1582 void __weak pcibios_disable_device(struct pci_dev *dev) {}
1583 
1584 /**
1585  * pcibios_penalize_isa_irq - penalize an ISA IRQ
1586  * @irq: ISA IRQ to penalize
1587  * @active: IRQ active or not
1588  *
1589  * Permits the platform to provide architecture-specific functionality when
1590  * penalizing ISA IRQs. This is the default implementation. Architecture
1591  * implementations can override this.
1592  */
1593 void __weak pcibios_penalize_isa_irq(int irq, int active) {}
1594 
1595 static void do_pci_disable_device(struct pci_dev *dev)
1596 {
1597 	u16 pci_command;
1598 
1599 	pci_read_config_word(dev, PCI_COMMAND, &pci_command);
1600 	if (pci_command & PCI_COMMAND_MASTER) {
1601 		pci_command &= ~PCI_COMMAND_MASTER;
1602 		pci_write_config_word(dev, PCI_COMMAND, pci_command);
1603 	}
1604 
1605 	pcibios_disable_device(dev);
1606 }
1607 
1608 /**
1609  * pci_disable_enabled_device - Disable device without updating enable_cnt
1610  * @dev: PCI device to disable
1611  *
1612  * NOTE: This function is a backend of PCI power management routines and is
1613  * not supposed to be called drivers.
1614  */
1615 void pci_disable_enabled_device(struct pci_dev *dev)
1616 {
1617 	if (pci_is_enabled(dev))
1618 		do_pci_disable_device(dev);
1619 }
1620 
1621 /**
1622  * pci_disable_device - Disable PCI device after use
1623  * @dev: PCI device to be disabled
1624  *
1625  * Signal to the system that the PCI device is not in use by the system
1626  * anymore.  This only involves disabling PCI bus-mastering, if active.
1627  *
1628  * Note we don't actually disable the device until all callers of
1629  * pci_enable_device() have called pci_disable_device().
1630  */
1631 void pci_disable_device(struct pci_dev *dev)
1632 {
1633 	struct pci_devres *dr;
1634 
1635 	dr = find_pci_dr(dev);
1636 	if (dr)
1637 		dr->enabled = 0;
1638 
1639 	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
1640 		      "disabling already-disabled device");
1641 
1642 	if (atomic_dec_return(&dev->enable_cnt) != 0)
1643 		return;
1644 
1645 	do_pci_disable_device(dev);
1646 
1647 	dev->is_busmaster = 0;
1648 }
1649 EXPORT_SYMBOL(pci_disable_device);
1650 
1651 /**
1652  * pcibios_set_pcie_reset_state - set reset state for device dev
1653  * @dev: the PCIe device reset
1654  * @state: Reset state to enter into
1655  *
1656  *
1657  * Sets the PCIe reset state for the device. This is the default
1658  * implementation. Architecture implementations can override this.
1659  */
1660 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
1661 					enum pcie_reset_state state)
1662 {
1663 	return -EINVAL;
1664 }
1665 
1666 /**
1667  * pci_set_pcie_reset_state - set reset state for device dev
1668  * @dev: the PCIe device reset
1669  * @state: Reset state to enter into
1670  *
1671  *
1672  * Sets the PCI reset state for the device.
1673  */
1674 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
1675 {
1676 	return pcibios_set_pcie_reset_state(dev, state);
1677 }
1678 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
1679 
1680 /**
1681  * pci_check_pme_status - Check if given device has generated PME.
1682  * @dev: Device to check.
1683  *
1684  * Check the PME status of the device and if set, clear it and clear PME enable
1685  * (if set).  Return 'true' if PME status and PME enable were both set or
1686  * 'false' otherwise.
1687  */
1688 bool pci_check_pme_status(struct pci_dev *dev)
1689 {
1690 	int pmcsr_pos;
1691 	u16 pmcsr;
1692 	bool ret = false;
1693 
1694 	if (!dev->pm_cap)
1695 		return false;
1696 
1697 	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
1698 	pci_read_config_word(dev, pmcsr_pos, &pmcsr);
1699 	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
1700 		return false;
1701 
1702 	/* Clear PME status. */
1703 	pmcsr |= PCI_PM_CTRL_PME_STATUS;
1704 	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
1705 		/* Disable PME to avoid interrupt flood. */
1706 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
1707 		ret = true;
1708 	}
1709 
1710 	pci_write_config_word(dev, pmcsr_pos, pmcsr);
1711 
1712 	return ret;
1713 }
1714 
1715 /**
1716  * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
1717  * @dev: Device to handle.
1718  * @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
1719  *
1720  * Check if @dev has generated PME and queue a resume request for it in that
1721  * case.
1722  */
1723 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
1724 {
1725 	if (pme_poll_reset && dev->pme_poll)
1726 		dev->pme_poll = false;
1727 
1728 	if (pci_check_pme_status(dev)) {
1729 		pci_wakeup_event(dev);
1730 		pm_request_resume(&dev->dev);
1731 	}
1732 	return 0;
1733 }
1734 
1735 /**
1736  * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
1737  * @bus: Top bus of the subtree to walk.
1738  */
1739 void pci_pme_wakeup_bus(struct pci_bus *bus)
1740 {
1741 	if (bus)
1742 		pci_walk_bus(bus, pci_pme_wakeup, (void *)true);
1743 }
1744 
1745 
1746 /**
1747  * pci_pme_capable - check the capability of PCI device to generate PME#
1748  * @dev: PCI device to handle.
1749  * @state: PCI state from which device will issue PME#.
1750  */
1751 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
1752 {
1753 	if (!dev->pm_cap)
1754 		return false;
1755 
1756 	return !!(dev->pme_support & (1 << state));
1757 }
1758 EXPORT_SYMBOL(pci_pme_capable);
1759 
1760 static void pci_pme_list_scan(struct work_struct *work)
1761 {
1762 	struct pci_pme_device *pme_dev, *n;
1763 
1764 	mutex_lock(&pci_pme_list_mutex);
1765 	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
1766 		if (pme_dev->dev->pme_poll) {
1767 			struct pci_dev *bridge;
1768 
1769 			bridge = pme_dev->dev->bus->self;
1770 			/*
1771 			 * If bridge is in low power state, the
1772 			 * configuration space of subordinate devices
1773 			 * may be not accessible
1774 			 */
1775 			if (bridge && bridge->current_state != PCI_D0)
1776 				continue;
1777 			pci_pme_wakeup(pme_dev->dev, NULL);
1778 		} else {
1779 			list_del(&pme_dev->list);
1780 			kfree(pme_dev);
1781 		}
1782 	}
1783 	if (!list_empty(&pci_pme_list))
1784 		queue_delayed_work(system_freezable_wq, &pci_pme_work,
1785 				   msecs_to_jiffies(PME_TIMEOUT));
1786 	mutex_unlock(&pci_pme_list_mutex);
1787 }
1788 
1789 static void __pci_pme_active(struct pci_dev *dev, bool enable)
1790 {
1791 	u16 pmcsr;
1792 
1793 	if (!dev->pme_support)
1794 		return;
1795 
1796 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1797 	/* Clear PME_Status by writing 1 to it and enable PME# */
1798 	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
1799 	if (!enable)
1800 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
1801 
1802 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
1803 }
1804 
1805 /**
1806  * pci_pme_restore - Restore PME configuration after config space restore.
1807  * @dev: PCI device to update.
1808  */
1809 void pci_pme_restore(struct pci_dev *dev)
1810 {
1811 	u16 pmcsr;
1812 
1813 	if (!dev->pme_support)
1814 		return;
1815 
1816 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr);
1817 	if (dev->wakeup_prepared) {
1818 		pmcsr |= PCI_PM_CTRL_PME_ENABLE;
1819 		pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
1820 	} else {
1821 		pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
1822 		pmcsr |= PCI_PM_CTRL_PME_STATUS;
1823 	}
1824 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr);
1825 }
1826 
1827 /**
1828  * pci_pme_active - enable or disable PCI device's PME# function
1829  * @dev: PCI device to handle.
1830  * @enable: 'true' to enable PME# generation; 'false' to disable it.
1831  *
1832  * The caller must verify that the device is capable of generating PME# before
1833  * calling this function with @enable equal to 'true'.
1834  */
1835 void pci_pme_active(struct pci_dev *dev, bool enable)
1836 {
1837 	__pci_pme_active(dev, enable);
1838 
1839 	/*
1840 	 * PCI (as opposed to PCIe) PME requires that the device have
1841 	 * its PME# line hooked up correctly. Not all hardware vendors
1842 	 * do this, so the PME never gets delivered and the device
1843 	 * remains asleep. The easiest way around this is to
1844 	 * periodically walk the list of suspended devices and check
1845 	 * whether any have their PME flag set. The assumption is that
1846 	 * we'll wake up often enough anyway that this won't be a huge
1847 	 * hit, and the power savings from the devices will still be a
1848 	 * win.
1849 	 *
1850 	 * Although PCIe uses in-band PME message instead of PME# line
1851 	 * to report PME, PME does not work for some PCIe devices in
1852 	 * reality.  For example, there are devices that set their PME
1853 	 * status bits, but don't really bother to send a PME message;
1854 	 * there are PCI Express Root Ports that don't bother to
1855 	 * trigger interrupts when they receive PME messages from the
1856 	 * devices below.  So PME poll is used for PCIe devices too.
1857 	 */
1858 
1859 	if (dev->pme_poll) {
1860 		struct pci_pme_device *pme_dev;
1861 		if (enable) {
1862 			pme_dev = kmalloc(sizeof(struct pci_pme_device),
1863 					  GFP_KERNEL);
1864 			if (!pme_dev) {
1865 				dev_warn(&dev->dev, "can't enable PME#\n");
1866 				return;
1867 			}
1868 			pme_dev->dev = dev;
1869 			mutex_lock(&pci_pme_list_mutex);
1870 			list_add(&pme_dev->list, &pci_pme_list);
1871 			if (list_is_singular(&pci_pme_list))
1872 				queue_delayed_work(system_freezable_wq,
1873 						   &pci_pme_work,
1874 						   msecs_to_jiffies(PME_TIMEOUT));
1875 			mutex_unlock(&pci_pme_list_mutex);
1876 		} else {
1877 			mutex_lock(&pci_pme_list_mutex);
1878 			list_for_each_entry(pme_dev, &pci_pme_list, list) {
1879 				if (pme_dev->dev == dev) {
1880 					list_del(&pme_dev->list);
1881 					kfree(pme_dev);
1882 					break;
1883 				}
1884 			}
1885 			mutex_unlock(&pci_pme_list_mutex);
1886 		}
1887 	}
1888 
1889 	dev_dbg(&dev->dev, "PME# %s\n", enable ? "enabled" : "disabled");
1890 }
1891 EXPORT_SYMBOL(pci_pme_active);
1892 
1893 /**
1894  * pci_enable_wake - enable PCI device as wakeup event source
1895  * @dev: PCI device affected
1896  * @state: PCI state from which device will issue wakeup events
1897  * @enable: True to enable event generation; false to disable
1898  *
1899  * This enables the device as a wakeup event source, or disables it.
1900  * When such events involves platform-specific hooks, those hooks are
1901  * called automatically by this routine.
1902  *
1903  * Devices with legacy power management (no standard PCI PM capabilities)
1904  * always require such platform hooks.
1905  *
1906  * RETURN VALUE:
1907  * 0 is returned on success
1908  * -EINVAL is returned if device is not supposed to wake up the system
1909  * Error code depending on the platform is returned if both the platform and
1910  * the native mechanism fail to enable the generation of wake-up events
1911  */
1912 int pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
1913 {
1914 	int ret = 0;
1915 
1916 	/* Don't do the same thing twice in a row for one device. */
1917 	if (!!enable == !!dev->wakeup_prepared)
1918 		return 0;
1919 
1920 	/*
1921 	 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
1922 	 * Anderson we should be doing PME# wake enable followed by ACPI wake
1923 	 * enable.  To disable wake-up we call the platform first, for symmetry.
1924 	 */
1925 
1926 	if (enable) {
1927 		int error;
1928 
1929 		if (pci_pme_capable(dev, state))
1930 			pci_pme_active(dev, true);
1931 		else
1932 			ret = 1;
1933 		error = platform_pci_set_wakeup(dev, true);
1934 		if (ret)
1935 			ret = error;
1936 		if (!ret)
1937 			dev->wakeup_prepared = true;
1938 	} else {
1939 		platform_pci_set_wakeup(dev, false);
1940 		pci_pme_active(dev, false);
1941 		dev->wakeup_prepared = false;
1942 	}
1943 
1944 	return ret;
1945 }
1946 EXPORT_SYMBOL(pci_enable_wake);
1947 
1948 /**
1949  * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
1950  * @dev: PCI device to prepare
1951  * @enable: True to enable wake-up event generation; false to disable
1952  *
1953  * Many drivers want the device to wake up the system from D3_hot or D3_cold
1954  * and this function allows them to set that up cleanly - pci_enable_wake()
1955  * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
1956  * ordering constraints.
1957  *
1958  * This function only returns error code if the device is not capable of
1959  * generating PME# from both D3_hot and D3_cold, and the platform is unable to
1960  * enable wake-up power for it.
1961  */
1962 int pci_wake_from_d3(struct pci_dev *dev, bool enable)
1963 {
1964 	return pci_pme_capable(dev, PCI_D3cold) ?
1965 			pci_enable_wake(dev, PCI_D3cold, enable) :
1966 			pci_enable_wake(dev, PCI_D3hot, enable);
1967 }
1968 EXPORT_SYMBOL(pci_wake_from_d3);
1969 
1970 /**
1971  * pci_target_state - find an appropriate low power state for a given PCI dev
1972  * @dev: PCI device
1973  * @wakeup: Whether or not wakeup functionality will be enabled for the device.
1974  *
1975  * Use underlying platform code to find a supported low power state for @dev.
1976  * If the platform can't manage @dev, return the deepest state from which it
1977  * can generate wake events, based on any available PME info.
1978  */
1979 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
1980 {
1981 	pci_power_t target_state = PCI_D3hot;
1982 
1983 	if (platform_pci_power_manageable(dev)) {
1984 		/*
1985 		 * Call the platform to choose the target state of the device
1986 		 * and enable wake-up from this state if supported.
1987 		 */
1988 		pci_power_t state = platform_pci_choose_state(dev);
1989 
1990 		switch (state) {
1991 		case PCI_POWER_ERROR:
1992 		case PCI_UNKNOWN:
1993 			break;
1994 		case PCI_D1:
1995 		case PCI_D2:
1996 			if (pci_no_d1d2(dev))
1997 				break;
1998 		default:
1999 			target_state = state;
2000 		}
2001 
2002 		return target_state;
2003 	}
2004 
2005 	if (!dev->pm_cap)
2006 		target_state = PCI_D0;
2007 
2008 	/*
2009 	 * If the device is in D3cold even though it's not power-manageable by
2010 	 * the platform, it may have been powered down by non-standard means.
2011 	 * Best to let it slumber.
2012 	 */
2013 	if (dev->current_state == PCI_D3cold)
2014 		target_state = PCI_D3cold;
2015 
2016 	if (wakeup) {
2017 		/*
2018 		 * Find the deepest state from which the device can generate
2019 		 * wake-up events, make it the target state and enable device
2020 		 * to generate PME#.
2021 		 */
2022 		if (dev->pme_support) {
2023 			while (target_state
2024 			      && !(dev->pme_support & (1 << target_state)))
2025 				target_state--;
2026 		}
2027 	}
2028 
2029 	return target_state;
2030 }
2031 
2032 /**
2033  * pci_prepare_to_sleep - prepare PCI device for system-wide transition into a sleep state
2034  * @dev: Device to handle.
2035  *
2036  * Choose the power state appropriate for the device depending on whether
2037  * it can wake up the system and/or is power manageable by the platform
2038  * (PCI_D3hot is the default) and put the device into that state.
2039  */
2040 int pci_prepare_to_sleep(struct pci_dev *dev)
2041 {
2042 	bool wakeup = device_may_wakeup(&dev->dev);
2043 	pci_power_t target_state = pci_target_state(dev, wakeup);
2044 	int error;
2045 
2046 	if (target_state == PCI_POWER_ERROR)
2047 		return -EIO;
2048 
2049 	pci_enable_wake(dev, target_state, wakeup);
2050 
2051 	error = pci_set_power_state(dev, target_state);
2052 
2053 	if (error)
2054 		pci_enable_wake(dev, target_state, false);
2055 
2056 	return error;
2057 }
2058 EXPORT_SYMBOL(pci_prepare_to_sleep);
2059 
2060 /**
2061  * pci_back_from_sleep - turn PCI device on during system-wide transition into working state
2062  * @dev: Device to handle.
2063  *
2064  * Disable device's system wake-up capability and put it into D0.
2065  */
2066 int pci_back_from_sleep(struct pci_dev *dev)
2067 {
2068 	pci_enable_wake(dev, PCI_D0, false);
2069 	return pci_set_power_state(dev, PCI_D0);
2070 }
2071 EXPORT_SYMBOL(pci_back_from_sleep);
2072 
2073 /**
2074  * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2075  * @dev: PCI device being suspended.
2076  *
2077  * Prepare @dev to generate wake-up events at run time and put it into a low
2078  * power state.
2079  */
2080 int pci_finish_runtime_suspend(struct pci_dev *dev)
2081 {
2082 	pci_power_t target_state;
2083 	int error;
2084 
2085 	target_state = pci_target_state(dev, device_can_wakeup(&dev->dev));
2086 	if (target_state == PCI_POWER_ERROR)
2087 		return -EIO;
2088 
2089 	dev->runtime_d3cold = target_state == PCI_D3cold;
2090 
2091 	pci_enable_wake(dev, target_state, pci_dev_run_wake(dev));
2092 
2093 	error = pci_set_power_state(dev, target_state);
2094 
2095 	if (error) {
2096 		pci_enable_wake(dev, target_state, false);
2097 		dev->runtime_d3cold = false;
2098 	}
2099 
2100 	return error;
2101 }
2102 
2103 /**
2104  * pci_dev_run_wake - Check if device can generate run-time wake-up events.
2105  * @dev: Device to check.
2106  *
2107  * Return true if the device itself is capable of generating wake-up events
2108  * (through the platform or using the native PCIe PME) or if the device supports
2109  * PME and one of its upstream bridges can generate wake-up events.
2110  */
2111 bool pci_dev_run_wake(struct pci_dev *dev)
2112 {
2113 	struct pci_bus *bus = dev->bus;
2114 
2115 	if (device_can_wakeup(&dev->dev))
2116 		return true;
2117 
2118 	if (!dev->pme_support)
2119 		return false;
2120 
2121 	/* PME-capable in principle, but not from the target power state */
2122 	if (!pci_pme_capable(dev, pci_target_state(dev, false)))
2123 		return false;
2124 
2125 	while (bus->parent) {
2126 		struct pci_dev *bridge = bus->self;
2127 
2128 		if (device_can_wakeup(&bridge->dev))
2129 			return true;
2130 
2131 		bus = bus->parent;
2132 	}
2133 
2134 	/* We have reached the root bus. */
2135 	if (bus->bridge)
2136 		return device_can_wakeup(bus->bridge);
2137 
2138 	return false;
2139 }
2140 EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2141 
2142 /**
2143  * pci_dev_keep_suspended - Check if the device can stay in the suspended state.
2144  * @pci_dev: Device to check.
2145  *
2146  * Return 'true' if the device is runtime-suspended, it doesn't have to be
2147  * reconfigured due to wakeup settings difference between system and runtime
2148  * suspend and the current power state of it is suitable for the upcoming
2149  * (system) transition.
2150  *
2151  * If the device is not configured for system wakeup, disable PME for it before
2152  * returning 'true' to prevent it from waking up the system unnecessarily.
2153  */
2154 bool pci_dev_keep_suspended(struct pci_dev *pci_dev)
2155 {
2156 	struct device *dev = &pci_dev->dev;
2157 	bool wakeup = device_may_wakeup(dev);
2158 
2159 	if (!pm_runtime_suspended(dev)
2160 	    || pci_target_state(pci_dev, wakeup) != pci_dev->current_state
2161 	    || platform_pci_need_resume(pci_dev)
2162 	    || (pci_dev->dev_flags & PCI_DEV_FLAGS_NEEDS_RESUME))
2163 		return false;
2164 
2165 	/*
2166 	 * At this point the device is good to go unless it's been configured
2167 	 * to generate PME at the runtime suspend time, but it is not supposed
2168 	 * to wake up the system.  In that case, simply disable PME for it
2169 	 * (it will have to be re-enabled on exit from system resume).
2170 	 *
2171 	 * If the device's power state is D3cold and the platform check above
2172 	 * hasn't triggered, the device's configuration is suitable and we don't
2173 	 * need to manipulate it at all.
2174 	 */
2175 	spin_lock_irq(&dev->power.lock);
2176 
2177 	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold &&
2178 	    !wakeup)
2179 		__pci_pme_active(pci_dev, false);
2180 
2181 	spin_unlock_irq(&dev->power.lock);
2182 	return true;
2183 }
2184 
2185 /**
2186  * pci_dev_complete_resume - Finalize resume from system sleep for a device.
2187  * @pci_dev: Device to handle.
2188  *
2189  * If the device is runtime suspended and wakeup-capable, enable PME for it as
2190  * it might have been disabled during the prepare phase of system suspend if
2191  * the device was not configured for system wakeup.
2192  */
2193 void pci_dev_complete_resume(struct pci_dev *pci_dev)
2194 {
2195 	struct device *dev = &pci_dev->dev;
2196 
2197 	if (!pci_dev_run_wake(pci_dev))
2198 		return;
2199 
2200 	spin_lock_irq(&dev->power.lock);
2201 
2202 	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2203 		__pci_pme_active(pci_dev, true);
2204 
2205 	spin_unlock_irq(&dev->power.lock);
2206 }
2207 
2208 void pci_config_pm_runtime_get(struct pci_dev *pdev)
2209 {
2210 	struct device *dev = &pdev->dev;
2211 	struct device *parent = dev->parent;
2212 
2213 	if (parent)
2214 		pm_runtime_get_sync(parent);
2215 	pm_runtime_get_noresume(dev);
2216 	/*
2217 	 * pdev->current_state is set to PCI_D3cold during suspending,
2218 	 * so wait until suspending completes
2219 	 */
2220 	pm_runtime_barrier(dev);
2221 	/*
2222 	 * Only need to resume devices in D3cold, because config
2223 	 * registers are still accessible for devices suspended but
2224 	 * not in D3cold.
2225 	 */
2226 	if (pdev->current_state == PCI_D3cold)
2227 		pm_runtime_resume(dev);
2228 }
2229 
2230 void pci_config_pm_runtime_put(struct pci_dev *pdev)
2231 {
2232 	struct device *dev = &pdev->dev;
2233 	struct device *parent = dev->parent;
2234 
2235 	pm_runtime_put(dev);
2236 	if (parent)
2237 		pm_runtime_put_sync(parent);
2238 }
2239 
2240 /**
2241  * pci_bridge_d3_possible - Is it possible to put the bridge into D3
2242  * @bridge: Bridge to check
2243  *
2244  * This function checks if it is possible to move the bridge to D3.
2245  * Currently we only allow D3 for recent enough PCIe ports.
2246  */
2247 bool pci_bridge_d3_possible(struct pci_dev *bridge)
2248 {
2249 	unsigned int year;
2250 
2251 	if (!pci_is_pcie(bridge))
2252 		return false;
2253 
2254 	switch (pci_pcie_type(bridge)) {
2255 	case PCI_EXP_TYPE_ROOT_PORT:
2256 	case PCI_EXP_TYPE_UPSTREAM:
2257 	case PCI_EXP_TYPE_DOWNSTREAM:
2258 		if (pci_bridge_d3_disable)
2259 			return false;
2260 
2261 		/*
2262 		 * Hotplug interrupts cannot be delivered if the link is down,
2263 		 * so parents of a hotplug port must stay awake. In addition,
2264 		 * hotplug ports handled by firmware in System Management Mode
2265 		 * may not be put into D3 by the OS (Thunderbolt on non-Macs).
2266 		 * For simplicity, disallow in general for now.
2267 		 */
2268 		if (bridge->is_hotplug_bridge)
2269 			return false;
2270 
2271 		if (pci_bridge_d3_force)
2272 			return true;
2273 
2274 		/*
2275 		 * It should be safe to put PCIe ports from 2015 or newer
2276 		 * to D3.
2277 		 */
2278 		if (dmi_get_date(DMI_BIOS_DATE, &year, NULL, NULL) &&
2279 		    year >= 2015) {
2280 			return true;
2281 		}
2282 		break;
2283 	}
2284 
2285 	return false;
2286 }
2287 
2288 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
2289 {
2290 	bool *d3cold_ok = data;
2291 
2292 	if (/* The device needs to be allowed to go D3cold ... */
2293 	    dev->no_d3cold || !dev->d3cold_allowed ||
2294 
2295 	    /* ... and if it is wakeup capable to do so from D3cold. */
2296 	    (device_may_wakeup(&dev->dev) &&
2297 	     !pci_pme_capable(dev, PCI_D3cold)) ||
2298 
2299 	    /* If it is a bridge it must be allowed to go to D3. */
2300 	    !pci_power_manageable(dev))
2301 
2302 		*d3cold_ok = false;
2303 
2304 	return !*d3cold_ok;
2305 }
2306 
2307 /*
2308  * pci_bridge_d3_update - Update bridge D3 capabilities
2309  * @dev: PCI device which is changed
2310  *
2311  * Update upstream bridge PM capabilities accordingly depending on if the
2312  * device PM configuration was changed or the device is being removed.  The
2313  * change is also propagated upstream.
2314  */
2315 void pci_bridge_d3_update(struct pci_dev *dev)
2316 {
2317 	bool remove = !device_is_registered(&dev->dev);
2318 	struct pci_dev *bridge;
2319 	bool d3cold_ok = true;
2320 
2321 	bridge = pci_upstream_bridge(dev);
2322 	if (!bridge || !pci_bridge_d3_possible(bridge))
2323 		return;
2324 
2325 	/*
2326 	 * If D3 is currently allowed for the bridge, removing one of its
2327 	 * children won't change that.
2328 	 */
2329 	if (remove && bridge->bridge_d3)
2330 		return;
2331 
2332 	/*
2333 	 * If D3 is currently allowed for the bridge and a child is added or
2334 	 * changed, disallowance of D3 can only be caused by that child, so
2335 	 * we only need to check that single device, not any of its siblings.
2336 	 *
2337 	 * If D3 is currently not allowed for the bridge, checking the device
2338 	 * first may allow us to skip checking its siblings.
2339 	 */
2340 	if (!remove)
2341 		pci_dev_check_d3cold(dev, &d3cold_ok);
2342 
2343 	/*
2344 	 * If D3 is currently not allowed for the bridge, this may be caused
2345 	 * either by the device being changed/removed or any of its siblings,
2346 	 * so we need to go through all children to find out if one of them
2347 	 * continues to block D3.
2348 	 */
2349 	if (d3cold_ok && !bridge->bridge_d3)
2350 		pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold,
2351 			     &d3cold_ok);
2352 
2353 	if (bridge->bridge_d3 != d3cold_ok) {
2354 		bridge->bridge_d3 = d3cold_ok;
2355 		/* Propagate change to upstream bridges */
2356 		pci_bridge_d3_update(bridge);
2357 	}
2358 }
2359 
2360 /**
2361  * pci_d3cold_enable - Enable D3cold for device
2362  * @dev: PCI device to handle
2363  *
2364  * This function can be used in drivers to enable D3cold from the device
2365  * they handle.  It also updates upstream PCI bridge PM capabilities
2366  * accordingly.
2367  */
2368 void pci_d3cold_enable(struct pci_dev *dev)
2369 {
2370 	if (dev->no_d3cold) {
2371 		dev->no_d3cold = false;
2372 		pci_bridge_d3_update(dev);
2373 	}
2374 }
2375 EXPORT_SYMBOL_GPL(pci_d3cold_enable);
2376 
2377 /**
2378  * pci_d3cold_disable - Disable D3cold for device
2379  * @dev: PCI device to handle
2380  *
2381  * This function can be used in drivers to disable D3cold from the device
2382  * they handle.  It also updates upstream PCI bridge PM capabilities
2383  * accordingly.
2384  */
2385 void pci_d3cold_disable(struct pci_dev *dev)
2386 {
2387 	if (!dev->no_d3cold) {
2388 		dev->no_d3cold = true;
2389 		pci_bridge_d3_update(dev);
2390 	}
2391 }
2392 EXPORT_SYMBOL_GPL(pci_d3cold_disable);
2393 
2394 /**
2395  * pci_pm_init - Initialize PM functions of given PCI device
2396  * @dev: PCI device to handle.
2397  */
2398 void pci_pm_init(struct pci_dev *dev)
2399 {
2400 	int pm;
2401 	u16 pmc;
2402 
2403 	pm_runtime_forbid(&dev->dev);
2404 	pm_runtime_set_active(&dev->dev);
2405 	pm_runtime_enable(&dev->dev);
2406 	device_enable_async_suspend(&dev->dev);
2407 	dev->wakeup_prepared = false;
2408 
2409 	dev->pm_cap = 0;
2410 	dev->pme_support = 0;
2411 
2412 	/* find PCI PM capability in list */
2413 	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
2414 	if (!pm)
2415 		return;
2416 	/* Check device's ability to generate PME# */
2417 	pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc);
2418 
2419 	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
2420 		dev_err(&dev->dev, "unsupported PM cap regs version (%u)\n",
2421 			pmc & PCI_PM_CAP_VER_MASK);
2422 		return;
2423 	}
2424 
2425 	dev->pm_cap = pm;
2426 	dev->d3_delay = PCI_PM_D3_WAIT;
2427 	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
2428 	dev->bridge_d3 = pci_bridge_d3_possible(dev);
2429 	dev->d3cold_allowed = true;
2430 
2431 	dev->d1_support = false;
2432 	dev->d2_support = false;
2433 	if (!pci_no_d1d2(dev)) {
2434 		if (pmc & PCI_PM_CAP_D1)
2435 			dev->d1_support = true;
2436 		if (pmc & PCI_PM_CAP_D2)
2437 			dev->d2_support = true;
2438 
2439 		if (dev->d1_support || dev->d2_support)
2440 			dev_printk(KERN_DEBUG, &dev->dev, "supports%s%s\n",
2441 				   dev->d1_support ? " D1" : "",
2442 				   dev->d2_support ? " D2" : "");
2443 	}
2444 
2445 	pmc &= PCI_PM_CAP_PME_MASK;
2446 	if (pmc) {
2447 		dev_printk(KERN_DEBUG, &dev->dev,
2448 			 "PME# supported from%s%s%s%s%s\n",
2449 			 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
2450 			 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
2451 			 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
2452 			 (pmc & PCI_PM_CAP_PME_D3) ? " D3hot" : "",
2453 			 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
2454 		dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT;
2455 		dev->pme_poll = true;
2456 		/*
2457 		 * Make device's PM flags reflect the wake-up capability, but
2458 		 * let the user space enable it to wake up the system as needed.
2459 		 */
2460 		device_set_wakeup_capable(&dev->dev, true);
2461 		/* Disable the PME# generation functionality */
2462 		pci_pme_active(dev, false);
2463 	}
2464 }
2465 
2466 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
2467 {
2468 	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
2469 
2470 	switch (prop) {
2471 	case PCI_EA_P_MEM:
2472 	case PCI_EA_P_VF_MEM:
2473 		flags |= IORESOURCE_MEM;
2474 		break;
2475 	case PCI_EA_P_MEM_PREFETCH:
2476 	case PCI_EA_P_VF_MEM_PREFETCH:
2477 		flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
2478 		break;
2479 	case PCI_EA_P_IO:
2480 		flags |= IORESOURCE_IO;
2481 		break;
2482 	default:
2483 		return 0;
2484 	}
2485 
2486 	return flags;
2487 }
2488 
2489 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
2490 					    u8 prop)
2491 {
2492 	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
2493 		return &dev->resource[bei];
2494 #ifdef CONFIG_PCI_IOV
2495 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
2496 		 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
2497 		return &dev->resource[PCI_IOV_RESOURCES +
2498 				      bei - PCI_EA_BEI_VF_BAR0];
2499 #endif
2500 	else if (bei == PCI_EA_BEI_ROM)
2501 		return &dev->resource[PCI_ROM_RESOURCE];
2502 	else
2503 		return NULL;
2504 }
2505 
2506 /* Read an Enhanced Allocation (EA) entry */
2507 static int pci_ea_read(struct pci_dev *dev, int offset)
2508 {
2509 	struct resource *res;
2510 	int ent_size, ent_offset = offset;
2511 	resource_size_t start, end;
2512 	unsigned long flags;
2513 	u32 dw0, bei, base, max_offset;
2514 	u8 prop;
2515 	bool support_64 = (sizeof(resource_size_t) >= 8);
2516 
2517 	pci_read_config_dword(dev, ent_offset, &dw0);
2518 	ent_offset += 4;
2519 
2520 	/* Entry size field indicates DWORDs after 1st */
2521 	ent_size = ((dw0 & PCI_EA_ES) + 1) << 2;
2522 
2523 	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
2524 		goto out;
2525 
2526 	bei = (dw0 & PCI_EA_BEI) >> 4;
2527 	prop = (dw0 & PCI_EA_PP) >> 8;
2528 
2529 	/*
2530 	 * If the Property is in the reserved range, try the Secondary
2531 	 * Property instead.
2532 	 */
2533 	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
2534 		prop = (dw0 & PCI_EA_SP) >> 16;
2535 	if (prop > PCI_EA_P_BRIDGE_IO)
2536 		goto out;
2537 
2538 	res = pci_ea_get_resource(dev, bei, prop);
2539 	if (!res) {
2540 		dev_err(&dev->dev, "Unsupported EA entry BEI: %u\n", bei);
2541 		goto out;
2542 	}
2543 
2544 	flags = pci_ea_flags(dev, prop);
2545 	if (!flags) {
2546 		dev_err(&dev->dev, "Unsupported EA properties: %#x\n", prop);
2547 		goto out;
2548 	}
2549 
2550 	/* Read Base */
2551 	pci_read_config_dword(dev, ent_offset, &base);
2552 	start = (base & PCI_EA_FIELD_MASK);
2553 	ent_offset += 4;
2554 
2555 	/* Read MaxOffset */
2556 	pci_read_config_dword(dev, ent_offset, &max_offset);
2557 	ent_offset += 4;
2558 
2559 	/* Read Base MSBs (if 64-bit entry) */
2560 	if (base & PCI_EA_IS_64) {
2561 		u32 base_upper;
2562 
2563 		pci_read_config_dword(dev, ent_offset, &base_upper);
2564 		ent_offset += 4;
2565 
2566 		flags |= IORESOURCE_MEM_64;
2567 
2568 		/* entry starts above 32-bit boundary, can't use */
2569 		if (!support_64 && base_upper)
2570 			goto out;
2571 
2572 		if (support_64)
2573 			start |= ((u64)base_upper << 32);
2574 	}
2575 
2576 	end = start + (max_offset | 0x03);
2577 
2578 	/* Read MaxOffset MSBs (if 64-bit entry) */
2579 	if (max_offset & PCI_EA_IS_64) {
2580 		u32 max_offset_upper;
2581 
2582 		pci_read_config_dword(dev, ent_offset, &max_offset_upper);
2583 		ent_offset += 4;
2584 
2585 		flags |= IORESOURCE_MEM_64;
2586 
2587 		/* entry too big, can't use */
2588 		if (!support_64 && max_offset_upper)
2589 			goto out;
2590 
2591 		if (support_64)
2592 			end += ((u64)max_offset_upper << 32);
2593 	}
2594 
2595 	if (end < start) {
2596 		dev_err(&dev->dev, "EA Entry crosses address boundary\n");
2597 		goto out;
2598 	}
2599 
2600 	if (ent_size != ent_offset - offset) {
2601 		dev_err(&dev->dev,
2602 			"EA Entry Size (%d) does not match length read (%d)\n",
2603 			ent_size, ent_offset - offset);
2604 		goto out;
2605 	}
2606 
2607 	res->name = pci_name(dev);
2608 	res->start = start;
2609 	res->end = end;
2610 	res->flags = flags;
2611 
2612 	if (bei <= PCI_EA_BEI_BAR5)
2613 		dev_printk(KERN_DEBUG, &dev->dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
2614 			   bei, res, prop);
2615 	else if (bei == PCI_EA_BEI_ROM)
2616 		dev_printk(KERN_DEBUG, &dev->dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n",
2617 			   res, prop);
2618 	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
2619 		dev_printk(KERN_DEBUG, &dev->dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
2620 			   bei - PCI_EA_BEI_VF_BAR0, res, prop);
2621 	else
2622 		dev_printk(KERN_DEBUG, &dev->dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n",
2623 			   bei, res, prop);
2624 
2625 out:
2626 	return offset + ent_size;
2627 }
2628 
2629 /* Enhanced Allocation Initialization */
2630 void pci_ea_init(struct pci_dev *dev)
2631 {
2632 	int ea;
2633 	u8 num_ent;
2634 	int offset;
2635 	int i;
2636 
2637 	/* find PCI EA capability in list */
2638 	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
2639 	if (!ea)
2640 		return;
2641 
2642 	/* determine the number of entries */
2643 	pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT,
2644 					&num_ent);
2645 	num_ent &= PCI_EA_NUM_ENT_MASK;
2646 
2647 	offset = ea + PCI_EA_FIRST_ENT;
2648 
2649 	/* Skip DWORD 2 for type 1 functions */
2650 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
2651 		offset += 4;
2652 
2653 	/* parse each EA entry */
2654 	for (i = 0; i < num_ent; ++i)
2655 		offset = pci_ea_read(dev, offset);
2656 }
2657 
2658 static void pci_add_saved_cap(struct pci_dev *pci_dev,
2659 	struct pci_cap_saved_state *new_cap)
2660 {
2661 	hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space);
2662 }
2663 
2664 /**
2665  * _pci_add_cap_save_buffer - allocate buffer for saving given
2666  *                            capability registers
2667  * @dev: the PCI device
2668  * @cap: the capability to allocate the buffer for
2669  * @extended: Standard or Extended capability ID
2670  * @size: requested size of the buffer
2671  */
2672 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
2673 				    bool extended, unsigned int size)
2674 {
2675 	int pos;
2676 	struct pci_cap_saved_state *save_state;
2677 
2678 	if (extended)
2679 		pos = pci_find_ext_capability(dev, cap);
2680 	else
2681 		pos = pci_find_capability(dev, cap);
2682 
2683 	if (!pos)
2684 		return 0;
2685 
2686 	save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL);
2687 	if (!save_state)
2688 		return -ENOMEM;
2689 
2690 	save_state->cap.cap_nr = cap;
2691 	save_state->cap.cap_extended = extended;
2692 	save_state->cap.size = size;
2693 	pci_add_saved_cap(dev, save_state);
2694 
2695 	return 0;
2696 }
2697 
2698 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
2699 {
2700 	return _pci_add_cap_save_buffer(dev, cap, false, size);
2701 }
2702 
2703 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
2704 {
2705 	return _pci_add_cap_save_buffer(dev, cap, true, size);
2706 }
2707 
2708 /**
2709  * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
2710  * @dev: the PCI device
2711  */
2712 void pci_allocate_cap_save_buffers(struct pci_dev *dev)
2713 {
2714 	int error;
2715 
2716 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
2717 					PCI_EXP_SAVE_REGS * sizeof(u16));
2718 	if (error)
2719 		dev_err(&dev->dev,
2720 			"unable to preallocate PCI Express save buffer\n");
2721 
2722 	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16));
2723 	if (error)
2724 		dev_err(&dev->dev,
2725 			"unable to preallocate PCI-X save buffer\n");
2726 
2727 	pci_allocate_vc_save_buffers(dev);
2728 }
2729 
2730 void pci_free_cap_save_buffers(struct pci_dev *dev)
2731 {
2732 	struct pci_cap_saved_state *tmp;
2733 	struct hlist_node *n;
2734 
2735 	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
2736 		kfree(tmp);
2737 }
2738 
2739 /**
2740  * pci_configure_ari - enable or disable ARI forwarding
2741  * @dev: the PCI device
2742  *
2743  * If @dev and its upstream bridge both support ARI, enable ARI in the
2744  * bridge.  Otherwise, disable ARI in the bridge.
2745  */
2746 void pci_configure_ari(struct pci_dev *dev)
2747 {
2748 	u32 cap;
2749 	struct pci_dev *bridge;
2750 
2751 	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
2752 		return;
2753 
2754 	bridge = dev->bus->self;
2755 	if (!bridge)
2756 		return;
2757 
2758 	pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap);
2759 	if (!(cap & PCI_EXP_DEVCAP2_ARI))
2760 		return;
2761 
2762 	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
2763 		pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2,
2764 					 PCI_EXP_DEVCTL2_ARI);
2765 		bridge->ari_enabled = 1;
2766 	} else {
2767 		pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2,
2768 					   PCI_EXP_DEVCTL2_ARI);
2769 		bridge->ari_enabled = 0;
2770 	}
2771 }
2772 
2773 static int pci_acs_enable;
2774 
2775 /**
2776  * pci_request_acs - ask for ACS to be enabled if supported
2777  */
2778 void pci_request_acs(void)
2779 {
2780 	pci_acs_enable = 1;
2781 }
2782 
2783 /**
2784  * pci_std_enable_acs - enable ACS on devices using standard ACS capabilites
2785  * @dev: the PCI device
2786  */
2787 static void pci_std_enable_acs(struct pci_dev *dev)
2788 {
2789 	int pos;
2790 	u16 cap;
2791 	u16 ctrl;
2792 
2793 	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
2794 	if (!pos)
2795 		return;
2796 
2797 	pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap);
2798 	pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl);
2799 
2800 	/* Source Validation */
2801 	ctrl |= (cap & PCI_ACS_SV);
2802 
2803 	/* P2P Request Redirect */
2804 	ctrl |= (cap & PCI_ACS_RR);
2805 
2806 	/* P2P Completion Redirect */
2807 	ctrl |= (cap & PCI_ACS_CR);
2808 
2809 	/* Upstream Forwarding */
2810 	ctrl |= (cap & PCI_ACS_UF);
2811 
2812 	pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl);
2813 }
2814 
2815 /**
2816  * pci_enable_acs - enable ACS if hardware support it
2817  * @dev: the PCI device
2818  */
2819 void pci_enable_acs(struct pci_dev *dev)
2820 {
2821 	if (!pci_acs_enable)
2822 		return;
2823 
2824 	if (!pci_dev_specific_enable_acs(dev))
2825 		return;
2826 
2827 	pci_std_enable_acs(dev);
2828 }
2829 
2830 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
2831 {
2832 	int pos;
2833 	u16 cap, ctrl;
2834 
2835 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ACS);
2836 	if (!pos)
2837 		return false;
2838 
2839 	/*
2840 	 * Except for egress control, capabilities are either required
2841 	 * or only required if controllable.  Features missing from the
2842 	 * capability field can therefore be assumed as hard-wired enabled.
2843 	 */
2844 	pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap);
2845 	acs_flags &= (cap | PCI_ACS_EC);
2846 
2847 	pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl);
2848 	return (ctrl & acs_flags) == acs_flags;
2849 }
2850 
2851 /**
2852  * pci_acs_enabled - test ACS against required flags for a given device
2853  * @pdev: device to test
2854  * @acs_flags: required PCI ACS flags
2855  *
2856  * Return true if the device supports the provided flags.  Automatically
2857  * filters out flags that are not implemented on multifunction devices.
2858  *
2859  * Note that this interface checks the effective ACS capabilities of the
2860  * device rather than the actual capabilities.  For instance, most single
2861  * function endpoints are not required to support ACS because they have no
2862  * opportunity for peer-to-peer access.  We therefore return 'true'
2863  * regardless of whether the device exposes an ACS capability.  This makes
2864  * it much easier for callers of this function to ignore the actual type
2865  * or topology of the device when testing ACS support.
2866  */
2867 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
2868 {
2869 	int ret;
2870 
2871 	ret = pci_dev_specific_acs_enabled(pdev, acs_flags);
2872 	if (ret >= 0)
2873 		return ret > 0;
2874 
2875 	/*
2876 	 * Conventional PCI and PCI-X devices never support ACS, either
2877 	 * effectively or actually.  The shared bus topology implies that
2878 	 * any device on the bus can receive or snoop DMA.
2879 	 */
2880 	if (!pci_is_pcie(pdev))
2881 		return false;
2882 
2883 	switch (pci_pcie_type(pdev)) {
2884 	/*
2885 	 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
2886 	 * but since their primary interface is PCI/X, we conservatively
2887 	 * handle them as we would a non-PCIe device.
2888 	 */
2889 	case PCI_EXP_TYPE_PCIE_BRIDGE:
2890 	/*
2891 	 * PCIe 3.0, 6.12.1 excludes ACS on these devices.  "ACS is never
2892 	 * applicable... must never implement an ACS Extended Capability...".
2893 	 * This seems arbitrary, but we take a conservative interpretation
2894 	 * of this statement.
2895 	 */
2896 	case PCI_EXP_TYPE_PCI_BRIDGE:
2897 	case PCI_EXP_TYPE_RC_EC:
2898 		return false;
2899 	/*
2900 	 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
2901 	 * implement ACS in order to indicate their peer-to-peer capabilities,
2902 	 * regardless of whether they are single- or multi-function devices.
2903 	 */
2904 	case PCI_EXP_TYPE_DOWNSTREAM:
2905 	case PCI_EXP_TYPE_ROOT_PORT:
2906 		return pci_acs_flags_enabled(pdev, acs_flags);
2907 	/*
2908 	 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
2909 	 * implemented by the remaining PCIe types to indicate peer-to-peer
2910 	 * capabilities, but only when they are part of a multifunction
2911 	 * device.  The footnote for section 6.12 indicates the specific
2912 	 * PCIe types included here.
2913 	 */
2914 	case PCI_EXP_TYPE_ENDPOINT:
2915 	case PCI_EXP_TYPE_UPSTREAM:
2916 	case PCI_EXP_TYPE_LEG_END:
2917 	case PCI_EXP_TYPE_RC_END:
2918 		if (!pdev->multifunction)
2919 			break;
2920 
2921 		return pci_acs_flags_enabled(pdev, acs_flags);
2922 	}
2923 
2924 	/*
2925 	 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
2926 	 * to single function devices with the exception of downstream ports.
2927 	 */
2928 	return true;
2929 }
2930 
2931 /**
2932  * pci_acs_path_enable - test ACS flags from start to end in a hierarchy
2933  * @start: starting downstream device
2934  * @end: ending upstream device or NULL to search to the root bus
2935  * @acs_flags: required flags
2936  *
2937  * Walk up a device tree from start to end testing PCI ACS support.  If
2938  * any step along the way does not support the required flags, return false.
2939  */
2940 bool pci_acs_path_enabled(struct pci_dev *start,
2941 			  struct pci_dev *end, u16 acs_flags)
2942 {
2943 	struct pci_dev *pdev, *parent = start;
2944 
2945 	do {
2946 		pdev = parent;
2947 
2948 		if (!pci_acs_enabled(pdev, acs_flags))
2949 			return false;
2950 
2951 		if (pci_is_root_bus(pdev->bus))
2952 			return (end == NULL);
2953 
2954 		parent = pdev->bus->self;
2955 	} while (pdev != end);
2956 
2957 	return true;
2958 }
2959 
2960 /**
2961  * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge
2962  * @dev: the PCI device
2963  * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD)
2964  *
2965  * Perform INTx swizzling for a device behind one level of bridge.  This is
2966  * required by section 9.1 of the PCI-to-PCI bridge specification for devices
2967  * behind bridges on add-in cards.  For devices with ARI enabled, the slot
2968  * number is always 0 (see the Implementation Note in section 2.2.8.1 of
2969  * the PCI Express Base Specification, Revision 2.1)
2970  */
2971 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin)
2972 {
2973 	int slot;
2974 
2975 	if (pci_ari_enabled(dev->bus))
2976 		slot = 0;
2977 	else
2978 		slot = PCI_SLOT(dev->devfn);
2979 
2980 	return (((pin - 1) + slot) % 4) + 1;
2981 }
2982 
2983 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge)
2984 {
2985 	u8 pin;
2986 
2987 	pin = dev->pin;
2988 	if (!pin)
2989 		return -1;
2990 
2991 	while (!pci_is_root_bus(dev->bus)) {
2992 		pin = pci_swizzle_interrupt_pin(dev, pin);
2993 		dev = dev->bus->self;
2994 	}
2995 	*bridge = dev;
2996 	return pin;
2997 }
2998 
2999 /**
3000  * pci_common_swizzle - swizzle INTx all the way to root bridge
3001  * @dev: the PCI device
3002  * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD)
3003  *
3004  * Perform INTx swizzling for a device.  This traverses through all PCI-to-PCI
3005  * bridges all the way up to a PCI root bus.
3006  */
3007 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp)
3008 {
3009 	u8 pin = *pinp;
3010 
3011 	while (!pci_is_root_bus(dev->bus)) {
3012 		pin = pci_swizzle_interrupt_pin(dev, pin);
3013 		dev = dev->bus->self;
3014 	}
3015 	*pinp = pin;
3016 	return PCI_SLOT(dev->devfn);
3017 }
3018 EXPORT_SYMBOL_GPL(pci_common_swizzle);
3019 
3020 /**
3021  *	pci_release_region - Release a PCI bar
3022  *	@pdev: PCI device whose resources were previously reserved by pci_request_region
3023  *	@bar: BAR to release
3024  *
3025  *	Releases the PCI I/O and memory resources previously reserved by a
3026  *	successful call to pci_request_region.  Call this function only
3027  *	after all use of the PCI regions has ceased.
3028  */
3029 void pci_release_region(struct pci_dev *pdev, int bar)
3030 {
3031 	struct pci_devres *dr;
3032 
3033 	if (pci_resource_len(pdev, bar) == 0)
3034 		return;
3035 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3036 		release_region(pci_resource_start(pdev, bar),
3037 				pci_resource_len(pdev, bar));
3038 	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3039 		release_mem_region(pci_resource_start(pdev, bar),
3040 				pci_resource_len(pdev, bar));
3041 
3042 	dr = find_pci_dr(pdev);
3043 	if (dr)
3044 		dr->region_mask &= ~(1 << bar);
3045 }
3046 EXPORT_SYMBOL(pci_release_region);
3047 
3048 /**
3049  *	__pci_request_region - Reserved PCI I/O and memory resource
3050  *	@pdev: PCI device whose resources are to be reserved
3051  *	@bar: BAR to be reserved
3052  *	@res_name: Name to be associated with resource.
3053  *	@exclusive: whether the region access is exclusive or not
3054  *
3055  *	Mark the PCI region associated with PCI device @pdev BR @bar as
3056  *	being reserved by owner @res_name.  Do not access any
3057  *	address inside the PCI regions unless this call returns
3058  *	successfully.
3059  *
3060  *	If @exclusive is set, then the region is marked so that userspace
3061  *	is explicitly not allowed to map the resource via /dev/mem or
3062  *	sysfs MMIO access.
3063  *
3064  *	Returns 0 on success, or %EBUSY on error.  A warning
3065  *	message is also printed on failure.
3066  */
3067 static int __pci_request_region(struct pci_dev *pdev, int bar,
3068 				const char *res_name, int exclusive)
3069 {
3070 	struct pci_devres *dr;
3071 
3072 	if (pci_resource_len(pdev, bar) == 0)
3073 		return 0;
3074 
3075 	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
3076 		if (!request_region(pci_resource_start(pdev, bar),
3077 			    pci_resource_len(pdev, bar), res_name))
3078 			goto err_out;
3079 	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
3080 		if (!__request_mem_region(pci_resource_start(pdev, bar),
3081 					pci_resource_len(pdev, bar), res_name,
3082 					exclusive))
3083 			goto err_out;
3084 	}
3085 
3086 	dr = find_pci_dr(pdev);
3087 	if (dr)
3088 		dr->region_mask |= 1 << bar;
3089 
3090 	return 0;
3091 
3092 err_out:
3093 	dev_warn(&pdev->dev, "BAR %d: can't reserve %pR\n", bar,
3094 		 &pdev->resource[bar]);
3095 	return -EBUSY;
3096 }
3097 
3098 /**
3099  *	pci_request_region - Reserve PCI I/O and memory resource
3100  *	@pdev: PCI device whose resources are to be reserved
3101  *	@bar: BAR to be reserved
3102  *	@res_name: Name to be associated with resource
3103  *
3104  *	Mark the PCI region associated with PCI device @pdev BAR @bar as
3105  *	being reserved by owner @res_name.  Do not access any
3106  *	address inside the PCI regions unless this call returns
3107  *	successfully.
3108  *
3109  *	Returns 0 on success, or %EBUSY on error.  A warning
3110  *	message is also printed on failure.
3111  */
3112 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
3113 {
3114 	return __pci_request_region(pdev, bar, res_name, 0);
3115 }
3116 EXPORT_SYMBOL(pci_request_region);
3117 
3118 /**
3119  *	pci_request_region_exclusive - Reserved PCI I/O and memory resource
3120  *	@pdev: PCI device whose resources are to be reserved
3121  *	@bar: BAR to be reserved
3122  *	@res_name: Name to be associated with resource.
3123  *
3124  *	Mark the PCI region associated with PCI device @pdev BR @bar as
3125  *	being reserved by owner @res_name.  Do not access any
3126  *	address inside the PCI regions unless this call returns
3127  *	successfully.
3128  *
3129  *	Returns 0 on success, or %EBUSY on error.  A warning
3130  *	message is also printed on failure.
3131  *
3132  *	The key difference that _exclusive makes it that userspace is
3133  *	explicitly not allowed to map the resource via /dev/mem or
3134  *	sysfs.
3135  */
3136 int pci_request_region_exclusive(struct pci_dev *pdev, int bar,
3137 				 const char *res_name)
3138 {
3139 	return __pci_request_region(pdev, bar, res_name, IORESOURCE_EXCLUSIVE);
3140 }
3141 EXPORT_SYMBOL(pci_request_region_exclusive);
3142 
3143 /**
3144  * pci_release_selected_regions - Release selected PCI I/O and memory resources
3145  * @pdev: PCI device whose resources were previously reserved
3146  * @bars: Bitmask of BARs to be released
3147  *
3148  * Release selected PCI I/O and memory resources previously reserved.
3149  * Call this function only after all use of the PCI regions has ceased.
3150  */
3151 void pci_release_selected_regions(struct pci_dev *pdev, int bars)
3152 {
3153 	int i;
3154 
3155 	for (i = 0; i < 6; i++)
3156 		if (bars & (1 << i))
3157 			pci_release_region(pdev, i);
3158 }
3159 EXPORT_SYMBOL(pci_release_selected_regions);
3160 
3161 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
3162 					  const char *res_name, int excl)
3163 {
3164 	int i;
3165 
3166 	for (i = 0; i < 6; i++)
3167 		if (bars & (1 << i))
3168 			if (__pci_request_region(pdev, i, res_name, excl))
3169 				goto err_out;
3170 	return 0;
3171 
3172 err_out:
3173 	while (--i >= 0)
3174 		if (bars & (1 << i))
3175 			pci_release_region(pdev, i);
3176 
3177 	return -EBUSY;
3178 }
3179 
3180 
3181 /**
3182  * pci_request_selected_regions - Reserve selected PCI I/O and memory resources
3183  * @pdev: PCI device whose resources are to be reserved
3184  * @bars: Bitmask of BARs to be requested
3185  * @res_name: Name to be associated with resource
3186  */
3187 int pci_request_selected_regions(struct pci_dev *pdev, int bars,
3188 				 const char *res_name)
3189 {
3190 	return __pci_request_selected_regions(pdev, bars, res_name, 0);
3191 }
3192 EXPORT_SYMBOL(pci_request_selected_regions);
3193 
3194 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
3195 					   const char *res_name)
3196 {
3197 	return __pci_request_selected_regions(pdev, bars, res_name,
3198 			IORESOURCE_EXCLUSIVE);
3199 }
3200 EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
3201 
3202 /**
3203  *	pci_release_regions - Release reserved PCI I/O and memory resources
3204  *	@pdev: PCI device whose resources were previously reserved by pci_request_regions
3205  *
3206  *	Releases all PCI I/O and memory resources previously reserved by a
3207  *	successful call to pci_request_regions.  Call this function only
3208  *	after all use of the PCI regions has ceased.
3209  */
3210 
3211 void pci_release_regions(struct pci_dev *pdev)
3212 {
3213 	pci_release_selected_regions(pdev, (1 << 6) - 1);
3214 }
3215 EXPORT_SYMBOL(pci_release_regions);
3216 
3217 /**
3218  *	pci_request_regions - Reserved PCI I/O and memory resources
3219  *	@pdev: PCI device whose resources are to be reserved
3220  *	@res_name: Name to be associated with resource.
3221  *
3222  *	Mark all PCI regions associated with PCI device @pdev as
3223  *	being reserved by owner @res_name.  Do not access any
3224  *	address inside the PCI regions unless this call returns
3225  *	successfully.
3226  *
3227  *	Returns 0 on success, or %EBUSY on error.  A warning
3228  *	message is also printed on failure.
3229  */
3230 int pci_request_regions(struct pci_dev *pdev, const char *res_name)
3231 {
3232 	return pci_request_selected_regions(pdev, ((1 << 6) - 1), res_name);
3233 }
3234 EXPORT_SYMBOL(pci_request_regions);
3235 
3236 /**
3237  *	pci_request_regions_exclusive - Reserved PCI I/O and memory resources
3238  *	@pdev: PCI device whose resources are to be reserved
3239  *	@res_name: Name to be associated with resource.
3240  *
3241  *	Mark all PCI regions associated with PCI device @pdev as
3242  *	being reserved by owner @res_name.  Do not access any
3243  *	address inside the PCI regions unless this call returns
3244  *	successfully.
3245  *
3246  *	pci_request_regions_exclusive() will mark the region so that
3247  *	/dev/mem and the sysfs MMIO access will not be allowed.
3248  *
3249  *	Returns 0 on success, or %EBUSY on error.  A warning
3250  *	message is also printed on failure.
3251  */
3252 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
3253 {
3254 	return pci_request_selected_regions_exclusive(pdev,
3255 					((1 << 6) - 1), res_name);
3256 }
3257 EXPORT_SYMBOL(pci_request_regions_exclusive);
3258 
3259 #ifdef PCI_IOBASE
3260 struct io_range {
3261 	struct list_head list;
3262 	phys_addr_t start;
3263 	resource_size_t size;
3264 };
3265 
3266 static LIST_HEAD(io_range_list);
3267 static DEFINE_SPINLOCK(io_range_lock);
3268 #endif
3269 
3270 /*
3271  * Record the PCI IO range (expressed as CPU physical address + size).
3272  * Return a negative value if an error has occured, zero otherwise
3273  */
3274 int __weak pci_register_io_range(phys_addr_t addr, resource_size_t size)
3275 {
3276 	int err = 0;
3277 
3278 #ifdef PCI_IOBASE
3279 	struct io_range *range;
3280 	resource_size_t allocated_size = 0;
3281 
3282 	/* check if the range hasn't been previously recorded */
3283 	spin_lock(&io_range_lock);
3284 	list_for_each_entry(range, &io_range_list, list) {
3285 		if (addr >= range->start && addr + size <= range->start + size) {
3286 			/* range already registered, bail out */
3287 			goto end_register;
3288 		}
3289 		allocated_size += range->size;
3290 	}
3291 
3292 	/* range not registed yet, check for available space */
3293 	if (allocated_size + size - 1 > IO_SPACE_LIMIT) {
3294 		/* if it's too big check if 64K space can be reserved */
3295 		if (allocated_size + SZ_64K - 1 > IO_SPACE_LIMIT) {
3296 			err = -E2BIG;
3297 			goto end_register;
3298 		}
3299 
3300 		size = SZ_64K;
3301 		pr_warn("Requested IO range too big, new size set to 64K\n");
3302 	}
3303 
3304 	/* add the range to the list */
3305 	range = kzalloc(sizeof(*range), GFP_ATOMIC);
3306 	if (!range) {
3307 		err = -ENOMEM;
3308 		goto end_register;
3309 	}
3310 
3311 	range->start = addr;
3312 	range->size = size;
3313 
3314 	list_add_tail(&range->list, &io_range_list);
3315 
3316 end_register:
3317 	spin_unlock(&io_range_lock);
3318 #endif
3319 
3320 	return err;
3321 }
3322 
3323 phys_addr_t pci_pio_to_address(unsigned long pio)
3324 {
3325 	phys_addr_t address = (phys_addr_t)OF_BAD_ADDR;
3326 
3327 #ifdef PCI_IOBASE
3328 	struct io_range *range;
3329 	resource_size_t allocated_size = 0;
3330 
3331 	if (pio > IO_SPACE_LIMIT)
3332 		return address;
3333 
3334 	spin_lock(&io_range_lock);
3335 	list_for_each_entry(range, &io_range_list, list) {
3336 		if (pio >= allocated_size && pio < allocated_size + range->size) {
3337 			address = range->start + pio - allocated_size;
3338 			break;
3339 		}
3340 		allocated_size += range->size;
3341 	}
3342 	spin_unlock(&io_range_lock);
3343 #endif
3344 
3345 	return address;
3346 }
3347 
3348 unsigned long __weak pci_address_to_pio(phys_addr_t address)
3349 {
3350 #ifdef PCI_IOBASE
3351 	struct io_range *res;
3352 	resource_size_t offset = 0;
3353 	unsigned long addr = -1;
3354 
3355 	spin_lock(&io_range_lock);
3356 	list_for_each_entry(res, &io_range_list, list) {
3357 		if (address >= res->start && address < res->start + res->size) {
3358 			addr = address - res->start + offset;
3359 			break;
3360 		}
3361 		offset += res->size;
3362 	}
3363 	spin_unlock(&io_range_lock);
3364 
3365 	return addr;
3366 #else
3367 	if (address > IO_SPACE_LIMIT)
3368 		return (unsigned long)-1;
3369 
3370 	return (unsigned long) address;
3371 #endif
3372 }
3373 
3374 /**
3375  *	pci_remap_iospace - Remap the memory mapped I/O space
3376  *	@res: Resource describing the I/O space
3377  *	@phys_addr: physical address of range to be mapped
3378  *
3379  *	Remap the memory mapped I/O space described by the @res
3380  *	and the CPU physical address @phys_addr into virtual address space.
3381  *	Only architectures that have memory mapped IO functions defined
3382  *	(and the PCI_IOBASE value defined) should call this function.
3383  */
3384 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
3385 {
3386 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
3387 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
3388 
3389 	if (!(res->flags & IORESOURCE_IO))
3390 		return -EINVAL;
3391 
3392 	if (res->end > IO_SPACE_LIMIT)
3393 		return -EINVAL;
3394 
3395 	return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
3396 				  pgprot_device(PAGE_KERNEL));
3397 #else
3398 	/* this architecture does not have memory mapped I/O space,
3399 	   so this function should never be called */
3400 	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
3401 	return -ENODEV;
3402 #endif
3403 }
3404 EXPORT_SYMBOL(pci_remap_iospace);
3405 
3406 /**
3407  *	pci_unmap_iospace - Unmap the memory mapped I/O space
3408  *	@res: resource to be unmapped
3409  *
3410  *	Unmap the CPU virtual address @res from virtual address space.
3411  *	Only architectures that have memory mapped IO functions defined
3412  *	(and the PCI_IOBASE value defined) should call this function.
3413  */
3414 void pci_unmap_iospace(struct resource *res)
3415 {
3416 #if defined(PCI_IOBASE) && defined(CONFIG_MMU)
3417 	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
3418 
3419 	unmap_kernel_range(vaddr, resource_size(res));
3420 #endif
3421 }
3422 EXPORT_SYMBOL(pci_unmap_iospace);
3423 
3424 /**
3425  * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace()
3426  * @dev: Generic device to remap IO address for
3427  * @offset: Resource address to map
3428  * @size: Size of map
3429  *
3430  * Managed pci_remap_cfgspace().  Map is automatically unmapped on driver
3431  * detach.
3432  */
3433 void __iomem *devm_pci_remap_cfgspace(struct device *dev,
3434 				      resource_size_t offset,
3435 				      resource_size_t size)
3436 {
3437 	void __iomem **ptr, *addr;
3438 
3439 	ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL);
3440 	if (!ptr)
3441 		return NULL;
3442 
3443 	addr = pci_remap_cfgspace(offset, size);
3444 	if (addr) {
3445 		*ptr = addr;
3446 		devres_add(dev, ptr);
3447 	} else
3448 		devres_free(ptr);
3449 
3450 	return addr;
3451 }
3452 EXPORT_SYMBOL(devm_pci_remap_cfgspace);
3453 
3454 /**
3455  * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource
3456  * @dev: generic device to handle the resource for
3457  * @res: configuration space resource to be handled
3458  *
3459  * Checks that a resource is a valid memory region, requests the memory
3460  * region and ioremaps with pci_remap_cfgspace() API that ensures the
3461  * proper PCI configuration space memory attributes are guaranteed.
3462  *
3463  * All operations are managed and will be undone on driver detach.
3464  *
3465  * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code
3466  * on failure. Usage example:
3467  *
3468  *	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3469  *	base = devm_pci_remap_cfg_resource(&pdev->dev, res);
3470  *	if (IS_ERR(base))
3471  *		return PTR_ERR(base);
3472  */
3473 void __iomem *devm_pci_remap_cfg_resource(struct device *dev,
3474 					  struct resource *res)
3475 {
3476 	resource_size_t size;
3477 	const char *name;
3478 	void __iomem *dest_ptr;
3479 
3480 	BUG_ON(!dev);
3481 
3482 	if (!res || resource_type(res) != IORESOURCE_MEM) {
3483 		dev_err(dev, "invalid resource\n");
3484 		return IOMEM_ERR_PTR(-EINVAL);
3485 	}
3486 
3487 	size = resource_size(res);
3488 	name = res->name ?: dev_name(dev);
3489 
3490 	if (!devm_request_mem_region(dev, res->start, size, name)) {
3491 		dev_err(dev, "can't request region for resource %pR\n", res);
3492 		return IOMEM_ERR_PTR(-EBUSY);
3493 	}
3494 
3495 	dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size);
3496 	if (!dest_ptr) {
3497 		dev_err(dev, "ioremap failed for resource %pR\n", res);
3498 		devm_release_mem_region(dev, res->start, size);
3499 		dest_ptr = IOMEM_ERR_PTR(-ENOMEM);
3500 	}
3501 
3502 	return dest_ptr;
3503 }
3504 EXPORT_SYMBOL(devm_pci_remap_cfg_resource);
3505 
3506 static void __pci_set_master(struct pci_dev *dev, bool enable)
3507 {
3508 	u16 old_cmd, cmd;
3509 
3510 	pci_read_config_word(dev, PCI_COMMAND, &old_cmd);
3511 	if (enable)
3512 		cmd = old_cmd | PCI_COMMAND_MASTER;
3513 	else
3514 		cmd = old_cmd & ~PCI_COMMAND_MASTER;
3515 	if (cmd != old_cmd) {
3516 		dev_dbg(&dev->dev, "%s bus mastering\n",
3517 			enable ? "enabling" : "disabling");
3518 		pci_write_config_word(dev, PCI_COMMAND, cmd);
3519 	}
3520 	dev->is_busmaster = enable;
3521 }
3522 
3523 /**
3524  * pcibios_setup - process "pci=" kernel boot arguments
3525  * @str: string used to pass in "pci=" kernel boot arguments
3526  *
3527  * Process kernel boot arguments.  This is the default implementation.
3528  * Architecture specific implementations can override this as necessary.
3529  */
3530 char * __weak __init pcibios_setup(char *str)
3531 {
3532 	return str;
3533 }
3534 
3535 /**
3536  * pcibios_set_master - enable PCI bus-mastering for device dev
3537  * @dev: the PCI device to enable
3538  *
3539  * Enables PCI bus-mastering for the device.  This is the default
3540  * implementation.  Architecture specific implementations can override
3541  * this if necessary.
3542  */
3543 void __weak pcibios_set_master(struct pci_dev *dev)
3544 {
3545 	u8 lat;
3546 
3547 	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
3548 	if (pci_is_pcie(dev))
3549 		return;
3550 
3551 	pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat);
3552 	if (lat < 16)
3553 		lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
3554 	else if (lat > pcibios_max_latency)
3555 		lat = pcibios_max_latency;
3556 	else
3557 		return;
3558 
3559 	pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat);
3560 }
3561 
3562 /**
3563  * pci_set_master - enables bus-mastering for device dev
3564  * @dev: the PCI device to enable
3565  *
3566  * Enables bus-mastering on the device and calls pcibios_set_master()
3567  * to do the needed arch specific settings.
3568  */
3569 void pci_set_master(struct pci_dev *dev)
3570 {
3571 	__pci_set_master(dev, true);
3572 	pcibios_set_master(dev);
3573 }
3574 EXPORT_SYMBOL(pci_set_master);
3575 
3576 /**
3577  * pci_clear_master - disables bus-mastering for device dev
3578  * @dev: the PCI device to disable
3579  */
3580 void pci_clear_master(struct pci_dev *dev)
3581 {
3582 	__pci_set_master(dev, false);
3583 }
3584 EXPORT_SYMBOL(pci_clear_master);
3585 
3586 /**
3587  * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
3588  * @dev: the PCI device for which MWI is to be enabled
3589  *
3590  * Helper function for pci_set_mwi.
3591  * Originally copied from drivers/net/acenic.c.
3592  * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
3593  *
3594  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
3595  */
3596 int pci_set_cacheline_size(struct pci_dev *dev)
3597 {
3598 	u8 cacheline_size;
3599 
3600 	if (!pci_cache_line_size)
3601 		return -EINVAL;
3602 
3603 	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
3604 	   equal to or multiple of the right value. */
3605 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
3606 	if (cacheline_size >= pci_cache_line_size &&
3607 	    (cacheline_size % pci_cache_line_size) == 0)
3608 		return 0;
3609 
3610 	/* Write the correct value. */
3611 	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size);
3612 	/* Read it back. */
3613 	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size);
3614 	if (cacheline_size == pci_cache_line_size)
3615 		return 0;
3616 
3617 	dev_printk(KERN_DEBUG, &dev->dev, "cache line size of %d is not supported\n",
3618 		   pci_cache_line_size << 2);
3619 
3620 	return -EINVAL;
3621 }
3622 EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
3623 
3624 /**
3625  * pci_set_mwi - enables memory-write-invalidate PCI transaction
3626  * @dev: the PCI device for which MWI is enabled
3627  *
3628  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
3629  *
3630  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
3631  */
3632 int pci_set_mwi(struct pci_dev *dev)
3633 {
3634 #ifdef PCI_DISABLE_MWI
3635 	return 0;
3636 #else
3637 	int rc;
3638 	u16 cmd;
3639 
3640 	rc = pci_set_cacheline_size(dev);
3641 	if (rc)
3642 		return rc;
3643 
3644 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
3645 	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
3646 		dev_dbg(&dev->dev, "enabling Mem-Wr-Inval\n");
3647 		cmd |= PCI_COMMAND_INVALIDATE;
3648 		pci_write_config_word(dev, PCI_COMMAND, cmd);
3649 	}
3650 	return 0;
3651 #endif
3652 }
3653 EXPORT_SYMBOL(pci_set_mwi);
3654 
3655 /**
3656  * pci_try_set_mwi - enables memory-write-invalidate PCI transaction
3657  * @dev: the PCI device for which MWI is enabled
3658  *
3659  * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
3660  * Callers are not required to check the return value.
3661  *
3662  * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
3663  */
3664 int pci_try_set_mwi(struct pci_dev *dev)
3665 {
3666 #ifdef PCI_DISABLE_MWI
3667 	return 0;
3668 #else
3669 	return pci_set_mwi(dev);
3670 #endif
3671 }
3672 EXPORT_SYMBOL(pci_try_set_mwi);
3673 
3674 /**
3675  * pci_clear_mwi - disables Memory-Write-Invalidate for device dev
3676  * @dev: the PCI device to disable
3677  *
3678  * Disables PCI Memory-Write-Invalidate transaction on the device
3679  */
3680 void pci_clear_mwi(struct pci_dev *dev)
3681 {
3682 #ifndef PCI_DISABLE_MWI
3683 	u16 cmd;
3684 
3685 	pci_read_config_word(dev, PCI_COMMAND, &cmd);
3686 	if (cmd & PCI_COMMAND_INVALIDATE) {
3687 		cmd &= ~PCI_COMMAND_INVALIDATE;
3688 		pci_write_config_word(dev, PCI_COMMAND, cmd);
3689 	}
3690 #endif
3691 }
3692 EXPORT_SYMBOL(pci_clear_mwi);
3693 
3694 /**
3695  * pci_intx - enables/disables PCI INTx for device dev
3696  * @pdev: the PCI device to operate on
3697  * @enable: boolean: whether to enable or disable PCI INTx
3698  *
3699  * Enables/disables PCI INTx for device dev
3700  */
3701 void pci_intx(struct pci_dev *pdev, int enable)
3702 {
3703 	u16 pci_command, new;
3704 
3705 	pci_read_config_word(pdev, PCI_COMMAND, &pci_command);
3706 
3707 	if (enable)
3708 		new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
3709 	else
3710 		new = pci_command | PCI_COMMAND_INTX_DISABLE;
3711 
3712 	if (new != pci_command) {
3713 		struct pci_devres *dr;
3714 
3715 		pci_write_config_word(pdev, PCI_COMMAND, new);
3716 
3717 		dr = find_pci_dr(pdev);
3718 		if (dr && !dr->restore_intx) {
3719 			dr->restore_intx = 1;
3720 			dr->orig_intx = !enable;
3721 		}
3722 	}
3723 }
3724 EXPORT_SYMBOL_GPL(pci_intx);
3725 
3726 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask)
3727 {
3728 	struct pci_bus *bus = dev->bus;
3729 	bool mask_updated = true;
3730 	u32 cmd_status_dword;
3731 	u16 origcmd, newcmd;
3732 	unsigned long flags;
3733 	bool irq_pending;
3734 
3735 	/*
3736 	 * We do a single dword read to retrieve both command and status.
3737 	 * Document assumptions that make this possible.
3738 	 */
3739 	BUILD_BUG_ON(PCI_COMMAND % 4);
3740 	BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS);
3741 
3742 	raw_spin_lock_irqsave(&pci_lock, flags);
3743 
3744 	bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword);
3745 
3746 	irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT;
3747 
3748 	/*
3749 	 * Check interrupt status register to see whether our device
3750 	 * triggered the interrupt (when masking) or the next IRQ is
3751 	 * already pending (when unmasking).
3752 	 */
3753 	if (mask != irq_pending) {
3754 		mask_updated = false;
3755 		goto done;
3756 	}
3757 
3758 	origcmd = cmd_status_dword;
3759 	newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE;
3760 	if (mask)
3761 		newcmd |= PCI_COMMAND_INTX_DISABLE;
3762 	if (newcmd != origcmd)
3763 		bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd);
3764 
3765 done:
3766 	raw_spin_unlock_irqrestore(&pci_lock, flags);
3767 
3768 	return mask_updated;
3769 }
3770 
3771 /**
3772  * pci_check_and_mask_intx - mask INTx on pending interrupt
3773  * @dev: the PCI device to operate on
3774  *
3775  * Check if the device dev has its INTx line asserted, mask it and
3776  * return true in that case. False is returned if no interrupt was
3777  * pending.
3778  */
3779 bool pci_check_and_mask_intx(struct pci_dev *dev)
3780 {
3781 	return pci_check_and_set_intx_mask(dev, true);
3782 }
3783 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx);
3784 
3785 /**
3786  * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending
3787  * @dev: the PCI device to operate on
3788  *
3789  * Check if the device dev has its INTx line asserted, unmask it if not
3790  * and return true. False is returned and the mask remains active if
3791  * there was still an interrupt pending.
3792  */
3793 bool pci_check_and_unmask_intx(struct pci_dev *dev)
3794 {
3795 	return pci_check_and_set_intx_mask(dev, false);
3796 }
3797 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx);
3798 
3799 /**
3800  * pci_wait_for_pending_transaction - waits for pending transaction
3801  * @dev: the PCI device to operate on
3802  *
3803  * Return 0 if transaction is pending 1 otherwise.
3804  */
3805 int pci_wait_for_pending_transaction(struct pci_dev *dev)
3806 {
3807 	if (!pci_is_pcie(dev))
3808 		return 1;
3809 
3810 	return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
3811 				    PCI_EXP_DEVSTA_TRPND);
3812 }
3813 EXPORT_SYMBOL(pci_wait_for_pending_transaction);
3814 
3815 /*
3816  * We should only need to wait 100ms after FLR, but some devices take longer.
3817  * Wait for up to 1000ms for config space to return something other than -1.
3818  * Intel IGD requires this when an LCD panel is attached.  We read the 2nd
3819  * dword because VFs don't implement the 1st dword.
3820  */
3821 static void pci_flr_wait(struct pci_dev *dev)
3822 {
3823 	int i = 0;
3824 	u32 id;
3825 
3826 	do {
3827 		msleep(100);
3828 		pci_read_config_dword(dev, PCI_COMMAND, &id);
3829 	} while (i++ < 10 && id == ~0);
3830 
3831 	if (id == ~0)
3832 		dev_warn(&dev->dev, "Failed to return from FLR\n");
3833 	else if (i > 1)
3834 		dev_info(&dev->dev, "Required additional %dms to return from FLR\n",
3835 			 (i - 1) * 100);
3836 }
3837 
3838 /**
3839  * pcie_has_flr - check if a device supports function level resets
3840  * @dev:	device to check
3841  *
3842  * Returns true if the device advertises support for PCIe function level
3843  * resets.
3844  */
3845 static bool pcie_has_flr(struct pci_dev *dev)
3846 {
3847 	u32 cap;
3848 
3849 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
3850 		return false;
3851 
3852 	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap);
3853 	return cap & PCI_EXP_DEVCAP_FLR;
3854 }
3855 
3856 /**
3857  * pcie_flr - initiate a PCIe function level reset
3858  * @dev:	device to reset
3859  *
3860  * Initiate a function level reset on @dev.  The caller should ensure the
3861  * device supports FLR before calling this function, e.g. by using the
3862  * pcie_has_flr() helper.
3863  */
3864 void pcie_flr(struct pci_dev *dev)
3865 {
3866 	if (!pci_wait_for_pending_transaction(dev))
3867 		dev_err(&dev->dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
3868 
3869 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
3870 	pci_flr_wait(dev);
3871 }
3872 EXPORT_SYMBOL_GPL(pcie_flr);
3873 
3874 static int pci_af_flr(struct pci_dev *dev, int probe)
3875 {
3876 	int pos;
3877 	u8 cap;
3878 
3879 	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
3880 	if (!pos)
3881 		return -ENOTTY;
3882 
3883 	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
3884 		return -ENOTTY;
3885 
3886 	pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap);
3887 	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
3888 		return -ENOTTY;
3889 
3890 	if (probe)
3891 		return 0;
3892 
3893 	/*
3894 	 * Wait for Transaction Pending bit to clear.  A word-aligned test
3895 	 * is used, so we use the conrol offset rather than status and shift
3896 	 * the test bit to match.
3897 	 */
3898 	if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL,
3899 				 PCI_AF_STATUS_TP << 8))
3900 		dev_err(&dev->dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
3901 
3902 	pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
3903 	pci_flr_wait(dev);
3904 	return 0;
3905 }
3906 
3907 /**
3908  * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
3909  * @dev: Device to reset.
3910  * @probe: If set, only check if the device can be reset this way.
3911  *
3912  * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
3913  * unset, it will be reinitialized internally when going from PCI_D3hot to
3914  * PCI_D0.  If that's the case and the device is not in a low-power state
3915  * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
3916  *
3917  * NOTE: This causes the caller to sleep for twice the device power transition
3918  * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
3919  * by default (i.e. unless the @dev's d3_delay field has a different value).
3920  * Moreover, only devices in D0 can be reset by this function.
3921  */
3922 static int pci_pm_reset(struct pci_dev *dev, int probe)
3923 {
3924 	u16 csr;
3925 
3926 	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
3927 		return -ENOTTY;
3928 
3929 	pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr);
3930 	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
3931 		return -ENOTTY;
3932 
3933 	if (probe)
3934 		return 0;
3935 
3936 	if (dev->current_state != PCI_D0)
3937 		return -EINVAL;
3938 
3939 	csr &= ~PCI_PM_CTRL_STATE_MASK;
3940 	csr |= PCI_D3hot;
3941 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
3942 	pci_dev_d3_sleep(dev);
3943 
3944 	csr &= ~PCI_PM_CTRL_STATE_MASK;
3945 	csr |= PCI_D0;
3946 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr);
3947 	pci_dev_d3_sleep(dev);
3948 
3949 	return 0;
3950 }
3951 
3952 void pci_reset_secondary_bus(struct pci_dev *dev)
3953 {
3954 	u16 ctrl;
3955 
3956 	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl);
3957 	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
3958 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
3959 	/*
3960 	 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms.  Double
3961 	 * this to 2ms to ensure that we meet the minimum requirement.
3962 	 */
3963 	msleep(2);
3964 
3965 	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
3966 	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl);
3967 
3968 	/*
3969 	 * Trhfa for conventional PCI is 2^25 clock cycles.
3970 	 * Assuming a minimum 33MHz clock this results in a 1s
3971 	 * delay before we can consider subordinate devices to
3972 	 * be re-initialized.  PCIe has some ways to shorten this,
3973 	 * but we don't make use of them yet.
3974 	 */
3975 	ssleep(1);
3976 }
3977 
3978 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
3979 {
3980 	pci_reset_secondary_bus(dev);
3981 }
3982 
3983 /**
3984  * pci_reset_bridge_secondary_bus - Reset the secondary bus on a PCI bridge.
3985  * @dev: Bridge device
3986  *
3987  * Use the bridge control register to assert reset on the secondary bus.
3988  * Devices on the secondary bus are left in power-on state.
3989  */
3990 void pci_reset_bridge_secondary_bus(struct pci_dev *dev)
3991 {
3992 	pcibios_reset_secondary_bus(dev);
3993 }
3994 EXPORT_SYMBOL_GPL(pci_reset_bridge_secondary_bus);
3995 
3996 static int pci_parent_bus_reset(struct pci_dev *dev, int probe)
3997 {
3998 	struct pci_dev *pdev;
3999 
4000 	if (pci_is_root_bus(dev->bus) || dev->subordinate ||
4001 	    !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4002 		return -ENOTTY;
4003 
4004 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4005 		if (pdev != dev)
4006 			return -ENOTTY;
4007 
4008 	if (probe)
4009 		return 0;
4010 
4011 	pci_reset_bridge_secondary_bus(dev->bus->self);
4012 
4013 	return 0;
4014 }
4015 
4016 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe)
4017 {
4018 	int rc = -ENOTTY;
4019 
4020 	if (!hotplug || !try_module_get(hotplug->ops->owner))
4021 		return rc;
4022 
4023 	if (hotplug->ops->reset_slot)
4024 		rc = hotplug->ops->reset_slot(hotplug, probe);
4025 
4026 	module_put(hotplug->ops->owner);
4027 
4028 	return rc;
4029 }
4030 
4031 static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe)
4032 {
4033 	struct pci_dev *pdev;
4034 
4035 	if (dev->subordinate || !dev->slot ||
4036 	    dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4037 		return -ENOTTY;
4038 
4039 	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4040 		if (pdev != dev && pdev->slot == dev->slot)
4041 			return -ENOTTY;
4042 
4043 	return pci_reset_hotplug_slot(dev->slot->hotplug, probe);
4044 }
4045 
4046 static void pci_dev_lock(struct pci_dev *dev)
4047 {
4048 	pci_cfg_access_lock(dev);
4049 	/* block PM suspend, driver probe, etc. */
4050 	device_lock(&dev->dev);
4051 }
4052 
4053 /* Return 1 on successful lock, 0 on contention */
4054 static int pci_dev_trylock(struct pci_dev *dev)
4055 {
4056 	if (pci_cfg_access_trylock(dev)) {
4057 		if (device_trylock(&dev->dev))
4058 			return 1;
4059 		pci_cfg_access_unlock(dev);
4060 	}
4061 
4062 	return 0;
4063 }
4064 
4065 static void pci_dev_unlock(struct pci_dev *dev)
4066 {
4067 	device_unlock(&dev->dev);
4068 	pci_cfg_access_unlock(dev);
4069 }
4070 
4071 static void pci_dev_save_and_disable(struct pci_dev *dev)
4072 {
4073 	const struct pci_error_handlers *err_handler =
4074 			dev->driver ? dev->driver->err_handler : NULL;
4075 
4076 	/*
4077 	 * dev->driver->err_handler->reset_prepare() is protected against
4078 	 * races with ->remove() by the device lock, which must be held by
4079 	 * the caller.
4080 	 */
4081 	if (err_handler && err_handler->reset_prepare)
4082 		err_handler->reset_prepare(dev);
4083 
4084 	/*
4085 	 * Wake-up device prior to save.  PM registers default to D0 after
4086 	 * reset and a simple register restore doesn't reliably return
4087 	 * to a non-D0 state anyway.
4088 	 */
4089 	pci_set_power_state(dev, PCI_D0);
4090 
4091 	pci_save_state(dev);
4092 	/*
4093 	 * Disable the device by clearing the Command register, except for
4094 	 * INTx-disable which is set.  This not only disables MMIO and I/O port
4095 	 * BARs, but also prevents the device from being Bus Master, preventing
4096 	 * DMA from the device including MSI/MSI-X interrupts.  For PCI 2.3
4097 	 * compliant devices, INTx-disable prevents legacy interrupts.
4098 	 */
4099 	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
4100 }
4101 
4102 static void pci_dev_restore(struct pci_dev *dev)
4103 {
4104 	const struct pci_error_handlers *err_handler =
4105 			dev->driver ? dev->driver->err_handler : NULL;
4106 
4107 	pci_restore_state(dev);
4108 
4109 	/*
4110 	 * dev->driver->err_handler->reset_done() is protected against
4111 	 * races with ->remove() by the device lock, which must be held by
4112 	 * the caller.
4113 	 */
4114 	if (err_handler && err_handler->reset_done)
4115 		err_handler->reset_done(dev);
4116 }
4117 
4118 /**
4119  * __pci_reset_function - reset a PCI device function
4120  * @dev: PCI device to reset
4121  *
4122  * Some devices allow an individual function to be reset without affecting
4123  * other functions in the same device.  The PCI device must be responsive
4124  * to PCI config space in order to use this function.
4125  *
4126  * The device function is presumed to be unused when this function is called.
4127  * Resetting the device will make the contents of PCI configuration space
4128  * random, so any caller of this must be prepared to reinitialise the
4129  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
4130  * etc.
4131  *
4132  * Returns 0 if the device function was successfully reset or negative if the
4133  * device doesn't support resetting a single function.
4134  */
4135 int __pci_reset_function(struct pci_dev *dev)
4136 {
4137 	int ret;
4138 
4139 	pci_dev_lock(dev);
4140 	ret = __pci_reset_function_locked(dev);
4141 	pci_dev_unlock(dev);
4142 
4143 	return ret;
4144 }
4145 EXPORT_SYMBOL_GPL(__pci_reset_function);
4146 
4147 /**
4148  * __pci_reset_function_locked - reset a PCI device function while holding
4149  * the @dev mutex lock.
4150  * @dev: PCI device to reset
4151  *
4152  * Some devices allow an individual function to be reset without affecting
4153  * other functions in the same device.  The PCI device must be responsive
4154  * to PCI config space in order to use this function.
4155  *
4156  * The device function is presumed to be unused and the caller is holding
4157  * the device mutex lock when this function is called.
4158  * Resetting the device will make the contents of PCI configuration space
4159  * random, so any caller of this must be prepared to reinitialise the
4160  * device including MSI, bus mastering, BARs, decoding IO and memory spaces,
4161  * etc.
4162  *
4163  * Returns 0 if the device function was successfully reset or negative if the
4164  * device doesn't support resetting a single function.
4165  */
4166 int __pci_reset_function_locked(struct pci_dev *dev)
4167 {
4168 	int rc;
4169 
4170 	might_sleep();
4171 
4172 	rc = pci_dev_specific_reset(dev, 0);
4173 	if (rc != -ENOTTY)
4174 		return rc;
4175 	if (pcie_has_flr(dev)) {
4176 		pcie_flr(dev);
4177 		return 0;
4178 	}
4179 	rc = pci_af_flr(dev, 0);
4180 	if (rc != -ENOTTY)
4181 		return rc;
4182 	rc = pci_pm_reset(dev, 0);
4183 	if (rc != -ENOTTY)
4184 		return rc;
4185 	rc = pci_dev_reset_slot_function(dev, 0);
4186 	if (rc != -ENOTTY)
4187 		return rc;
4188 	return pci_parent_bus_reset(dev, 0);
4189 }
4190 EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
4191 
4192 /**
4193  * pci_probe_reset_function - check whether the device can be safely reset
4194  * @dev: PCI device to reset
4195  *
4196  * Some devices allow an individual function to be reset without affecting
4197  * other functions in the same device.  The PCI device must be responsive
4198  * to PCI config space in order to use this function.
4199  *
4200  * Returns 0 if the device function can be reset or negative if the
4201  * device doesn't support resetting a single function.
4202  */
4203 int pci_probe_reset_function(struct pci_dev *dev)
4204 {
4205 	int rc;
4206 
4207 	might_sleep();
4208 
4209 	rc = pci_dev_specific_reset(dev, 1);
4210 	if (rc != -ENOTTY)
4211 		return rc;
4212 	if (pcie_has_flr(dev))
4213 		return 0;
4214 	rc = pci_af_flr(dev, 1);
4215 	if (rc != -ENOTTY)
4216 		return rc;
4217 	rc = pci_pm_reset(dev, 1);
4218 	if (rc != -ENOTTY)
4219 		return rc;
4220 	rc = pci_dev_reset_slot_function(dev, 1);
4221 	if (rc != -ENOTTY)
4222 		return rc;
4223 
4224 	return pci_parent_bus_reset(dev, 1);
4225 }
4226 
4227 /**
4228  * pci_reset_function - quiesce and reset a PCI device function
4229  * @dev: PCI device to reset
4230  *
4231  * Some devices allow an individual function to be reset without affecting
4232  * other functions in the same device.  The PCI device must be responsive
4233  * to PCI config space in order to use this function.
4234  *
4235  * This function does not just reset the PCI portion of a device, but
4236  * clears all the state associated with the device.  This function differs
4237  * from __pci_reset_function in that it saves and restores device state
4238  * over the reset.
4239  *
4240  * Returns 0 if the device function was successfully reset or negative if the
4241  * device doesn't support resetting a single function.
4242  */
4243 int pci_reset_function(struct pci_dev *dev)
4244 {
4245 	int rc;
4246 
4247 	rc = pci_probe_reset_function(dev);
4248 	if (rc)
4249 		return rc;
4250 
4251 	pci_dev_lock(dev);
4252 	pci_dev_save_and_disable(dev);
4253 
4254 	rc = __pci_reset_function_locked(dev);
4255 
4256 	pci_dev_restore(dev);
4257 	pci_dev_unlock(dev);
4258 
4259 	return rc;
4260 }
4261 EXPORT_SYMBOL_GPL(pci_reset_function);
4262 
4263 /**
4264  * pci_reset_function_locked - quiesce and reset a PCI device function
4265  * @dev: PCI device to reset
4266  *
4267  * Some devices allow an individual function to be reset without affecting
4268  * other functions in the same device.  The PCI device must be responsive
4269  * to PCI config space in order to use this function.
4270  *
4271  * This function does not just reset the PCI portion of a device, but
4272  * clears all the state associated with the device.  This function differs
4273  * from __pci_reset_function() in that it saves and restores device state
4274  * over the reset.  It also differs from pci_reset_function() in that it
4275  * requires the PCI device lock to be held.
4276  *
4277  * Returns 0 if the device function was successfully reset or negative if the
4278  * device doesn't support resetting a single function.
4279  */
4280 int pci_reset_function_locked(struct pci_dev *dev)
4281 {
4282 	int rc;
4283 
4284 	rc = pci_probe_reset_function(dev);
4285 	if (rc)
4286 		return rc;
4287 
4288 	pci_dev_save_and_disable(dev);
4289 
4290 	rc = __pci_reset_function_locked(dev);
4291 
4292 	pci_dev_restore(dev);
4293 
4294 	return rc;
4295 }
4296 EXPORT_SYMBOL_GPL(pci_reset_function_locked);
4297 
4298 /**
4299  * pci_try_reset_function - quiesce and reset a PCI device function
4300  * @dev: PCI device to reset
4301  *
4302  * Same as above, except return -EAGAIN if unable to lock device.
4303  */
4304 int pci_try_reset_function(struct pci_dev *dev)
4305 {
4306 	int rc;
4307 
4308 	rc = pci_probe_reset_function(dev);
4309 	if (rc)
4310 		return rc;
4311 
4312 	if (!pci_dev_trylock(dev))
4313 		return -EAGAIN;
4314 
4315 	pci_dev_save_and_disable(dev);
4316 	rc = __pci_reset_function_locked(dev);
4317 	pci_dev_unlock(dev);
4318 
4319 	pci_dev_restore(dev);
4320 	return rc;
4321 }
4322 EXPORT_SYMBOL_GPL(pci_try_reset_function);
4323 
4324 /* Do any devices on or below this bus prevent a bus reset? */
4325 static bool pci_bus_resetable(struct pci_bus *bus)
4326 {
4327 	struct pci_dev *dev;
4328 
4329 	list_for_each_entry(dev, &bus->devices, bus_list) {
4330 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
4331 		    (dev->subordinate && !pci_bus_resetable(dev->subordinate)))
4332 			return false;
4333 	}
4334 
4335 	return true;
4336 }
4337 
4338 /* Lock devices from the top of the tree down */
4339 static void pci_bus_lock(struct pci_bus *bus)
4340 {
4341 	struct pci_dev *dev;
4342 
4343 	list_for_each_entry(dev, &bus->devices, bus_list) {
4344 		pci_dev_lock(dev);
4345 		if (dev->subordinate)
4346 			pci_bus_lock(dev->subordinate);
4347 	}
4348 }
4349 
4350 /* Unlock devices from the bottom of the tree up */
4351 static void pci_bus_unlock(struct pci_bus *bus)
4352 {
4353 	struct pci_dev *dev;
4354 
4355 	list_for_each_entry(dev, &bus->devices, bus_list) {
4356 		if (dev->subordinate)
4357 			pci_bus_unlock(dev->subordinate);
4358 		pci_dev_unlock(dev);
4359 	}
4360 }
4361 
4362 /* Return 1 on successful lock, 0 on contention */
4363 static int pci_bus_trylock(struct pci_bus *bus)
4364 {
4365 	struct pci_dev *dev;
4366 
4367 	list_for_each_entry(dev, &bus->devices, bus_list) {
4368 		if (!pci_dev_trylock(dev))
4369 			goto unlock;
4370 		if (dev->subordinate) {
4371 			if (!pci_bus_trylock(dev->subordinate)) {
4372 				pci_dev_unlock(dev);
4373 				goto unlock;
4374 			}
4375 		}
4376 	}
4377 	return 1;
4378 
4379 unlock:
4380 	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
4381 		if (dev->subordinate)
4382 			pci_bus_unlock(dev->subordinate);
4383 		pci_dev_unlock(dev);
4384 	}
4385 	return 0;
4386 }
4387 
4388 /* Do any devices on or below this slot prevent a bus reset? */
4389 static bool pci_slot_resetable(struct pci_slot *slot)
4390 {
4391 	struct pci_dev *dev;
4392 
4393 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
4394 		if (!dev->slot || dev->slot != slot)
4395 			continue;
4396 		if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
4397 		    (dev->subordinate && !pci_bus_resetable(dev->subordinate)))
4398 			return false;
4399 	}
4400 
4401 	return true;
4402 }
4403 
4404 /* Lock devices from the top of the tree down */
4405 static void pci_slot_lock(struct pci_slot *slot)
4406 {
4407 	struct pci_dev *dev;
4408 
4409 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
4410 		if (!dev->slot || dev->slot != slot)
4411 			continue;
4412 		pci_dev_lock(dev);
4413 		if (dev->subordinate)
4414 			pci_bus_lock(dev->subordinate);
4415 	}
4416 }
4417 
4418 /* Unlock devices from the bottom of the tree up */
4419 static void pci_slot_unlock(struct pci_slot *slot)
4420 {
4421 	struct pci_dev *dev;
4422 
4423 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
4424 		if (!dev->slot || dev->slot != slot)
4425 			continue;
4426 		if (dev->subordinate)
4427 			pci_bus_unlock(dev->subordinate);
4428 		pci_dev_unlock(dev);
4429 	}
4430 }
4431 
4432 /* Return 1 on successful lock, 0 on contention */
4433 static int pci_slot_trylock(struct pci_slot *slot)
4434 {
4435 	struct pci_dev *dev;
4436 
4437 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
4438 		if (!dev->slot || dev->slot != slot)
4439 			continue;
4440 		if (!pci_dev_trylock(dev))
4441 			goto unlock;
4442 		if (dev->subordinate) {
4443 			if (!pci_bus_trylock(dev->subordinate)) {
4444 				pci_dev_unlock(dev);
4445 				goto unlock;
4446 			}
4447 		}
4448 	}
4449 	return 1;
4450 
4451 unlock:
4452 	list_for_each_entry_continue_reverse(dev,
4453 					     &slot->bus->devices, bus_list) {
4454 		if (!dev->slot || dev->slot != slot)
4455 			continue;
4456 		if (dev->subordinate)
4457 			pci_bus_unlock(dev->subordinate);
4458 		pci_dev_unlock(dev);
4459 	}
4460 	return 0;
4461 }
4462 
4463 /* Save and disable devices from the top of the tree down */
4464 static void pci_bus_save_and_disable(struct pci_bus *bus)
4465 {
4466 	struct pci_dev *dev;
4467 
4468 	list_for_each_entry(dev, &bus->devices, bus_list) {
4469 		pci_dev_lock(dev);
4470 		pci_dev_save_and_disable(dev);
4471 		pci_dev_unlock(dev);
4472 		if (dev->subordinate)
4473 			pci_bus_save_and_disable(dev->subordinate);
4474 	}
4475 }
4476 
4477 /*
4478  * Restore devices from top of the tree down - parent bridges need to be
4479  * restored before we can get to subordinate devices.
4480  */
4481 static void pci_bus_restore(struct pci_bus *bus)
4482 {
4483 	struct pci_dev *dev;
4484 
4485 	list_for_each_entry(dev, &bus->devices, bus_list) {
4486 		pci_dev_lock(dev);
4487 		pci_dev_restore(dev);
4488 		pci_dev_unlock(dev);
4489 		if (dev->subordinate)
4490 			pci_bus_restore(dev->subordinate);
4491 	}
4492 }
4493 
4494 /* Save and disable devices from the top of the tree down */
4495 static void pci_slot_save_and_disable(struct pci_slot *slot)
4496 {
4497 	struct pci_dev *dev;
4498 
4499 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
4500 		if (!dev->slot || dev->slot != slot)
4501 			continue;
4502 		pci_dev_save_and_disable(dev);
4503 		if (dev->subordinate)
4504 			pci_bus_save_and_disable(dev->subordinate);
4505 	}
4506 }
4507 
4508 /*
4509  * Restore devices from top of the tree down - parent bridges need to be
4510  * restored before we can get to subordinate devices.
4511  */
4512 static void pci_slot_restore(struct pci_slot *slot)
4513 {
4514 	struct pci_dev *dev;
4515 
4516 	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
4517 		if (!dev->slot || dev->slot != slot)
4518 			continue;
4519 		pci_dev_restore(dev);
4520 		if (dev->subordinate)
4521 			pci_bus_restore(dev->subordinate);
4522 	}
4523 }
4524 
4525 static int pci_slot_reset(struct pci_slot *slot, int probe)
4526 {
4527 	int rc;
4528 
4529 	if (!slot || !pci_slot_resetable(slot))
4530 		return -ENOTTY;
4531 
4532 	if (!probe)
4533 		pci_slot_lock(slot);
4534 
4535 	might_sleep();
4536 
4537 	rc = pci_reset_hotplug_slot(slot->hotplug, probe);
4538 
4539 	if (!probe)
4540 		pci_slot_unlock(slot);
4541 
4542 	return rc;
4543 }
4544 
4545 /**
4546  * pci_probe_reset_slot - probe whether a PCI slot can be reset
4547  * @slot: PCI slot to probe
4548  *
4549  * Return 0 if slot can be reset, negative if a slot reset is not supported.
4550  */
4551 int pci_probe_reset_slot(struct pci_slot *slot)
4552 {
4553 	return pci_slot_reset(slot, 1);
4554 }
4555 EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
4556 
4557 /**
4558  * pci_reset_slot - reset a PCI slot
4559  * @slot: PCI slot to reset
4560  *
4561  * A PCI bus may host multiple slots, each slot may support a reset mechanism
4562  * independent of other slots.  For instance, some slots may support slot power
4563  * control.  In the case of a 1:1 bus to slot architecture, this function may
4564  * wrap the bus reset to avoid spurious slot related events such as hotplug.
4565  * Generally a slot reset should be attempted before a bus reset.  All of the
4566  * function of the slot and any subordinate buses behind the slot are reset
4567  * through this function.  PCI config space of all devices in the slot and
4568  * behind the slot is saved before and restored after reset.
4569  *
4570  * Return 0 on success, non-zero on error.
4571  */
4572 int pci_reset_slot(struct pci_slot *slot)
4573 {
4574 	int rc;
4575 
4576 	rc = pci_slot_reset(slot, 1);
4577 	if (rc)
4578 		return rc;
4579 
4580 	pci_slot_save_and_disable(slot);
4581 
4582 	rc = pci_slot_reset(slot, 0);
4583 
4584 	pci_slot_restore(slot);
4585 
4586 	return rc;
4587 }
4588 EXPORT_SYMBOL_GPL(pci_reset_slot);
4589 
4590 /**
4591  * pci_try_reset_slot - Try to reset a PCI slot
4592  * @slot: PCI slot to reset
4593  *
4594  * Same as above except return -EAGAIN if the slot cannot be locked
4595  */
4596 int pci_try_reset_slot(struct pci_slot *slot)
4597 {
4598 	int rc;
4599 
4600 	rc = pci_slot_reset(slot, 1);
4601 	if (rc)
4602 		return rc;
4603 
4604 	pci_slot_save_and_disable(slot);
4605 
4606 	if (pci_slot_trylock(slot)) {
4607 		might_sleep();
4608 		rc = pci_reset_hotplug_slot(slot->hotplug, 0);
4609 		pci_slot_unlock(slot);
4610 	} else
4611 		rc = -EAGAIN;
4612 
4613 	pci_slot_restore(slot);
4614 
4615 	return rc;
4616 }
4617 EXPORT_SYMBOL_GPL(pci_try_reset_slot);
4618 
4619 static int pci_bus_reset(struct pci_bus *bus, int probe)
4620 {
4621 	if (!bus->self || !pci_bus_resetable(bus))
4622 		return -ENOTTY;
4623 
4624 	if (probe)
4625 		return 0;
4626 
4627 	pci_bus_lock(bus);
4628 
4629 	might_sleep();
4630 
4631 	pci_reset_bridge_secondary_bus(bus->self);
4632 
4633 	pci_bus_unlock(bus);
4634 
4635 	return 0;
4636 }
4637 
4638 /**
4639  * pci_probe_reset_bus - probe whether a PCI bus can be reset
4640  * @bus: PCI bus to probe
4641  *
4642  * Return 0 if bus can be reset, negative if a bus reset is not supported.
4643  */
4644 int pci_probe_reset_bus(struct pci_bus *bus)
4645 {
4646 	return pci_bus_reset(bus, 1);
4647 }
4648 EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
4649 
4650 /**
4651  * pci_reset_bus - reset a PCI bus
4652  * @bus: top level PCI bus to reset
4653  *
4654  * Do a bus reset on the given bus and any subordinate buses, saving
4655  * and restoring state of all devices.
4656  *
4657  * Return 0 on success, non-zero on error.
4658  */
4659 int pci_reset_bus(struct pci_bus *bus)
4660 {
4661 	int rc;
4662 
4663 	rc = pci_bus_reset(bus, 1);
4664 	if (rc)
4665 		return rc;
4666 
4667 	pci_bus_save_and_disable(bus);
4668 
4669 	rc = pci_bus_reset(bus, 0);
4670 
4671 	pci_bus_restore(bus);
4672 
4673 	return rc;
4674 }
4675 EXPORT_SYMBOL_GPL(pci_reset_bus);
4676 
4677 /**
4678  * pci_try_reset_bus - Try to reset a PCI bus
4679  * @bus: top level PCI bus to reset
4680  *
4681  * Same as above except return -EAGAIN if the bus cannot be locked
4682  */
4683 int pci_try_reset_bus(struct pci_bus *bus)
4684 {
4685 	int rc;
4686 
4687 	rc = pci_bus_reset(bus, 1);
4688 	if (rc)
4689 		return rc;
4690 
4691 	pci_bus_save_and_disable(bus);
4692 
4693 	if (pci_bus_trylock(bus)) {
4694 		might_sleep();
4695 		pci_reset_bridge_secondary_bus(bus->self);
4696 		pci_bus_unlock(bus);
4697 	} else
4698 		rc = -EAGAIN;
4699 
4700 	pci_bus_restore(bus);
4701 
4702 	return rc;
4703 }
4704 EXPORT_SYMBOL_GPL(pci_try_reset_bus);
4705 
4706 /**
4707  * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
4708  * @dev: PCI device to query
4709  *
4710  * Returns mmrbc: maximum designed memory read count in bytes
4711  *    or appropriate error value.
4712  */
4713 int pcix_get_max_mmrbc(struct pci_dev *dev)
4714 {
4715 	int cap;
4716 	u32 stat;
4717 
4718 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
4719 	if (!cap)
4720 		return -EINVAL;
4721 
4722 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
4723 		return -EINVAL;
4724 
4725 	return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21);
4726 }
4727 EXPORT_SYMBOL(pcix_get_max_mmrbc);
4728 
4729 /**
4730  * pcix_get_mmrbc - get PCI-X maximum memory read byte count
4731  * @dev: PCI device to query
4732  *
4733  * Returns mmrbc: maximum memory read count in bytes
4734  *    or appropriate error value.
4735  */
4736 int pcix_get_mmrbc(struct pci_dev *dev)
4737 {
4738 	int cap;
4739 	u16 cmd;
4740 
4741 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
4742 	if (!cap)
4743 		return -EINVAL;
4744 
4745 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
4746 		return -EINVAL;
4747 
4748 	return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2);
4749 }
4750 EXPORT_SYMBOL(pcix_get_mmrbc);
4751 
4752 /**
4753  * pcix_set_mmrbc - set PCI-X maximum memory read byte count
4754  * @dev: PCI device to query
4755  * @mmrbc: maximum memory read count in bytes
4756  *    valid values are 512, 1024, 2048, 4096
4757  *
4758  * If possible sets maximum memory read byte count, some bridges have erratas
4759  * that prevent this.
4760  */
4761 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
4762 {
4763 	int cap;
4764 	u32 stat, v, o;
4765 	u16 cmd;
4766 
4767 	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc))
4768 		return -EINVAL;
4769 
4770 	v = ffs(mmrbc) - 10;
4771 
4772 	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
4773 	if (!cap)
4774 		return -EINVAL;
4775 
4776 	if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat))
4777 		return -EINVAL;
4778 
4779 	if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21)
4780 		return -E2BIG;
4781 
4782 	if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd))
4783 		return -EINVAL;
4784 
4785 	o = (cmd & PCI_X_CMD_MAX_READ) >> 2;
4786 	if (o != v) {
4787 		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
4788 			return -EIO;
4789 
4790 		cmd &= ~PCI_X_CMD_MAX_READ;
4791 		cmd |= v << 2;
4792 		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
4793 			return -EIO;
4794 	}
4795 	return 0;
4796 }
4797 EXPORT_SYMBOL(pcix_set_mmrbc);
4798 
4799 /**
4800  * pcie_get_readrq - get PCI Express read request size
4801  * @dev: PCI device to query
4802  *
4803  * Returns maximum memory read request in bytes
4804  *    or appropriate error value.
4805  */
4806 int pcie_get_readrq(struct pci_dev *dev)
4807 {
4808 	u16 ctl;
4809 
4810 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
4811 
4812 	return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12);
4813 }
4814 EXPORT_SYMBOL(pcie_get_readrq);
4815 
4816 /**
4817  * pcie_set_readrq - set PCI Express maximum memory read request
4818  * @dev: PCI device to query
4819  * @rq: maximum memory read count in bytes
4820  *    valid values are 128, 256, 512, 1024, 2048, 4096
4821  *
4822  * If possible sets maximum memory read request in bytes
4823  */
4824 int pcie_set_readrq(struct pci_dev *dev, int rq)
4825 {
4826 	u16 v;
4827 
4828 	if (rq < 128 || rq > 4096 || !is_power_of_2(rq))
4829 		return -EINVAL;
4830 
4831 	/*
4832 	 * If using the "performance" PCIe config, we clamp the
4833 	 * read rq size to the max packet size to prevent the
4834 	 * host bridge generating requests larger than we can
4835 	 * cope with
4836 	 */
4837 	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
4838 		int mps = pcie_get_mps(dev);
4839 
4840 		if (mps < rq)
4841 			rq = mps;
4842 	}
4843 
4844 	v = (ffs(rq) - 8) << 12;
4845 
4846 	return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
4847 						  PCI_EXP_DEVCTL_READRQ, v);
4848 }
4849 EXPORT_SYMBOL(pcie_set_readrq);
4850 
4851 /**
4852  * pcie_get_mps - get PCI Express maximum payload size
4853  * @dev: PCI device to query
4854  *
4855  * Returns maximum payload size in bytes
4856  */
4857 int pcie_get_mps(struct pci_dev *dev)
4858 {
4859 	u16 ctl;
4860 
4861 	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl);
4862 
4863 	return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5);
4864 }
4865 EXPORT_SYMBOL(pcie_get_mps);
4866 
4867 /**
4868  * pcie_set_mps - set PCI Express maximum payload size
4869  * @dev: PCI device to query
4870  * @mps: maximum payload size in bytes
4871  *    valid values are 128, 256, 512, 1024, 2048, 4096
4872  *
4873  * If possible sets maximum payload size
4874  */
4875 int pcie_set_mps(struct pci_dev *dev, int mps)
4876 {
4877 	u16 v;
4878 
4879 	if (mps < 128 || mps > 4096 || !is_power_of_2(mps))
4880 		return -EINVAL;
4881 
4882 	v = ffs(mps) - 8;
4883 	if (v > dev->pcie_mpss)
4884 		return -EINVAL;
4885 	v <<= 5;
4886 
4887 	return pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
4888 						  PCI_EXP_DEVCTL_PAYLOAD, v);
4889 }
4890 EXPORT_SYMBOL(pcie_set_mps);
4891 
4892 /**
4893  * pcie_get_minimum_link - determine minimum link settings of a PCI device
4894  * @dev: PCI device to query
4895  * @speed: storage for minimum speed
4896  * @width: storage for minimum width
4897  *
4898  * This function will walk up the PCI device chain and determine the minimum
4899  * link width and speed of the device.
4900  */
4901 int pcie_get_minimum_link(struct pci_dev *dev, enum pci_bus_speed *speed,
4902 			  enum pcie_link_width *width)
4903 {
4904 	int ret;
4905 
4906 	*speed = PCI_SPEED_UNKNOWN;
4907 	*width = PCIE_LNK_WIDTH_UNKNOWN;
4908 
4909 	while (dev) {
4910 		u16 lnksta;
4911 		enum pci_bus_speed next_speed;
4912 		enum pcie_link_width next_width;
4913 
4914 		ret = pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
4915 		if (ret)
4916 			return ret;
4917 
4918 		next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS];
4919 		next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >>
4920 			PCI_EXP_LNKSTA_NLW_SHIFT;
4921 
4922 		if (next_speed < *speed)
4923 			*speed = next_speed;
4924 
4925 		if (next_width < *width)
4926 			*width = next_width;
4927 
4928 		dev = dev->bus->self;
4929 	}
4930 
4931 	return 0;
4932 }
4933 EXPORT_SYMBOL(pcie_get_minimum_link);
4934 
4935 /**
4936  * pci_select_bars - Make BAR mask from the type of resource
4937  * @dev: the PCI device for which BAR mask is made
4938  * @flags: resource type mask to be selected
4939  *
4940  * This helper routine makes bar mask from the type of resource.
4941  */
4942 int pci_select_bars(struct pci_dev *dev, unsigned long flags)
4943 {
4944 	int i, bars = 0;
4945 	for (i = 0; i < PCI_NUM_RESOURCES; i++)
4946 		if (pci_resource_flags(dev, i) & flags)
4947 			bars |= (1 << i);
4948 	return bars;
4949 }
4950 EXPORT_SYMBOL(pci_select_bars);
4951 
4952 /* Some architectures require additional programming to enable VGA */
4953 static arch_set_vga_state_t arch_set_vga_state;
4954 
4955 void __init pci_register_set_vga_state(arch_set_vga_state_t func)
4956 {
4957 	arch_set_vga_state = func;	/* NULL disables */
4958 }
4959 
4960 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
4961 				  unsigned int command_bits, u32 flags)
4962 {
4963 	if (arch_set_vga_state)
4964 		return arch_set_vga_state(dev, decode, command_bits,
4965 						flags);
4966 	return 0;
4967 }
4968 
4969 /**
4970  * pci_set_vga_state - set VGA decode state on device and parents if requested
4971  * @dev: the PCI device
4972  * @decode: true = enable decoding, false = disable decoding
4973  * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
4974  * @flags: traverse ancestors and change bridges
4975  * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
4976  */
4977 int pci_set_vga_state(struct pci_dev *dev, bool decode,
4978 		      unsigned int command_bits, u32 flags)
4979 {
4980 	struct pci_bus *bus;
4981 	struct pci_dev *bridge;
4982 	u16 cmd;
4983 	int rc;
4984 
4985 	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
4986 
4987 	/* ARCH specific VGA enables */
4988 	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
4989 	if (rc)
4990 		return rc;
4991 
4992 	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
4993 		pci_read_config_word(dev, PCI_COMMAND, &cmd);
4994 		if (decode == true)
4995 			cmd |= command_bits;
4996 		else
4997 			cmd &= ~command_bits;
4998 		pci_write_config_word(dev, PCI_COMMAND, cmd);
4999 	}
5000 
5001 	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
5002 		return 0;
5003 
5004 	bus = dev->bus;
5005 	while (bus) {
5006 		bridge = bus->self;
5007 		if (bridge) {
5008 			pci_read_config_word(bridge, PCI_BRIDGE_CONTROL,
5009 					     &cmd);
5010 			if (decode == true)
5011 				cmd |= PCI_BRIDGE_CTL_VGA;
5012 			else
5013 				cmd &= ~PCI_BRIDGE_CTL_VGA;
5014 			pci_write_config_word(bridge, PCI_BRIDGE_CONTROL,
5015 					      cmd);
5016 		}
5017 		bus = bus->parent;
5018 	}
5019 	return 0;
5020 }
5021 
5022 /**
5023  * pci_add_dma_alias - Add a DMA devfn alias for a device
5024  * @dev: the PCI device for which alias is added
5025  * @devfn: alias slot and function
5026  *
5027  * This helper encodes 8-bit devfn as bit number in dma_alias_mask.
5028  * It should be called early, preferably as PCI fixup header quirk.
5029  */
5030 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn)
5031 {
5032 	if (!dev->dma_alias_mask)
5033 		dev->dma_alias_mask = kcalloc(BITS_TO_LONGS(U8_MAX),
5034 					      sizeof(long), GFP_KERNEL);
5035 	if (!dev->dma_alias_mask) {
5036 		dev_warn(&dev->dev, "Unable to allocate DMA alias mask\n");
5037 		return;
5038 	}
5039 
5040 	set_bit(devfn, dev->dma_alias_mask);
5041 	dev_info(&dev->dev, "Enabling fixed DMA alias to %02x.%d\n",
5042 		 PCI_SLOT(devfn), PCI_FUNC(devfn));
5043 }
5044 
5045 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
5046 {
5047 	return (dev1->dma_alias_mask &&
5048 		test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
5049 	       (dev2->dma_alias_mask &&
5050 		test_bit(dev1->devfn, dev2->dma_alias_mask));
5051 }
5052 
5053 bool pci_device_is_present(struct pci_dev *pdev)
5054 {
5055 	u32 v;
5056 
5057 	if (pci_dev_is_disconnected(pdev))
5058 		return false;
5059 	return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0);
5060 }
5061 EXPORT_SYMBOL_GPL(pci_device_is_present);
5062 
5063 void pci_ignore_hotplug(struct pci_dev *dev)
5064 {
5065 	struct pci_dev *bridge = dev->bus->self;
5066 
5067 	dev->ignore_hotplug = 1;
5068 	/* Propagate the "ignore hotplug" setting to the parent bridge. */
5069 	if (bridge)
5070 		bridge->ignore_hotplug = 1;
5071 }
5072 EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
5073 
5074 resource_size_t __weak pcibios_default_alignment(void)
5075 {
5076 	return 0;
5077 }
5078 
5079 #define RESOURCE_ALIGNMENT_PARAM_SIZE COMMAND_LINE_SIZE
5080 static char resource_alignment_param[RESOURCE_ALIGNMENT_PARAM_SIZE] = {0};
5081 static DEFINE_SPINLOCK(resource_alignment_lock);
5082 
5083 /**
5084  * pci_specified_resource_alignment - get resource alignment specified by user.
5085  * @dev: the PCI device to get
5086  * @resize: whether or not to change resources' size when reassigning alignment
5087  *
5088  * RETURNS: Resource alignment if it is specified.
5089  *          Zero if it is not specified.
5090  */
5091 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
5092 							bool *resize)
5093 {
5094 	int seg, bus, slot, func, align_order, count;
5095 	unsigned short vendor, device, subsystem_vendor, subsystem_device;
5096 	resource_size_t align = pcibios_default_alignment();
5097 	char *p;
5098 
5099 	spin_lock(&resource_alignment_lock);
5100 	p = resource_alignment_param;
5101 	if (!*p && !align)
5102 		goto out;
5103 	if (pci_has_flag(PCI_PROBE_ONLY)) {
5104 		align = 0;
5105 		pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
5106 		goto out;
5107 	}
5108 
5109 	while (*p) {
5110 		count = 0;
5111 		if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
5112 							p[count] == '@') {
5113 			p += count + 1;
5114 		} else {
5115 			align_order = -1;
5116 		}
5117 		if (strncmp(p, "pci:", 4) == 0) {
5118 			/* PCI vendor/device (subvendor/subdevice) ids are specified */
5119 			p += 4;
5120 			if (sscanf(p, "%hx:%hx:%hx:%hx%n",
5121 				&vendor, &device, &subsystem_vendor, &subsystem_device, &count) != 4) {
5122 				if (sscanf(p, "%hx:%hx%n", &vendor, &device, &count) != 2) {
5123 					printk(KERN_ERR "PCI: Can't parse resource_alignment parameter: pci:%s\n",
5124 						p);
5125 					break;
5126 				}
5127 				subsystem_vendor = subsystem_device = 0;
5128 			}
5129 			p += count;
5130 			if ((!vendor || (vendor == dev->vendor)) &&
5131 				(!device || (device == dev->device)) &&
5132 				(!subsystem_vendor || (subsystem_vendor == dev->subsystem_vendor)) &&
5133 				(!subsystem_device || (subsystem_device == dev->subsystem_device))) {
5134 				*resize = true;
5135 				if (align_order == -1)
5136 					align = PAGE_SIZE;
5137 				else
5138 					align = 1 << align_order;
5139 				/* Found */
5140 				break;
5141 			}
5142 		}
5143 		else {
5144 			if (sscanf(p, "%x:%x:%x.%x%n",
5145 				&seg, &bus, &slot, &func, &count) != 4) {
5146 				seg = 0;
5147 				if (sscanf(p, "%x:%x.%x%n",
5148 						&bus, &slot, &func, &count) != 3) {
5149 					/* Invalid format */
5150 					printk(KERN_ERR "PCI: Can't parse resource_alignment parameter: %s\n",
5151 						p);
5152 					break;
5153 				}
5154 			}
5155 			p += count;
5156 			if (seg == pci_domain_nr(dev->bus) &&
5157 				bus == dev->bus->number &&
5158 				slot == PCI_SLOT(dev->devfn) &&
5159 				func == PCI_FUNC(dev->devfn)) {
5160 				*resize = true;
5161 				if (align_order == -1)
5162 					align = PAGE_SIZE;
5163 				else
5164 					align = 1 << align_order;
5165 				/* Found */
5166 				break;
5167 			}
5168 		}
5169 		if (*p != ';' && *p != ',') {
5170 			/* End of param or invalid format */
5171 			break;
5172 		}
5173 		p++;
5174 	}
5175 out:
5176 	spin_unlock(&resource_alignment_lock);
5177 	return align;
5178 }
5179 
5180 static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
5181 					   resource_size_t align, bool resize)
5182 {
5183 	struct resource *r = &dev->resource[bar];
5184 	resource_size_t size;
5185 
5186 	if (!(r->flags & IORESOURCE_MEM))
5187 		return;
5188 
5189 	if (r->flags & IORESOURCE_PCI_FIXED) {
5190 		dev_info(&dev->dev, "BAR%d %pR: ignoring requested alignment %#llx\n",
5191 			 bar, r, (unsigned long long)align);
5192 		return;
5193 	}
5194 
5195 	size = resource_size(r);
5196 	if (size >= align)
5197 		return;
5198 
5199 	/*
5200 	 * Increase the alignment of the resource.  There are two ways we
5201 	 * can do this:
5202 	 *
5203 	 * 1) Increase the size of the resource.  BARs are aligned on their
5204 	 *    size, so when we reallocate space for this resource, we'll
5205 	 *    allocate it with the larger alignment.  This also prevents
5206 	 *    assignment of any other BARs inside the alignment region, so
5207 	 *    if we're requesting page alignment, this means no other BARs
5208 	 *    will share the page.
5209 	 *
5210 	 *    The disadvantage is that this makes the resource larger than
5211 	 *    the hardware BAR, which may break drivers that compute things
5212 	 *    based on the resource size, e.g., to find registers at a
5213 	 *    fixed offset before the end of the BAR.
5214 	 *
5215 	 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
5216 	 *    set r->start to the desired alignment.  By itself this
5217 	 *    doesn't prevent other BARs being put inside the alignment
5218 	 *    region, but if we realign *every* resource of every device in
5219 	 *    the system, none of them will share an alignment region.
5220 	 *
5221 	 * When the user has requested alignment for only some devices via
5222 	 * the "pci=resource_alignment" argument, "resize" is true and we
5223 	 * use the first method.  Otherwise we assume we're aligning all
5224 	 * devices and we use the second.
5225 	 */
5226 
5227 	dev_info(&dev->dev, "BAR%d %pR: requesting alignment to %#llx\n",
5228 		 bar, r, (unsigned long long)align);
5229 
5230 	if (resize) {
5231 		r->start = 0;
5232 		r->end = align - 1;
5233 	} else {
5234 		r->flags &= ~IORESOURCE_SIZEALIGN;
5235 		r->flags |= IORESOURCE_STARTALIGN;
5236 		r->start = align;
5237 		r->end = r->start + size - 1;
5238 	}
5239 	r->flags |= IORESOURCE_UNSET;
5240 }
5241 
5242 /*
5243  * This function disables memory decoding and releases memory resources
5244  * of the device specified by kernel's boot parameter 'pci=resource_alignment='.
5245  * It also rounds up size to specified alignment.
5246  * Later on, the kernel will assign page-aligned memory resource back
5247  * to the device.
5248  */
5249 void pci_reassigndev_resource_alignment(struct pci_dev *dev)
5250 {
5251 	int i;
5252 	struct resource *r;
5253 	resource_size_t align;
5254 	u16 command;
5255 	bool resize = false;
5256 
5257 	/*
5258 	 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec
5259 	 * 3.4.1.11.  Their resources are allocated from the space
5260 	 * described by the VF BARx register in the PF's SR-IOV capability.
5261 	 * We can't influence their alignment here.
5262 	 */
5263 	if (dev->is_virtfn)
5264 		return;
5265 
5266 	/* check if specified PCI is target device to reassign */
5267 	align = pci_specified_resource_alignment(dev, &resize);
5268 	if (!align)
5269 		return;
5270 
5271 	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
5272 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
5273 		dev_warn(&dev->dev,
5274 			"Can't reassign resources to host bridge.\n");
5275 		return;
5276 	}
5277 
5278 	dev_info(&dev->dev,
5279 		"Disabling memory decoding and releasing memory resources.\n");
5280 	pci_read_config_word(dev, PCI_COMMAND, &command);
5281 	command &= ~PCI_COMMAND_MEMORY;
5282 	pci_write_config_word(dev, PCI_COMMAND, command);
5283 
5284 	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
5285 		pci_request_resource_alignment(dev, i, align, resize);
5286 
5287 	/*
5288 	 * Need to disable bridge's resource window,
5289 	 * to enable the kernel to reassign new resource
5290 	 * window later on.
5291 	 */
5292 	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE &&
5293 	    (dev->class >> 8) == PCI_CLASS_BRIDGE_PCI) {
5294 		for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
5295 			r = &dev->resource[i];
5296 			if (!(r->flags & IORESOURCE_MEM))
5297 				continue;
5298 			r->flags |= IORESOURCE_UNSET;
5299 			r->end = resource_size(r) - 1;
5300 			r->start = 0;
5301 		}
5302 		pci_disable_bridge_window(dev);
5303 	}
5304 }
5305 
5306 static ssize_t pci_set_resource_alignment_param(const char *buf, size_t count)
5307 {
5308 	if (count > RESOURCE_ALIGNMENT_PARAM_SIZE - 1)
5309 		count = RESOURCE_ALIGNMENT_PARAM_SIZE - 1;
5310 	spin_lock(&resource_alignment_lock);
5311 	strncpy(resource_alignment_param, buf, count);
5312 	resource_alignment_param[count] = '\0';
5313 	spin_unlock(&resource_alignment_lock);
5314 	return count;
5315 }
5316 
5317 static ssize_t pci_get_resource_alignment_param(char *buf, size_t size)
5318 {
5319 	size_t count;
5320 	spin_lock(&resource_alignment_lock);
5321 	count = snprintf(buf, size, "%s", resource_alignment_param);
5322 	spin_unlock(&resource_alignment_lock);
5323 	return count;
5324 }
5325 
5326 static ssize_t pci_resource_alignment_show(struct bus_type *bus, char *buf)
5327 {
5328 	return pci_get_resource_alignment_param(buf, PAGE_SIZE);
5329 }
5330 
5331 static ssize_t pci_resource_alignment_store(struct bus_type *bus,
5332 					const char *buf, size_t count)
5333 {
5334 	return pci_set_resource_alignment_param(buf, count);
5335 }
5336 
5337 static BUS_ATTR(resource_alignment, 0644, pci_resource_alignment_show,
5338 					pci_resource_alignment_store);
5339 
5340 static int __init pci_resource_alignment_sysfs_init(void)
5341 {
5342 	return bus_create_file(&pci_bus_type,
5343 					&bus_attr_resource_alignment);
5344 }
5345 late_initcall(pci_resource_alignment_sysfs_init);
5346 
5347 static void pci_no_domains(void)
5348 {
5349 #ifdef CONFIG_PCI_DOMAINS
5350 	pci_domains_supported = 0;
5351 #endif
5352 }
5353 
5354 #ifdef CONFIG_PCI_DOMAINS
5355 static atomic_t __domain_nr = ATOMIC_INIT(-1);
5356 
5357 int pci_get_new_domain_nr(void)
5358 {
5359 	return atomic_inc_return(&__domain_nr);
5360 }
5361 
5362 #ifdef CONFIG_PCI_DOMAINS_GENERIC
5363 static int of_pci_bus_find_domain_nr(struct device *parent)
5364 {
5365 	static int use_dt_domains = -1;
5366 	int domain = -1;
5367 
5368 	if (parent)
5369 		domain = of_get_pci_domain_nr(parent->of_node);
5370 	/*
5371 	 * Check DT domain and use_dt_domains values.
5372 	 *
5373 	 * If DT domain property is valid (domain >= 0) and
5374 	 * use_dt_domains != 0, the DT assignment is valid since this means
5375 	 * we have not previously allocated a domain number by using
5376 	 * pci_get_new_domain_nr(); we should also update use_dt_domains to
5377 	 * 1, to indicate that we have just assigned a domain number from
5378 	 * DT.
5379 	 *
5380 	 * If DT domain property value is not valid (ie domain < 0), and we
5381 	 * have not previously assigned a domain number from DT
5382 	 * (use_dt_domains != 1) we should assign a domain number by
5383 	 * using the:
5384 	 *
5385 	 * pci_get_new_domain_nr()
5386 	 *
5387 	 * API and update the use_dt_domains value to keep track of method we
5388 	 * are using to assign domain numbers (use_dt_domains = 0).
5389 	 *
5390 	 * All other combinations imply we have a platform that is trying
5391 	 * to mix domain numbers obtained from DT and pci_get_new_domain_nr(),
5392 	 * which is a recipe for domain mishandling and it is prevented by
5393 	 * invalidating the domain value (domain = -1) and printing a
5394 	 * corresponding error.
5395 	 */
5396 	if (domain >= 0 && use_dt_domains) {
5397 		use_dt_domains = 1;
5398 	} else if (domain < 0 && use_dt_domains != 1) {
5399 		use_dt_domains = 0;
5400 		domain = pci_get_new_domain_nr();
5401 	} else {
5402 		dev_err(parent, "Node %s has inconsistent \"linux,pci-domain\" property in DT\n",
5403 			parent->of_node->full_name);
5404 		domain = -1;
5405 	}
5406 
5407 	return domain;
5408 }
5409 
5410 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
5411 {
5412 	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
5413 			       acpi_pci_bus_find_domain_nr(bus);
5414 }
5415 #endif
5416 #endif
5417 
5418 /**
5419  * pci_ext_cfg_avail - can we access extended PCI config space?
5420  *
5421  * Returns 1 if we can access PCI extended config space (offsets
5422  * greater than 0xff). This is the default implementation. Architecture
5423  * implementations can override this.
5424  */
5425 int __weak pci_ext_cfg_avail(void)
5426 {
5427 	return 1;
5428 }
5429 
5430 void __weak pci_fixup_cardbus(struct pci_bus *bus)
5431 {
5432 }
5433 EXPORT_SYMBOL(pci_fixup_cardbus);
5434 
5435 static int __init pci_setup(char *str)
5436 {
5437 	while (str) {
5438 		char *k = strchr(str, ',');
5439 		if (k)
5440 			*k++ = 0;
5441 		if (*str && (str = pcibios_setup(str)) && *str) {
5442 			if (!strcmp(str, "nomsi")) {
5443 				pci_no_msi();
5444 			} else if (!strcmp(str, "noaer")) {
5445 				pci_no_aer();
5446 			} else if (!strncmp(str, "realloc=", 8)) {
5447 				pci_realloc_get_opt(str + 8);
5448 			} else if (!strncmp(str, "realloc", 7)) {
5449 				pci_realloc_get_opt("on");
5450 			} else if (!strcmp(str, "nodomains")) {
5451 				pci_no_domains();
5452 			} else if (!strncmp(str, "noari", 5)) {
5453 				pcie_ari_disabled = true;
5454 			} else if (!strncmp(str, "cbiosize=", 9)) {
5455 				pci_cardbus_io_size = memparse(str + 9, &str);
5456 			} else if (!strncmp(str, "cbmemsize=", 10)) {
5457 				pci_cardbus_mem_size = memparse(str + 10, &str);
5458 			} else if (!strncmp(str, "resource_alignment=", 19)) {
5459 				pci_set_resource_alignment_param(str + 19,
5460 							strlen(str + 19));
5461 			} else if (!strncmp(str, "ecrc=", 5)) {
5462 				pcie_ecrc_get_policy(str + 5);
5463 			} else if (!strncmp(str, "hpiosize=", 9)) {
5464 				pci_hotplug_io_size = memparse(str + 9, &str);
5465 			} else if (!strncmp(str, "hpmemsize=", 10)) {
5466 				pci_hotplug_mem_size = memparse(str + 10, &str);
5467 			} else if (!strncmp(str, "hpbussize=", 10)) {
5468 				pci_hotplug_bus_size =
5469 					simple_strtoul(str + 10, &str, 0);
5470 				if (pci_hotplug_bus_size > 0xff)
5471 					pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
5472 			} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
5473 				pcie_bus_config = PCIE_BUS_TUNE_OFF;
5474 			} else if (!strncmp(str, "pcie_bus_safe", 13)) {
5475 				pcie_bus_config = PCIE_BUS_SAFE;
5476 			} else if (!strncmp(str, "pcie_bus_perf", 13)) {
5477 				pcie_bus_config = PCIE_BUS_PERFORMANCE;
5478 			} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
5479 				pcie_bus_config = PCIE_BUS_PEER2PEER;
5480 			} else if (!strncmp(str, "pcie_scan_all", 13)) {
5481 				pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS);
5482 			} else {
5483 				printk(KERN_ERR "PCI: Unknown option `%s'\n",
5484 						str);
5485 			}
5486 		}
5487 		str = k;
5488 	}
5489 	return 0;
5490 }
5491 early_param("pci", pci_setup);
5492