1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Bus Services, see include/linux/pci.h for further explanation. 4 * 5 * Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter, 6 * David Mosberger-Tang 7 * 8 * Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz> 9 */ 10 11 #include <linux/acpi.h> 12 #include <linux/kernel.h> 13 #include <linux/delay.h> 14 #include <linux/dmi.h> 15 #include <linux/init.h> 16 #include <linux/msi.h> 17 #include <linux/of.h> 18 #include <linux/pci.h> 19 #include <linux/pm.h> 20 #include <linux/slab.h> 21 #include <linux/module.h> 22 #include <linux/spinlock.h> 23 #include <linux/string.h> 24 #include <linux/log2.h> 25 #include <linux/logic_pio.h> 26 #include <linux/pm_wakeup.h> 27 #include <linux/interrupt.h> 28 #include <linux/device.h> 29 #include <linux/pm_runtime.h> 30 #include <linux/pci_hotplug.h> 31 #include <linux/vmalloc.h> 32 #include <asm/dma.h> 33 #include <linux/aer.h> 34 #include "pci.h" 35 36 DEFINE_MUTEX(pci_slot_mutex); 37 38 const char *pci_power_names[] = { 39 "error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown", 40 }; 41 EXPORT_SYMBOL_GPL(pci_power_names); 42 43 int isa_dma_bridge_buggy; 44 EXPORT_SYMBOL(isa_dma_bridge_buggy); 45 46 int pci_pci_problems; 47 EXPORT_SYMBOL(pci_pci_problems); 48 49 unsigned int pci_pm_d3hot_delay; 50 51 static void pci_pme_list_scan(struct work_struct *work); 52 53 static LIST_HEAD(pci_pme_list); 54 static DEFINE_MUTEX(pci_pme_list_mutex); 55 static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan); 56 57 struct pci_pme_device { 58 struct list_head list; 59 struct pci_dev *dev; 60 }; 61 62 #define PME_TIMEOUT 1000 /* How long between PME checks */ 63 64 static void pci_dev_d3_sleep(struct pci_dev *dev) 65 { 66 unsigned int delay = dev->d3hot_delay; 67 68 if (delay < pci_pm_d3hot_delay) 69 delay = pci_pm_d3hot_delay; 70 71 if (delay) 72 msleep(delay); 73 } 74 75 #ifdef CONFIG_PCI_DOMAINS 76 int pci_domains_supported = 1; 77 #endif 78 79 #define DEFAULT_CARDBUS_IO_SIZE (256) 80 #define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024) 81 /* pci=cbmemsize=nnM,cbiosize=nn can override this */ 82 unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE; 83 unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE; 84 85 #define DEFAULT_HOTPLUG_IO_SIZE (256) 86 #define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024) 87 #define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024) 88 /* hpiosize=nn can override this */ 89 unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE; 90 /* 91 * pci=hpmmiosize=nnM overrides non-prefetchable MMIO size, 92 * pci=hpmmioprefsize=nnM overrides prefetchable MMIO size; 93 * pci=hpmemsize=nnM overrides both 94 */ 95 unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE; 96 unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE; 97 98 #define DEFAULT_HOTPLUG_BUS_SIZE 1 99 unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 100 101 102 /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */ 103 #ifdef CONFIG_PCIE_BUS_TUNE_OFF 104 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF; 105 #elif defined CONFIG_PCIE_BUS_SAFE 106 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE; 107 #elif defined CONFIG_PCIE_BUS_PERFORMANCE 108 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE; 109 #elif defined CONFIG_PCIE_BUS_PEER2PEER 110 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER; 111 #else 112 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT; 113 #endif 114 115 /* 116 * The default CLS is used if arch didn't set CLS explicitly and not 117 * all pci devices agree on the same value. Arch can override either 118 * the dfl or actual value as it sees fit. Don't forget this is 119 * measured in 32-bit words, not bytes. 120 */ 121 u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2; 122 u8 pci_cache_line_size; 123 124 /* 125 * If we set up a device for bus mastering, we need to check the latency 126 * timer as certain BIOSes forget to set it properly. 127 */ 128 unsigned int pcibios_max_latency = 255; 129 130 /* If set, the PCIe ARI capability will not be used. */ 131 static bool pcie_ari_disabled; 132 133 /* If set, the PCIe ATS capability will not be used. */ 134 static bool pcie_ats_disabled; 135 136 /* If set, the PCI config space of each device is printed during boot. */ 137 bool pci_early_dump; 138 139 bool pci_ats_disabled(void) 140 { 141 return pcie_ats_disabled; 142 } 143 EXPORT_SYMBOL_GPL(pci_ats_disabled); 144 145 /* Disable bridge_d3 for all PCIe ports */ 146 static bool pci_bridge_d3_disable; 147 /* Force bridge_d3 for all PCIe ports */ 148 static bool pci_bridge_d3_force; 149 150 static int __init pcie_port_pm_setup(char *str) 151 { 152 if (!strcmp(str, "off")) 153 pci_bridge_d3_disable = true; 154 else if (!strcmp(str, "force")) 155 pci_bridge_d3_force = true; 156 return 1; 157 } 158 __setup("pcie_port_pm=", pcie_port_pm_setup); 159 160 /* Time to wait after a reset for device to become responsive */ 161 #define PCIE_RESET_READY_POLL_MS 60000 162 163 /** 164 * pci_bus_max_busnr - returns maximum PCI bus number of given bus' children 165 * @bus: pointer to PCI bus structure to search 166 * 167 * Given a PCI bus, returns the highest PCI bus number present in the set 168 * including the given PCI bus and its list of child PCI buses. 169 */ 170 unsigned char pci_bus_max_busnr(struct pci_bus *bus) 171 { 172 struct pci_bus *tmp; 173 unsigned char max, n; 174 175 max = bus->busn_res.end; 176 list_for_each_entry(tmp, &bus->children, node) { 177 n = pci_bus_max_busnr(tmp); 178 if (n > max) 179 max = n; 180 } 181 return max; 182 } 183 EXPORT_SYMBOL_GPL(pci_bus_max_busnr); 184 185 /** 186 * pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS 187 * @pdev: the PCI device 188 * 189 * Returns error bits set in PCI_STATUS and clears them. 190 */ 191 int pci_status_get_and_clear_errors(struct pci_dev *pdev) 192 { 193 u16 status; 194 int ret; 195 196 ret = pci_read_config_word(pdev, PCI_STATUS, &status); 197 if (ret != PCIBIOS_SUCCESSFUL) 198 return -EIO; 199 200 status &= PCI_STATUS_ERROR_BITS; 201 if (status) 202 pci_write_config_word(pdev, PCI_STATUS, status); 203 204 return status; 205 } 206 EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors); 207 208 #ifdef CONFIG_HAS_IOMEM 209 void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar) 210 { 211 struct resource *res = &pdev->resource[bar]; 212 213 /* 214 * Make sure the BAR is actually a memory resource, not an IO resource 215 */ 216 if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) { 217 pci_warn(pdev, "can't ioremap BAR %d: %pR\n", bar, res); 218 return NULL; 219 } 220 return ioremap(res->start, resource_size(res)); 221 } 222 EXPORT_SYMBOL_GPL(pci_ioremap_bar); 223 224 void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar) 225 { 226 /* 227 * Make sure the BAR is actually a memory resource, not an IO resource 228 */ 229 if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM)) { 230 WARN_ON(1); 231 return NULL; 232 } 233 return ioremap_wc(pci_resource_start(pdev, bar), 234 pci_resource_len(pdev, bar)); 235 } 236 EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar); 237 #endif 238 239 /** 240 * pci_dev_str_match_path - test if a path string matches a device 241 * @dev: the PCI device to test 242 * @path: string to match the device against 243 * @endptr: pointer to the string after the match 244 * 245 * Test if a string (typically from a kernel parameter) formatted as a 246 * path of device/function addresses matches a PCI device. The string must 247 * be of the form: 248 * 249 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 250 * 251 * A path for a device can be obtained using 'lspci -t'. Using a path 252 * is more robust against bus renumbering than using only a single bus, 253 * device and function address. 254 * 255 * Returns 1 if the string matches the device, 0 if it does not and 256 * a negative error code if it fails to parse the string. 257 */ 258 static int pci_dev_str_match_path(struct pci_dev *dev, const char *path, 259 const char **endptr) 260 { 261 int ret; 262 int seg, bus, slot, func; 263 char *wpath, *p; 264 char end; 265 266 *endptr = strchrnul(path, ';'); 267 268 wpath = kmemdup_nul(path, *endptr - path, GFP_KERNEL); 269 if (!wpath) 270 return -ENOMEM; 271 272 while (1) { 273 p = strrchr(wpath, '/'); 274 if (!p) 275 break; 276 ret = sscanf(p, "/%x.%x%c", &slot, &func, &end); 277 if (ret != 2) { 278 ret = -EINVAL; 279 goto free_and_exit; 280 } 281 282 if (dev->devfn != PCI_DEVFN(slot, func)) { 283 ret = 0; 284 goto free_and_exit; 285 } 286 287 /* 288 * Note: we don't need to get a reference to the upstream 289 * bridge because we hold a reference to the top level 290 * device which should hold a reference to the bridge, 291 * and so on. 292 */ 293 dev = pci_upstream_bridge(dev); 294 if (!dev) { 295 ret = 0; 296 goto free_and_exit; 297 } 298 299 *p = 0; 300 } 301 302 ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot, 303 &func, &end); 304 if (ret != 4) { 305 seg = 0; 306 ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end); 307 if (ret != 3) { 308 ret = -EINVAL; 309 goto free_and_exit; 310 } 311 } 312 313 ret = (seg == pci_domain_nr(dev->bus) && 314 bus == dev->bus->number && 315 dev->devfn == PCI_DEVFN(slot, func)); 316 317 free_and_exit: 318 kfree(wpath); 319 return ret; 320 } 321 322 /** 323 * pci_dev_str_match - test if a string matches a device 324 * @dev: the PCI device to test 325 * @p: string to match the device against 326 * @endptr: pointer to the string after the match 327 * 328 * Test if a string (typically from a kernel parameter) matches a specified 329 * PCI device. The string may be of one of the following formats: 330 * 331 * [<domain>:]<bus>:<device>.<func>[/<device>.<func>]* 332 * pci:<vendor>:<device>[:<subvendor>:<subdevice>] 333 * 334 * The first format specifies a PCI bus/device/function address which 335 * may change if new hardware is inserted, if motherboard firmware changes, 336 * or due to changes caused in kernel parameters. If the domain is 337 * left unspecified, it is taken to be 0. In order to be robust against 338 * bus renumbering issues, a path of PCI device/function numbers may be used 339 * to address the specific device. The path for a device can be determined 340 * through the use of 'lspci -t'. 341 * 342 * The second format matches devices using IDs in the configuration 343 * space which may match multiple devices in the system. A value of 0 344 * for any field will match all devices. (Note: this differs from 345 * in-kernel code that uses PCI_ANY_ID which is ~0; this is for 346 * legacy reasons and convenience so users don't have to specify 347 * FFFFFFFFs on the command line.) 348 * 349 * Returns 1 if the string matches the device, 0 if it does not and 350 * a negative error code if the string cannot be parsed. 351 */ 352 static int pci_dev_str_match(struct pci_dev *dev, const char *p, 353 const char **endptr) 354 { 355 int ret; 356 int count; 357 unsigned short vendor, device, subsystem_vendor, subsystem_device; 358 359 if (strncmp(p, "pci:", 4) == 0) { 360 /* PCI vendor/device (subvendor/subdevice) IDs are specified */ 361 p += 4; 362 ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device, 363 &subsystem_vendor, &subsystem_device, &count); 364 if (ret != 4) { 365 ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count); 366 if (ret != 2) 367 return -EINVAL; 368 369 subsystem_vendor = 0; 370 subsystem_device = 0; 371 } 372 373 p += count; 374 375 if ((!vendor || vendor == dev->vendor) && 376 (!device || device == dev->device) && 377 (!subsystem_vendor || 378 subsystem_vendor == dev->subsystem_vendor) && 379 (!subsystem_device || 380 subsystem_device == dev->subsystem_device)) 381 goto found; 382 } else { 383 /* 384 * PCI Bus, Device, Function IDs are specified 385 * (optionally, may include a path of devfns following it) 386 */ 387 ret = pci_dev_str_match_path(dev, p, &p); 388 if (ret < 0) 389 return ret; 390 else if (ret) 391 goto found; 392 } 393 394 *endptr = p; 395 return 0; 396 397 found: 398 *endptr = p; 399 return 1; 400 } 401 402 static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn, 403 u8 pos, int cap, int *ttl) 404 { 405 u8 id; 406 u16 ent; 407 408 pci_bus_read_config_byte(bus, devfn, pos, &pos); 409 410 while ((*ttl)--) { 411 if (pos < 0x40) 412 break; 413 pos &= ~3; 414 pci_bus_read_config_word(bus, devfn, pos, &ent); 415 416 id = ent & 0xff; 417 if (id == 0xff) 418 break; 419 if (id == cap) 420 return pos; 421 pos = (ent >> 8); 422 } 423 return 0; 424 } 425 426 static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn, 427 u8 pos, int cap) 428 { 429 int ttl = PCI_FIND_CAP_TTL; 430 431 return __pci_find_next_cap_ttl(bus, devfn, pos, cap, &ttl); 432 } 433 434 u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap) 435 { 436 return __pci_find_next_cap(dev->bus, dev->devfn, 437 pos + PCI_CAP_LIST_NEXT, cap); 438 } 439 EXPORT_SYMBOL_GPL(pci_find_next_capability); 440 441 static u8 __pci_bus_find_cap_start(struct pci_bus *bus, 442 unsigned int devfn, u8 hdr_type) 443 { 444 u16 status; 445 446 pci_bus_read_config_word(bus, devfn, PCI_STATUS, &status); 447 if (!(status & PCI_STATUS_CAP_LIST)) 448 return 0; 449 450 switch (hdr_type) { 451 case PCI_HEADER_TYPE_NORMAL: 452 case PCI_HEADER_TYPE_BRIDGE: 453 return PCI_CAPABILITY_LIST; 454 case PCI_HEADER_TYPE_CARDBUS: 455 return PCI_CB_CAPABILITY_LIST; 456 } 457 458 return 0; 459 } 460 461 /** 462 * pci_find_capability - query for devices' capabilities 463 * @dev: PCI device to query 464 * @cap: capability code 465 * 466 * Tell if a device supports a given PCI capability. 467 * Returns the address of the requested capability structure within the 468 * device's PCI configuration space or 0 in case the device does not 469 * support it. Possible values for @cap include: 470 * 471 * %PCI_CAP_ID_PM Power Management 472 * %PCI_CAP_ID_AGP Accelerated Graphics Port 473 * %PCI_CAP_ID_VPD Vital Product Data 474 * %PCI_CAP_ID_SLOTID Slot Identification 475 * %PCI_CAP_ID_MSI Message Signalled Interrupts 476 * %PCI_CAP_ID_CHSWP CompactPCI HotSwap 477 * %PCI_CAP_ID_PCIX PCI-X 478 * %PCI_CAP_ID_EXP PCI Express 479 */ 480 u8 pci_find_capability(struct pci_dev *dev, int cap) 481 { 482 u8 pos; 483 484 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 485 if (pos) 486 pos = __pci_find_next_cap(dev->bus, dev->devfn, pos, cap); 487 488 return pos; 489 } 490 EXPORT_SYMBOL(pci_find_capability); 491 492 /** 493 * pci_bus_find_capability - query for devices' capabilities 494 * @bus: the PCI bus to query 495 * @devfn: PCI device to query 496 * @cap: capability code 497 * 498 * Like pci_find_capability() but works for PCI devices that do not have a 499 * pci_dev structure set up yet. 500 * 501 * Returns the address of the requested capability structure within the 502 * device's PCI configuration space or 0 in case the device does not 503 * support it. 504 */ 505 u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap) 506 { 507 u8 hdr_type, pos; 508 509 pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, &hdr_type); 510 511 pos = __pci_bus_find_cap_start(bus, devfn, hdr_type & 0x7f); 512 if (pos) 513 pos = __pci_find_next_cap(bus, devfn, pos, cap); 514 515 return pos; 516 } 517 EXPORT_SYMBOL(pci_bus_find_capability); 518 519 /** 520 * pci_find_next_ext_capability - Find an extended capability 521 * @dev: PCI device to query 522 * @start: address at which to start looking (0 to start at beginning of list) 523 * @cap: capability code 524 * 525 * Returns the address of the next matching extended capability structure 526 * within the device's PCI configuration space or 0 if the device does 527 * not support it. Some capabilities can occur several times, e.g., the 528 * vendor-specific capability, and this provides a way to find them all. 529 */ 530 u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap) 531 { 532 u32 header; 533 int ttl; 534 u16 pos = PCI_CFG_SPACE_SIZE; 535 536 /* minimum 8 bytes per capability */ 537 ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8; 538 539 if (dev->cfg_size <= PCI_CFG_SPACE_SIZE) 540 return 0; 541 542 if (start) 543 pos = start; 544 545 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 546 return 0; 547 548 /* 549 * If we have no capabilities, this is indicated by cap ID, 550 * cap version and next pointer all being 0. 551 */ 552 if (header == 0) 553 return 0; 554 555 while (ttl-- > 0) { 556 if (PCI_EXT_CAP_ID(header) == cap && pos != start) 557 return pos; 558 559 pos = PCI_EXT_CAP_NEXT(header); 560 if (pos < PCI_CFG_SPACE_SIZE) 561 break; 562 563 if (pci_read_config_dword(dev, pos, &header) != PCIBIOS_SUCCESSFUL) 564 break; 565 } 566 567 return 0; 568 } 569 EXPORT_SYMBOL_GPL(pci_find_next_ext_capability); 570 571 /** 572 * pci_find_ext_capability - Find an extended capability 573 * @dev: PCI device to query 574 * @cap: capability code 575 * 576 * Returns the address of the requested extended capability structure 577 * within the device's PCI configuration space or 0 if the device does 578 * not support it. Possible values for @cap include: 579 * 580 * %PCI_EXT_CAP_ID_ERR Advanced Error Reporting 581 * %PCI_EXT_CAP_ID_VC Virtual Channel 582 * %PCI_EXT_CAP_ID_DSN Device Serial Number 583 * %PCI_EXT_CAP_ID_PWR Power Budgeting 584 */ 585 u16 pci_find_ext_capability(struct pci_dev *dev, int cap) 586 { 587 return pci_find_next_ext_capability(dev, 0, cap); 588 } 589 EXPORT_SYMBOL_GPL(pci_find_ext_capability); 590 591 /** 592 * pci_get_dsn - Read and return the 8-byte Device Serial Number 593 * @dev: PCI device to query 594 * 595 * Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial 596 * Number. 597 * 598 * Returns the DSN, or zero if the capability does not exist. 599 */ 600 u64 pci_get_dsn(struct pci_dev *dev) 601 { 602 u32 dword; 603 u64 dsn; 604 int pos; 605 606 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN); 607 if (!pos) 608 return 0; 609 610 /* 611 * The Device Serial Number is two dwords offset 4 bytes from the 612 * capability position. The specification says that the first dword is 613 * the lower half, and the second dword is the upper half. 614 */ 615 pos += 4; 616 pci_read_config_dword(dev, pos, &dword); 617 dsn = (u64)dword; 618 pci_read_config_dword(dev, pos + 4, &dword); 619 dsn |= ((u64)dword) << 32; 620 621 return dsn; 622 } 623 EXPORT_SYMBOL_GPL(pci_get_dsn); 624 625 static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap) 626 { 627 int rc, ttl = PCI_FIND_CAP_TTL; 628 u8 cap, mask; 629 630 if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST) 631 mask = HT_3BIT_CAP_MASK; 632 else 633 mask = HT_5BIT_CAP_MASK; 634 635 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, pos, 636 PCI_CAP_ID_HT, &ttl); 637 while (pos) { 638 rc = pci_read_config_byte(dev, pos + 3, &cap); 639 if (rc != PCIBIOS_SUCCESSFUL) 640 return 0; 641 642 if ((cap & mask) == ht_cap) 643 return pos; 644 645 pos = __pci_find_next_cap_ttl(dev->bus, dev->devfn, 646 pos + PCI_CAP_LIST_NEXT, 647 PCI_CAP_ID_HT, &ttl); 648 } 649 650 return 0; 651 } 652 653 /** 654 * pci_find_next_ht_capability - query a device's HyperTransport capabilities 655 * @dev: PCI device to query 656 * @pos: Position from which to continue searching 657 * @ht_cap: HyperTransport capability code 658 * 659 * To be used in conjunction with pci_find_ht_capability() to search for 660 * all capabilities matching @ht_cap. @pos should always be a value returned 661 * from pci_find_ht_capability(). 662 * 663 * NB. To be 100% safe against broken PCI devices, the caller should take 664 * steps to avoid an infinite loop. 665 */ 666 u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap) 667 { 668 return __pci_find_next_ht_cap(dev, pos + PCI_CAP_LIST_NEXT, ht_cap); 669 } 670 EXPORT_SYMBOL_GPL(pci_find_next_ht_capability); 671 672 /** 673 * pci_find_ht_capability - query a device's HyperTransport capabilities 674 * @dev: PCI device to query 675 * @ht_cap: HyperTransport capability code 676 * 677 * Tell if a device supports a given HyperTransport capability. 678 * Returns an address within the device's PCI configuration space 679 * or 0 in case the device does not support the request capability. 680 * The address points to the PCI capability, of type PCI_CAP_ID_HT, 681 * which has a HyperTransport capability matching @ht_cap. 682 */ 683 u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap) 684 { 685 u8 pos; 686 687 pos = __pci_bus_find_cap_start(dev->bus, dev->devfn, dev->hdr_type); 688 if (pos) 689 pos = __pci_find_next_ht_cap(dev, pos, ht_cap); 690 691 return pos; 692 } 693 EXPORT_SYMBOL_GPL(pci_find_ht_capability); 694 695 /** 696 * pci_find_parent_resource - return resource region of parent bus of given 697 * region 698 * @dev: PCI device structure contains resources to be searched 699 * @res: child resource record for which parent is sought 700 * 701 * For given resource region of given device, return the resource region of 702 * parent bus the given region is contained in. 703 */ 704 struct resource *pci_find_parent_resource(const struct pci_dev *dev, 705 struct resource *res) 706 { 707 const struct pci_bus *bus = dev->bus; 708 struct resource *r; 709 int i; 710 711 pci_bus_for_each_resource(bus, r, i) { 712 if (!r) 713 continue; 714 if (resource_contains(r, res)) { 715 716 /* 717 * If the window is prefetchable but the BAR is 718 * not, the allocator made a mistake. 719 */ 720 if (r->flags & IORESOURCE_PREFETCH && 721 !(res->flags & IORESOURCE_PREFETCH)) 722 return NULL; 723 724 /* 725 * If we're below a transparent bridge, there may 726 * be both a positively-decoded aperture and a 727 * subtractively-decoded region that contain the BAR. 728 * We want the positively-decoded one, so this depends 729 * on pci_bus_for_each_resource() giving us those 730 * first. 731 */ 732 return r; 733 } 734 } 735 return NULL; 736 } 737 EXPORT_SYMBOL(pci_find_parent_resource); 738 739 /** 740 * pci_find_resource - Return matching PCI device resource 741 * @dev: PCI device to query 742 * @res: Resource to look for 743 * 744 * Goes over standard PCI resources (BARs) and checks if the given resource 745 * is partially or fully contained in any of them. In that case the 746 * matching resource is returned, %NULL otherwise. 747 */ 748 struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res) 749 { 750 int i; 751 752 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 753 struct resource *r = &dev->resource[i]; 754 755 if (r->start && resource_contains(r, res)) 756 return r; 757 } 758 759 return NULL; 760 } 761 EXPORT_SYMBOL(pci_find_resource); 762 763 /** 764 * pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos 765 * @dev: the PCI device to operate on 766 * @pos: config space offset of status word 767 * @mask: mask of bit(s) to care about in status word 768 * 769 * Return 1 when mask bit(s) in status word clear, 0 otherwise. 770 */ 771 int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask) 772 { 773 int i; 774 775 /* Wait for Transaction Pending bit clean */ 776 for (i = 0; i < 4; i++) { 777 u16 status; 778 if (i) 779 msleep((1 << (i - 1)) * 100); 780 781 pci_read_config_word(dev, pos, &status); 782 if (!(status & mask)) 783 return 1; 784 } 785 786 return 0; 787 } 788 789 static int pci_acs_enable; 790 791 /** 792 * pci_request_acs - ask for ACS to be enabled if supported 793 */ 794 void pci_request_acs(void) 795 { 796 pci_acs_enable = 1; 797 } 798 799 static const char *disable_acs_redir_param; 800 801 /** 802 * pci_disable_acs_redir - disable ACS redirect capabilities 803 * @dev: the PCI device 804 * 805 * For only devices specified in the disable_acs_redir parameter. 806 */ 807 static void pci_disable_acs_redir(struct pci_dev *dev) 808 { 809 int ret = 0; 810 const char *p; 811 int pos; 812 u16 ctrl; 813 814 if (!disable_acs_redir_param) 815 return; 816 817 p = disable_acs_redir_param; 818 while (*p) { 819 ret = pci_dev_str_match(dev, p, &p); 820 if (ret < 0) { 821 pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n", 822 disable_acs_redir_param); 823 824 break; 825 } else if (ret == 1) { 826 /* Found a match */ 827 break; 828 } 829 830 if (*p != ';' && *p != ',') { 831 /* End of param or invalid format */ 832 break; 833 } 834 p++; 835 } 836 837 if (ret != 1) 838 return; 839 840 if (!pci_dev_specific_disable_acs_redir(dev)) 841 return; 842 843 pos = dev->acs_cap; 844 if (!pos) { 845 pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n"); 846 return; 847 } 848 849 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 850 851 /* P2P Request & Completion Redirect */ 852 ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC); 853 854 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 855 856 pci_info(dev, "disabled ACS redirect\n"); 857 } 858 859 /** 860 * pci_std_enable_acs - enable ACS on devices using standard ACS capabilities 861 * @dev: the PCI device 862 */ 863 static void pci_std_enable_acs(struct pci_dev *dev) 864 { 865 int pos; 866 u16 cap; 867 u16 ctrl; 868 869 pos = dev->acs_cap; 870 if (!pos) 871 return; 872 873 pci_read_config_word(dev, pos + PCI_ACS_CAP, &cap); 874 pci_read_config_word(dev, pos + PCI_ACS_CTRL, &ctrl); 875 876 /* Source Validation */ 877 ctrl |= (cap & PCI_ACS_SV); 878 879 /* P2P Request Redirect */ 880 ctrl |= (cap & PCI_ACS_RR); 881 882 /* P2P Completion Redirect */ 883 ctrl |= (cap & PCI_ACS_CR); 884 885 /* Upstream Forwarding */ 886 ctrl |= (cap & PCI_ACS_UF); 887 888 /* Enable Translation Blocking for external devices */ 889 if (dev->external_facing || dev->untrusted) 890 ctrl |= (cap & PCI_ACS_TB); 891 892 pci_write_config_word(dev, pos + PCI_ACS_CTRL, ctrl); 893 } 894 895 /** 896 * pci_enable_acs - enable ACS if hardware support it 897 * @dev: the PCI device 898 */ 899 static void pci_enable_acs(struct pci_dev *dev) 900 { 901 if (!pci_acs_enable) 902 goto disable_acs_redir; 903 904 if (!pci_dev_specific_enable_acs(dev)) 905 goto disable_acs_redir; 906 907 pci_std_enable_acs(dev); 908 909 disable_acs_redir: 910 /* 911 * Note: pci_disable_acs_redir() must be called even if ACS was not 912 * enabled by the kernel because it may have been enabled by 913 * platform firmware. So if we are told to disable it, we should 914 * always disable it after setting the kernel's default 915 * preferences. 916 */ 917 pci_disable_acs_redir(dev); 918 } 919 920 /** 921 * pci_restore_bars - restore a device's BAR values (e.g. after wake-up) 922 * @dev: PCI device to have its BARs restored 923 * 924 * Restore the BAR values for a given device, so as to make it 925 * accessible by its driver. 926 */ 927 static void pci_restore_bars(struct pci_dev *dev) 928 { 929 int i; 930 931 for (i = 0; i < PCI_BRIDGE_RESOURCES; i++) 932 pci_update_resource(dev, i); 933 } 934 935 static const struct pci_platform_pm_ops *pci_platform_pm; 936 937 int pci_set_platform_pm(const struct pci_platform_pm_ops *ops) 938 { 939 if (!ops->is_manageable || !ops->set_state || !ops->get_state || 940 !ops->choose_state || !ops->set_wakeup || !ops->need_resume) 941 return -EINVAL; 942 pci_platform_pm = ops; 943 return 0; 944 } 945 946 static inline bool platform_pci_power_manageable(struct pci_dev *dev) 947 { 948 return pci_platform_pm ? pci_platform_pm->is_manageable(dev) : false; 949 } 950 951 static inline int platform_pci_set_power_state(struct pci_dev *dev, 952 pci_power_t t) 953 { 954 return pci_platform_pm ? pci_platform_pm->set_state(dev, t) : -ENOSYS; 955 } 956 957 static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev) 958 { 959 return pci_platform_pm ? pci_platform_pm->get_state(dev) : PCI_UNKNOWN; 960 } 961 962 static inline void platform_pci_refresh_power_state(struct pci_dev *dev) 963 { 964 if (pci_platform_pm && pci_platform_pm->refresh_state) 965 pci_platform_pm->refresh_state(dev); 966 } 967 968 static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev) 969 { 970 return pci_platform_pm ? 971 pci_platform_pm->choose_state(dev) : PCI_POWER_ERROR; 972 } 973 974 static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable) 975 { 976 return pci_platform_pm ? 977 pci_platform_pm->set_wakeup(dev, enable) : -ENODEV; 978 } 979 980 static inline bool platform_pci_need_resume(struct pci_dev *dev) 981 { 982 return pci_platform_pm ? pci_platform_pm->need_resume(dev) : false; 983 } 984 985 static inline bool platform_pci_bridge_d3(struct pci_dev *dev) 986 { 987 if (pci_platform_pm && pci_platform_pm->bridge_d3) 988 return pci_platform_pm->bridge_d3(dev); 989 return false; 990 } 991 992 /** 993 * pci_raw_set_power_state - Use PCI PM registers to set the power state of 994 * given PCI device 995 * @dev: PCI device to handle. 996 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 997 * 998 * RETURN VALUE: 999 * -EINVAL if the requested state is invalid. 1000 * -EIO if device does not support PCI PM or its PM capabilities register has a 1001 * wrong version, or device doesn't support the requested state. 1002 * 0 if device already is in the requested state. 1003 * 0 if device's power state has been successfully changed. 1004 */ 1005 static int pci_raw_set_power_state(struct pci_dev *dev, pci_power_t state) 1006 { 1007 u16 pmcsr; 1008 bool need_restore = false; 1009 1010 /* Check if we're already there */ 1011 if (dev->current_state == state) 1012 return 0; 1013 1014 if (!dev->pm_cap) 1015 return -EIO; 1016 1017 if (state < PCI_D0 || state > PCI_D3hot) 1018 return -EINVAL; 1019 1020 /* 1021 * Validate transition: We can enter D0 from any state, but if 1022 * we're already in a low-power state, we can only go deeper. E.g., 1023 * we can go from D1 to D3, but we can't go directly from D3 to D1; 1024 * we'd have to go from D3 to D0, then to D1. 1025 */ 1026 if (state != PCI_D0 && dev->current_state <= PCI_D3cold 1027 && dev->current_state > state) { 1028 pci_err(dev, "invalid power transition (from %s to %s)\n", 1029 pci_power_name(dev->current_state), 1030 pci_power_name(state)); 1031 return -EINVAL; 1032 } 1033 1034 /* Check if this device supports the desired state */ 1035 if ((state == PCI_D1 && !dev->d1_support) 1036 || (state == PCI_D2 && !dev->d2_support)) 1037 return -EIO; 1038 1039 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1040 if (pmcsr == (u16) ~0) { 1041 pci_err(dev, "can't change power state from %s to %s (config space inaccessible)\n", 1042 pci_power_name(dev->current_state), 1043 pci_power_name(state)); 1044 return -EIO; 1045 } 1046 1047 /* 1048 * If we're (effectively) in D3, force entire word to 0. 1049 * This doesn't affect PME_Status, disables PME_En, and 1050 * sets PowerState to 0. 1051 */ 1052 switch (dev->current_state) { 1053 case PCI_D0: 1054 case PCI_D1: 1055 case PCI_D2: 1056 pmcsr &= ~PCI_PM_CTRL_STATE_MASK; 1057 pmcsr |= state; 1058 break; 1059 case PCI_D3hot: 1060 case PCI_D3cold: 1061 case PCI_UNKNOWN: /* Boot-up */ 1062 if ((pmcsr & PCI_PM_CTRL_STATE_MASK) == PCI_D3hot 1063 && !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET)) 1064 need_restore = true; 1065 fallthrough; /* force to D0 */ 1066 default: 1067 pmcsr = 0; 1068 break; 1069 } 1070 1071 /* Enter specified state */ 1072 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 1073 1074 /* 1075 * Mandatory power management transition delays; see PCI PM 1.1 1076 * 5.6.1 table 18 1077 */ 1078 if (state == PCI_D3hot || dev->current_state == PCI_D3hot) 1079 pci_dev_d3_sleep(dev); 1080 else if (state == PCI_D2 || dev->current_state == PCI_D2) 1081 udelay(PCI_PM_D2_DELAY); 1082 1083 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1084 dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); 1085 if (dev->current_state != state) 1086 pci_info_ratelimited(dev, "refused to change power state from %s to %s\n", 1087 pci_power_name(dev->current_state), 1088 pci_power_name(state)); 1089 1090 /* 1091 * According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT 1092 * INTERFACE SPECIFICATION, REV. 1.2", a device transitioning 1093 * from D3hot to D0 _may_ perform an internal reset, thereby 1094 * going to "D0 Uninitialized" rather than "D0 Initialized". 1095 * For example, at least some versions of the 3c905B and the 1096 * 3c556B exhibit this behaviour. 1097 * 1098 * At least some laptop BIOSen (e.g. the Thinkpad T21) leave 1099 * devices in a D3hot state at boot. Consequently, we need to 1100 * restore at least the BARs so that the device will be 1101 * accessible to its driver. 1102 */ 1103 if (need_restore) 1104 pci_restore_bars(dev); 1105 1106 if (dev->bus->self) 1107 pcie_aspm_pm_state_change(dev->bus->self); 1108 1109 return 0; 1110 } 1111 1112 /** 1113 * pci_update_current_state - Read power state of given device and cache it 1114 * @dev: PCI device to handle. 1115 * @state: State to cache in case the device doesn't have the PM capability 1116 * 1117 * The power state is read from the PMCSR register, which however is 1118 * inaccessible in D3cold. The platform firmware is therefore queried first 1119 * to detect accessibility of the register. In case the platform firmware 1120 * reports an incorrect state or the device isn't power manageable by the 1121 * platform at all, we try to detect D3cold by testing accessibility of the 1122 * vendor ID in config space. 1123 */ 1124 void pci_update_current_state(struct pci_dev *dev, pci_power_t state) 1125 { 1126 if (platform_pci_get_power_state(dev) == PCI_D3cold || 1127 !pci_device_is_present(dev)) { 1128 dev->current_state = PCI_D3cold; 1129 } else if (dev->pm_cap) { 1130 u16 pmcsr; 1131 1132 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1133 dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); 1134 } else { 1135 dev->current_state = state; 1136 } 1137 } 1138 1139 /** 1140 * pci_refresh_power_state - Refresh the given device's power state data 1141 * @dev: Target PCI device. 1142 * 1143 * Ask the platform to refresh the devices power state information and invoke 1144 * pci_update_current_state() to update its current PCI power state. 1145 */ 1146 void pci_refresh_power_state(struct pci_dev *dev) 1147 { 1148 if (platform_pci_power_manageable(dev)) 1149 platform_pci_refresh_power_state(dev); 1150 1151 pci_update_current_state(dev, dev->current_state); 1152 } 1153 1154 /** 1155 * pci_platform_power_transition - Use platform to change device power state 1156 * @dev: PCI device to handle. 1157 * @state: State to put the device into. 1158 */ 1159 int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state) 1160 { 1161 int error; 1162 1163 if (platform_pci_power_manageable(dev)) { 1164 error = platform_pci_set_power_state(dev, state); 1165 if (!error) 1166 pci_update_current_state(dev, state); 1167 } else 1168 error = -ENODEV; 1169 1170 if (error && !dev->pm_cap) /* Fall back to PCI_D0 */ 1171 dev->current_state = PCI_D0; 1172 1173 return error; 1174 } 1175 EXPORT_SYMBOL_GPL(pci_platform_power_transition); 1176 1177 static int pci_resume_one(struct pci_dev *pci_dev, void *ign) 1178 { 1179 pm_request_resume(&pci_dev->dev); 1180 return 0; 1181 } 1182 1183 /** 1184 * pci_resume_bus - Walk given bus and runtime resume devices on it 1185 * @bus: Top bus of the subtree to walk. 1186 */ 1187 void pci_resume_bus(struct pci_bus *bus) 1188 { 1189 if (bus) 1190 pci_walk_bus(bus, pci_resume_one, NULL); 1191 } 1192 1193 static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout) 1194 { 1195 int delay = 1; 1196 u32 id; 1197 1198 /* 1199 * After reset, the device should not silently discard config 1200 * requests, but it may still indicate that it needs more time by 1201 * responding to them with CRS completions. The Root Port will 1202 * generally synthesize ~0 data to complete the read (except when 1203 * CRS SV is enabled and the read was for the Vendor ID; in that 1204 * case it synthesizes 0x0001 data). 1205 * 1206 * Wait for the device to return a non-CRS completion. Read the 1207 * Command register instead of Vendor ID so we don't have to 1208 * contend with the CRS SV value. 1209 */ 1210 pci_read_config_dword(dev, PCI_COMMAND, &id); 1211 while (id == ~0) { 1212 if (delay > timeout) { 1213 pci_warn(dev, "not ready %dms after %s; giving up\n", 1214 delay - 1, reset_type); 1215 return -ENOTTY; 1216 } 1217 1218 if (delay > 1000) 1219 pci_info(dev, "not ready %dms after %s; waiting\n", 1220 delay - 1, reset_type); 1221 1222 msleep(delay); 1223 delay *= 2; 1224 pci_read_config_dword(dev, PCI_COMMAND, &id); 1225 } 1226 1227 if (delay > 1000) 1228 pci_info(dev, "ready %dms after %s\n", delay - 1, 1229 reset_type); 1230 1231 return 0; 1232 } 1233 1234 /** 1235 * pci_power_up - Put the given device into D0 1236 * @dev: PCI device to power up 1237 */ 1238 int pci_power_up(struct pci_dev *dev) 1239 { 1240 pci_platform_power_transition(dev, PCI_D0); 1241 1242 /* 1243 * Mandatory power management transition delays are handled in 1244 * pci_pm_resume_noirq() and pci_pm_runtime_resume() of the 1245 * corresponding bridge. 1246 */ 1247 if (dev->runtime_d3cold) { 1248 /* 1249 * When powering on a bridge from D3cold, the whole hierarchy 1250 * may be powered on into D0uninitialized state, resume them to 1251 * give them a chance to suspend again 1252 */ 1253 pci_resume_bus(dev->subordinate); 1254 } 1255 1256 return pci_raw_set_power_state(dev, PCI_D0); 1257 } 1258 1259 /** 1260 * __pci_dev_set_current_state - Set current state of a PCI device 1261 * @dev: Device to handle 1262 * @data: pointer to state to be set 1263 */ 1264 static int __pci_dev_set_current_state(struct pci_dev *dev, void *data) 1265 { 1266 pci_power_t state = *(pci_power_t *)data; 1267 1268 dev->current_state = state; 1269 return 0; 1270 } 1271 1272 /** 1273 * pci_bus_set_current_state - Walk given bus and set current state of devices 1274 * @bus: Top bus of the subtree to walk. 1275 * @state: state to be set 1276 */ 1277 void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state) 1278 { 1279 if (bus) 1280 pci_walk_bus(bus, __pci_dev_set_current_state, &state); 1281 } 1282 1283 /** 1284 * pci_set_power_state - Set the power state of a PCI device 1285 * @dev: PCI device to handle. 1286 * @state: PCI power state (D0, D1, D2, D3hot) to put the device into. 1287 * 1288 * Transition a device to a new power state, using the platform firmware and/or 1289 * the device's PCI PM registers. 1290 * 1291 * RETURN VALUE: 1292 * -EINVAL if the requested state is invalid. 1293 * -EIO if device does not support PCI PM or its PM capabilities register has a 1294 * wrong version, or device doesn't support the requested state. 1295 * 0 if the transition is to D1 or D2 but D1 and D2 are not supported. 1296 * 0 if device already is in the requested state. 1297 * 0 if the transition is to D3 but D3 is not supported. 1298 * 0 if device's power state has been successfully changed. 1299 */ 1300 int pci_set_power_state(struct pci_dev *dev, pci_power_t state) 1301 { 1302 int error; 1303 1304 /* Bound the state we're entering */ 1305 if (state > PCI_D3cold) 1306 state = PCI_D3cold; 1307 else if (state < PCI_D0) 1308 state = PCI_D0; 1309 else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev)) 1310 1311 /* 1312 * If the device or the parent bridge do not support PCI 1313 * PM, ignore the request if we're doing anything other 1314 * than putting it into D0 (which would only happen on 1315 * boot). 1316 */ 1317 return 0; 1318 1319 /* Check if we're already there */ 1320 if (dev->current_state == state) 1321 return 0; 1322 1323 if (state == PCI_D0) 1324 return pci_power_up(dev); 1325 1326 /* 1327 * This device is quirked not to be put into D3, so don't put it in 1328 * D3 1329 */ 1330 if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3)) 1331 return 0; 1332 1333 /* 1334 * To put device in D3cold, we put device into D3hot in native 1335 * way, then put device into D3cold with platform ops 1336 */ 1337 error = pci_raw_set_power_state(dev, state > PCI_D3hot ? 1338 PCI_D3hot : state); 1339 1340 if (pci_platform_power_transition(dev, state)) 1341 return error; 1342 1343 /* Powering off a bridge may power off the whole hierarchy */ 1344 if (state == PCI_D3cold) 1345 pci_bus_set_current_state(dev->subordinate, PCI_D3cold); 1346 1347 return 0; 1348 } 1349 EXPORT_SYMBOL(pci_set_power_state); 1350 1351 /** 1352 * pci_choose_state - Choose the power state of a PCI device 1353 * @dev: PCI device to be suspended 1354 * @state: target sleep state for the whole system. This is the value 1355 * that is passed to suspend() function. 1356 * 1357 * Returns PCI power state suitable for given device and given system 1358 * message. 1359 */ 1360 pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state) 1361 { 1362 pci_power_t ret; 1363 1364 if (!dev->pm_cap) 1365 return PCI_D0; 1366 1367 ret = platform_pci_choose_state(dev); 1368 if (ret != PCI_POWER_ERROR) 1369 return ret; 1370 1371 switch (state.event) { 1372 case PM_EVENT_ON: 1373 return PCI_D0; 1374 case PM_EVENT_FREEZE: 1375 case PM_EVENT_PRETHAW: 1376 /* REVISIT both freeze and pre-thaw "should" use D0 */ 1377 case PM_EVENT_SUSPEND: 1378 case PM_EVENT_HIBERNATE: 1379 return PCI_D3hot; 1380 default: 1381 pci_info(dev, "unrecognized suspend event %d\n", 1382 state.event); 1383 BUG(); 1384 } 1385 return PCI_D0; 1386 } 1387 EXPORT_SYMBOL(pci_choose_state); 1388 1389 #define PCI_EXP_SAVE_REGS 7 1390 1391 static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev, 1392 u16 cap, bool extended) 1393 { 1394 struct pci_cap_saved_state *tmp; 1395 1396 hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) { 1397 if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap) 1398 return tmp; 1399 } 1400 return NULL; 1401 } 1402 1403 struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap) 1404 { 1405 return _pci_find_saved_cap(dev, cap, false); 1406 } 1407 1408 struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap) 1409 { 1410 return _pci_find_saved_cap(dev, cap, true); 1411 } 1412 1413 static int pci_save_pcie_state(struct pci_dev *dev) 1414 { 1415 int i = 0; 1416 struct pci_cap_saved_state *save_state; 1417 u16 *cap; 1418 1419 if (!pci_is_pcie(dev)) 1420 return 0; 1421 1422 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1423 if (!save_state) { 1424 pci_err(dev, "buffer not found in %s\n", __func__); 1425 return -ENOMEM; 1426 } 1427 1428 cap = (u16 *)&save_state->cap.data[0]; 1429 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &cap[i++]); 1430 pcie_capability_read_word(dev, PCI_EXP_LNKCTL, &cap[i++]); 1431 pcie_capability_read_word(dev, PCI_EXP_SLTCTL, &cap[i++]); 1432 pcie_capability_read_word(dev, PCI_EXP_RTCTL, &cap[i++]); 1433 pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, &cap[i++]); 1434 pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &cap[i++]); 1435 pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, &cap[i++]); 1436 1437 return 0; 1438 } 1439 1440 static void pci_restore_pcie_state(struct pci_dev *dev) 1441 { 1442 int i = 0; 1443 struct pci_cap_saved_state *save_state; 1444 u16 *cap; 1445 1446 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); 1447 if (!save_state) 1448 return; 1449 1450 cap = (u16 *)&save_state->cap.data[0]; 1451 pcie_capability_write_word(dev, PCI_EXP_DEVCTL, cap[i++]); 1452 pcie_capability_write_word(dev, PCI_EXP_LNKCTL, cap[i++]); 1453 pcie_capability_write_word(dev, PCI_EXP_SLTCTL, cap[i++]); 1454 pcie_capability_write_word(dev, PCI_EXP_RTCTL, cap[i++]); 1455 pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, cap[i++]); 1456 pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, cap[i++]); 1457 pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, cap[i++]); 1458 } 1459 1460 static int pci_save_pcix_state(struct pci_dev *dev) 1461 { 1462 int pos; 1463 struct pci_cap_saved_state *save_state; 1464 1465 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1466 if (!pos) 1467 return 0; 1468 1469 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1470 if (!save_state) { 1471 pci_err(dev, "buffer not found in %s\n", __func__); 1472 return -ENOMEM; 1473 } 1474 1475 pci_read_config_word(dev, pos + PCI_X_CMD, 1476 (u16 *)save_state->cap.data); 1477 1478 return 0; 1479 } 1480 1481 static void pci_restore_pcix_state(struct pci_dev *dev) 1482 { 1483 int i = 0, pos; 1484 struct pci_cap_saved_state *save_state; 1485 u16 *cap; 1486 1487 save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX); 1488 pos = pci_find_capability(dev, PCI_CAP_ID_PCIX); 1489 if (!save_state || !pos) 1490 return; 1491 cap = (u16 *)&save_state->cap.data[0]; 1492 1493 pci_write_config_word(dev, pos + PCI_X_CMD, cap[i++]); 1494 } 1495 1496 static void pci_save_ltr_state(struct pci_dev *dev) 1497 { 1498 int ltr; 1499 struct pci_cap_saved_state *save_state; 1500 u16 *cap; 1501 1502 if (!pci_is_pcie(dev)) 1503 return; 1504 1505 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR); 1506 if (!ltr) 1507 return; 1508 1509 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR); 1510 if (!save_state) { 1511 pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n"); 1512 return; 1513 } 1514 1515 cap = (u16 *)&save_state->cap.data[0]; 1516 pci_read_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, cap++); 1517 pci_read_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, cap++); 1518 } 1519 1520 static void pci_restore_ltr_state(struct pci_dev *dev) 1521 { 1522 struct pci_cap_saved_state *save_state; 1523 int ltr; 1524 u16 *cap; 1525 1526 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR); 1527 ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR); 1528 if (!save_state || !ltr) 1529 return; 1530 1531 cap = (u16 *)&save_state->cap.data[0]; 1532 pci_write_config_word(dev, ltr + PCI_LTR_MAX_SNOOP_LAT, *cap++); 1533 pci_write_config_word(dev, ltr + PCI_LTR_MAX_NOSNOOP_LAT, *cap++); 1534 } 1535 1536 /** 1537 * pci_save_state - save the PCI configuration space of a device before 1538 * suspending 1539 * @dev: PCI device that we're dealing with 1540 */ 1541 int pci_save_state(struct pci_dev *dev) 1542 { 1543 int i; 1544 /* XXX: 100% dword access ok here? */ 1545 for (i = 0; i < 16; i++) { 1546 pci_read_config_dword(dev, i * 4, &dev->saved_config_space[i]); 1547 pci_dbg(dev, "saving config space at offset %#x (reading %#x)\n", 1548 i * 4, dev->saved_config_space[i]); 1549 } 1550 dev->state_saved = true; 1551 1552 i = pci_save_pcie_state(dev); 1553 if (i != 0) 1554 return i; 1555 1556 i = pci_save_pcix_state(dev); 1557 if (i != 0) 1558 return i; 1559 1560 pci_save_ltr_state(dev); 1561 pci_save_aspm_l1ss_state(dev); 1562 pci_save_dpc_state(dev); 1563 pci_save_aer_state(dev); 1564 pci_save_ptm_state(dev); 1565 return pci_save_vc_state(dev); 1566 } 1567 EXPORT_SYMBOL(pci_save_state); 1568 1569 static void pci_restore_config_dword(struct pci_dev *pdev, int offset, 1570 u32 saved_val, int retry, bool force) 1571 { 1572 u32 val; 1573 1574 pci_read_config_dword(pdev, offset, &val); 1575 if (!force && val == saved_val) 1576 return; 1577 1578 for (;;) { 1579 pci_dbg(pdev, "restoring config space at offset %#x (was %#x, writing %#x)\n", 1580 offset, val, saved_val); 1581 pci_write_config_dword(pdev, offset, saved_val); 1582 if (retry-- <= 0) 1583 return; 1584 1585 pci_read_config_dword(pdev, offset, &val); 1586 if (val == saved_val) 1587 return; 1588 1589 mdelay(1); 1590 } 1591 } 1592 1593 static void pci_restore_config_space_range(struct pci_dev *pdev, 1594 int start, int end, int retry, 1595 bool force) 1596 { 1597 int index; 1598 1599 for (index = end; index >= start; index--) 1600 pci_restore_config_dword(pdev, 4 * index, 1601 pdev->saved_config_space[index], 1602 retry, force); 1603 } 1604 1605 static void pci_restore_config_space(struct pci_dev *pdev) 1606 { 1607 if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 1608 pci_restore_config_space_range(pdev, 10, 15, 0, false); 1609 /* Restore BARs before the command register. */ 1610 pci_restore_config_space_range(pdev, 4, 9, 10, false); 1611 pci_restore_config_space_range(pdev, 0, 3, 0, false); 1612 } else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 1613 pci_restore_config_space_range(pdev, 12, 15, 0, false); 1614 1615 /* 1616 * Force rewriting of prefetch registers to avoid S3 resume 1617 * issues on Intel PCI bridges that occur when these 1618 * registers are not explicitly written. 1619 */ 1620 pci_restore_config_space_range(pdev, 9, 11, 0, true); 1621 pci_restore_config_space_range(pdev, 0, 8, 0, false); 1622 } else { 1623 pci_restore_config_space_range(pdev, 0, 15, 0, false); 1624 } 1625 } 1626 1627 static void pci_restore_rebar_state(struct pci_dev *pdev) 1628 { 1629 unsigned int pos, nbars, i; 1630 u32 ctrl; 1631 1632 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 1633 if (!pos) 1634 return; 1635 1636 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1637 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 1638 PCI_REBAR_CTRL_NBAR_SHIFT; 1639 1640 for (i = 0; i < nbars; i++, pos += 8) { 1641 struct resource *res; 1642 int bar_idx, size; 1643 1644 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 1645 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 1646 res = pdev->resource + bar_idx; 1647 size = pci_rebar_bytes_to_size(resource_size(res)); 1648 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 1649 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 1650 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 1651 } 1652 } 1653 1654 /** 1655 * pci_restore_state - Restore the saved state of a PCI device 1656 * @dev: PCI device that we're dealing with 1657 */ 1658 void pci_restore_state(struct pci_dev *dev) 1659 { 1660 if (!dev->state_saved) 1661 return; 1662 1663 /* 1664 * Restore max latencies (in the LTR capability) before enabling 1665 * LTR itself (in the PCIe capability). 1666 */ 1667 pci_restore_ltr_state(dev); 1668 pci_restore_aspm_l1ss_state(dev); 1669 1670 pci_restore_pcie_state(dev); 1671 pci_restore_pasid_state(dev); 1672 pci_restore_pri_state(dev); 1673 pci_restore_ats_state(dev); 1674 pci_restore_vc_state(dev); 1675 pci_restore_rebar_state(dev); 1676 pci_restore_dpc_state(dev); 1677 pci_restore_ptm_state(dev); 1678 1679 pci_aer_clear_status(dev); 1680 pci_restore_aer_state(dev); 1681 1682 pci_restore_config_space(dev); 1683 1684 pci_restore_pcix_state(dev); 1685 pci_restore_msi_state(dev); 1686 1687 /* Restore ACS and IOV configuration state */ 1688 pci_enable_acs(dev); 1689 pci_restore_iov_state(dev); 1690 1691 dev->state_saved = false; 1692 } 1693 EXPORT_SYMBOL(pci_restore_state); 1694 1695 struct pci_saved_state { 1696 u32 config_space[16]; 1697 struct pci_cap_saved_data cap[]; 1698 }; 1699 1700 /** 1701 * pci_store_saved_state - Allocate and return an opaque struct containing 1702 * the device saved state. 1703 * @dev: PCI device that we're dealing with 1704 * 1705 * Return NULL if no state or error. 1706 */ 1707 struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev) 1708 { 1709 struct pci_saved_state *state; 1710 struct pci_cap_saved_state *tmp; 1711 struct pci_cap_saved_data *cap; 1712 size_t size; 1713 1714 if (!dev->state_saved) 1715 return NULL; 1716 1717 size = sizeof(*state) + sizeof(struct pci_cap_saved_data); 1718 1719 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) 1720 size += sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1721 1722 state = kzalloc(size, GFP_KERNEL); 1723 if (!state) 1724 return NULL; 1725 1726 memcpy(state->config_space, dev->saved_config_space, 1727 sizeof(state->config_space)); 1728 1729 cap = state->cap; 1730 hlist_for_each_entry(tmp, &dev->saved_cap_space, next) { 1731 size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size; 1732 memcpy(cap, &tmp->cap, len); 1733 cap = (struct pci_cap_saved_data *)((u8 *)cap + len); 1734 } 1735 /* Empty cap_save terminates list */ 1736 1737 return state; 1738 } 1739 EXPORT_SYMBOL_GPL(pci_store_saved_state); 1740 1741 /** 1742 * pci_load_saved_state - Reload the provided save state into struct pci_dev. 1743 * @dev: PCI device that we're dealing with 1744 * @state: Saved state returned from pci_store_saved_state() 1745 */ 1746 int pci_load_saved_state(struct pci_dev *dev, 1747 struct pci_saved_state *state) 1748 { 1749 struct pci_cap_saved_data *cap; 1750 1751 dev->state_saved = false; 1752 1753 if (!state) 1754 return 0; 1755 1756 memcpy(dev->saved_config_space, state->config_space, 1757 sizeof(state->config_space)); 1758 1759 cap = state->cap; 1760 while (cap->size) { 1761 struct pci_cap_saved_state *tmp; 1762 1763 tmp = _pci_find_saved_cap(dev, cap->cap_nr, cap->cap_extended); 1764 if (!tmp || tmp->cap.size != cap->size) 1765 return -EINVAL; 1766 1767 memcpy(tmp->cap.data, cap->data, tmp->cap.size); 1768 cap = (struct pci_cap_saved_data *)((u8 *)cap + 1769 sizeof(struct pci_cap_saved_data) + cap->size); 1770 } 1771 1772 dev->state_saved = true; 1773 return 0; 1774 } 1775 EXPORT_SYMBOL_GPL(pci_load_saved_state); 1776 1777 /** 1778 * pci_load_and_free_saved_state - Reload the save state pointed to by state, 1779 * and free the memory allocated for it. 1780 * @dev: PCI device that we're dealing with 1781 * @state: Pointer to saved state returned from pci_store_saved_state() 1782 */ 1783 int pci_load_and_free_saved_state(struct pci_dev *dev, 1784 struct pci_saved_state **state) 1785 { 1786 int ret = pci_load_saved_state(dev, *state); 1787 kfree(*state); 1788 *state = NULL; 1789 return ret; 1790 } 1791 EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state); 1792 1793 int __weak pcibios_enable_device(struct pci_dev *dev, int bars) 1794 { 1795 return pci_enable_resources(dev, bars); 1796 } 1797 1798 static int do_pci_enable_device(struct pci_dev *dev, int bars) 1799 { 1800 int err; 1801 struct pci_dev *bridge; 1802 u16 cmd; 1803 u8 pin; 1804 1805 err = pci_set_power_state(dev, PCI_D0); 1806 if (err < 0 && err != -EIO) 1807 return err; 1808 1809 bridge = pci_upstream_bridge(dev); 1810 if (bridge) 1811 pcie_aspm_powersave_config_link(bridge); 1812 1813 err = pcibios_enable_device(dev, bars); 1814 if (err < 0) 1815 return err; 1816 pci_fixup_device(pci_fixup_enable, dev); 1817 1818 if (dev->msi_enabled || dev->msix_enabled) 1819 return 0; 1820 1821 pci_read_config_byte(dev, PCI_INTERRUPT_PIN, &pin); 1822 if (pin) { 1823 pci_read_config_word(dev, PCI_COMMAND, &cmd); 1824 if (cmd & PCI_COMMAND_INTX_DISABLE) 1825 pci_write_config_word(dev, PCI_COMMAND, 1826 cmd & ~PCI_COMMAND_INTX_DISABLE); 1827 } 1828 1829 return 0; 1830 } 1831 1832 /** 1833 * pci_reenable_device - Resume abandoned device 1834 * @dev: PCI device to be resumed 1835 * 1836 * NOTE: This function is a backend of pci_default_resume() and is not supposed 1837 * to be called by normal code, write proper resume handler and use it instead. 1838 */ 1839 int pci_reenable_device(struct pci_dev *dev) 1840 { 1841 if (pci_is_enabled(dev)) 1842 return do_pci_enable_device(dev, (1 << PCI_NUM_RESOURCES) - 1); 1843 return 0; 1844 } 1845 EXPORT_SYMBOL(pci_reenable_device); 1846 1847 static void pci_enable_bridge(struct pci_dev *dev) 1848 { 1849 struct pci_dev *bridge; 1850 int retval; 1851 1852 bridge = pci_upstream_bridge(dev); 1853 if (bridge) 1854 pci_enable_bridge(bridge); 1855 1856 if (pci_is_enabled(dev)) { 1857 if (!dev->is_busmaster) 1858 pci_set_master(dev); 1859 return; 1860 } 1861 1862 retval = pci_enable_device(dev); 1863 if (retval) 1864 pci_err(dev, "Error enabling bridge (%d), continuing\n", 1865 retval); 1866 pci_set_master(dev); 1867 } 1868 1869 static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags) 1870 { 1871 struct pci_dev *bridge; 1872 int err; 1873 int i, bars = 0; 1874 1875 /* 1876 * Power state could be unknown at this point, either due to a fresh 1877 * boot or a device removal call. So get the current power state 1878 * so that things like MSI message writing will behave as expected 1879 * (e.g. if the device really is in D0 at enable time). 1880 */ 1881 if (dev->pm_cap) { 1882 u16 pmcsr; 1883 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 1884 dev->current_state = (pmcsr & PCI_PM_CTRL_STATE_MASK); 1885 } 1886 1887 if (atomic_inc_return(&dev->enable_cnt) > 1) 1888 return 0; /* already enabled */ 1889 1890 bridge = pci_upstream_bridge(dev); 1891 if (bridge) 1892 pci_enable_bridge(bridge); 1893 1894 /* only skip sriov related */ 1895 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 1896 if (dev->resource[i].flags & flags) 1897 bars |= (1 << i); 1898 for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++) 1899 if (dev->resource[i].flags & flags) 1900 bars |= (1 << i); 1901 1902 err = do_pci_enable_device(dev, bars); 1903 if (err < 0) 1904 atomic_dec(&dev->enable_cnt); 1905 return err; 1906 } 1907 1908 /** 1909 * pci_enable_device_io - Initialize a device for use with IO space 1910 * @dev: PCI device to be initialized 1911 * 1912 * Initialize device before it's used by a driver. Ask low-level code 1913 * to enable I/O resources. Wake up the device if it was suspended. 1914 * Beware, this function can fail. 1915 */ 1916 int pci_enable_device_io(struct pci_dev *dev) 1917 { 1918 return pci_enable_device_flags(dev, IORESOURCE_IO); 1919 } 1920 EXPORT_SYMBOL(pci_enable_device_io); 1921 1922 /** 1923 * pci_enable_device_mem - Initialize a device for use with Memory space 1924 * @dev: PCI device to be initialized 1925 * 1926 * Initialize device before it's used by a driver. Ask low-level code 1927 * to enable Memory resources. Wake up the device if it was suspended. 1928 * Beware, this function can fail. 1929 */ 1930 int pci_enable_device_mem(struct pci_dev *dev) 1931 { 1932 return pci_enable_device_flags(dev, IORESOURCE_MEM); 1933 } 1934 EXPORT_SYMBOL(pci_enable_device_mem); 1935 1936 /** 1937 * pci_enable_device - Initialize device before it's used by a driver. 1938 * @dev: PCI device to be initialized 1939 * 1940 * Initialize device before it's used by a driver. Ask low-level code 1941 * to enable I/O and memory. Wake up the device if it was suspended. 1942 * Beware, this function can fail. 1943 * 1944 * Note we don't actually enable the device many times if we call 1945 * this function repeatedly (we just increment the count). 1946 */ 1947 int pci_enable_device(struct pci_dev *dev) 1948 { 1949 return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO); 1950 } 1951 EXPORT_SYMBOL(pci_enable_device); 1952 1953 /* 1954 * Managed PCI resources. This manages device on/off, INTx/MSI/MSI-X 1955 * on/off and BAR regions. pci_dev itself records MSI/MSI-X status, so 1956 * there's no need to track it separately. pci_devres is initialized 1957 * when a device is enabled using managed PCI device enable interface. 1958 */ 1959 struct pci_devres { 1960 unsigned int enabled:1; 1961 unsigned int pinned:1; 1962 unsigned int orig_intx:1; 1963 unsigned int restore_intx:1; 1964 unsigned int mwi:1; 1965 u32 region_mask; 1966 }; 1967 1968 static void pcim_release(struct device *gendev, void *res) 1969 { 1970 struct pci_dev *dev = to_pci_dev(gendev); 1971 struct pci_devres *this = res; 1972 int i; 1973 1974 if (dev->msi_enabled) 1975 pci_disable_msi(dev); 1976 if (dev->msix_enabled) 1977 pci_disable_msix(dev); 1978 1979 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) 1980 if (this->region_mask & (1 << i)) 1981 pci_release_region(dev, i); 1982 1983 if (this->mwi) 1984 pci_clear_mwi(dev); 1985 1986 if (this->restore_intx) 1987 pci_intx(dev, this->orig_intx); 1988 1989 if (this->enabled && !this->pinned) 1990 pci_disable_device(dev); 1991 } 1992 1993 static struct pci_devres *get_pci_dr(struct pci_dev *pdev) 1994 { 1995 struct pci_devres *dr, *new_dr; 1996 1997 dr = devres_find(&pdev->dev, pcim_release, NULL, NULL); 1998 if (dr) 1999 return dr; 2000 2001 new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL); 2002 if (!new_dr) 2003 return NULL; 2004 return devres_get(&pdev->dev, new_dr, NULL, NULL); 2005 } 2006 2007 static struct pci_devres *find_pci_dr(struct pci_dev *pdev) 2008 { 2009 if (pci_is_managed(pdev)) 2010 return devres_find(&pdev->dev, pcim_release, NULL, NULL); 2011 return NULL; 2012 } 2013 2014 /** 2015 * pcim_enable_device - Managed pci_enable_device() 2016 * @pdev: PCI device to be initialized 2017 * 2018 * Managed pci_enable_device(). 2019 */ 2020 int pcim_enable_device(struct pci_dev *pdev) 2021 { 2022 struct pci_devres *dr; 2023 int rc; 2024 2025 dr = get_pci_dr(pdev); 2026 if (unlikely(!dr)) 2027 return -ENOMEM; 2028 if (dr->enabled) 2029 return 0; 2030 2031 rc = pci_enable_device(pdev); 2032 if (!rc) { 2033 pdev->is_managed = 1; 2034 dr->enabled = 1; 2035 } 2036 return rc; 2037 } 2038 EXPORT_SYMBOL(pcim_enable_device); 2039 2040 /** 2041 * pcim_pin_device - Pin managed PCI device 2042 * @pdev: PCI device to pin 2043 * 2044 * Pin managed PCI device @pdev. Pinned device won't be disabled on 2045 * driver detach. @pdev must have been enabled with 2046 * pcim_enable_device(). 2047 */ 2048 void pcim_pin_device(struct pci_dev *pdev) 2049 { 2050 struct pci_devres *dr; 2051 2052 dr = find_pci_dr(pdev); 2053 WARN_ON(!dr || !dr->enabled); 2054 if (dr) 2055 dr->pinned = 1; 2056 } 2057 EXPORT_SYMBOL(pcim_pin_device); 2058 2059 /* 2060 * pcibios_add_device - provide arch specific hooks when adding device dev 2061 * @dev: the PCI device being added 2062 * 2063 * Permits the platform to provide architecture specific functionality when 2064 * devices are added. This is the default implementation. Architecture 2065 * implementations can override this. 2066 */ 2067 int __weak pcibios_add_device(struct pci_dev *dev) 2068 { 2069 return 0; 2070 } 2071 2072 /** 2073 * pcibios_release_device - provide arch specific hooks when releasing 2074 * device dev 2075 * @dev: the PCI device being released 2076 * 2077 * Permits the platform to provide architecture specific functionality when 2078 * devices are released. This is the default implementation. Architecture 2079 * implementations can override this. 2080 */ 2081 void __weak pcibios_release_device(struct pci_dev *dev) {} 2082 2083 /** 2084 * pcibios_disable_device - disable arch specific PCI resources for device dev 2085 * @dev: the PCI device to disable 2086 * 2087 * Disables architecture specific PCI resources for the device. This 2088 * is the default implementation. Architecture implementations can 2089 * override this. 2090 */ 2091 void __weak pcibios_disable_device(struct pci_dev *dev) {} 2092 2093 /** 2094 * pcibios_penalize_isa_irq - penalize an ISA IRQ 2095 * @irq: ISA IRQ to penalize 2096 * @active: IRQ active or not 2097 * 2098 * Permits the platform to provide architecture-specific functionality when 2099 * penalizing ISA IRQs. This is the default implementation. Architecture 2100 * implementations can override this. 2101 */ 2102 void __weak pcibios_penalize_isa_irq(int irq, int active) {} 2103 2104 static void do_pci_disable_device(struct pci_dev *dev) 2105 { 2106 u16 pci_command; 2107 2108 pci_read_config_word(dev, PCI_COMMAND, &pci_command); 2109 if (pci_command & PCI_COMMAND_MASTER) { 2110 pci_command &= ~PCI_COMMAND_MASTER; 2111 pci_write_config_word(dev, PCI_COMMAND, pci_command); 2112 } 2113 2114 pcibios_disable_device(dev); 2115 } 2116 2117 /** 2118 * pci_disable_enabled_device - Disable device without updating enable_cnt 2119 * @dev: PCI device to disable 2120 * 2121 * NOTE: This function is a backend of PCI power management routines and is 2122 * not supposed to be called drivers. 2123 */ 2124 void pci_disable_enabled_device(struct pci_dev *dev) 2125 { 2126 if (pci_is_enabled(dev)) 2127 do_pci_disable_device(dev); 2128 } 2129 2130 /** 2131 * pci_disable_device - Disable PCI device after use 2132 * @dev: PCI device to be disabled 2133 * 2134 * Signal to the system that the PCI device is not in use by the system 2135 * anymore. This only involves disabling PCI bus-mastering, if active. 2136 * 2137 * Note we don't actually disable the device until all callers of 2138 * pci_enable_device() have called pci_disable_device(). 2139 */ 2140 void pci_disable_device(struct pci_dev *dev) 2141 { 2142 struct pci_devres *dr; 2143 2144 dr = find_pci_dr(dev); 2145 if (dr) 2146 dr->enabled = 0; 2147 2148 dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0, 2149 "disabling already-disabled device"); 2150 2151 if (atomic_dec_return(&dev->enable_cnt) != 0) 2152 return; 2153 2154 do_pci_disable_device(dev); 2155 2156 dev->is_busmaster = 0; 2157 } 2158 EXPORT_SYMBOL(pci_disable_device); 2159 2160 /** 2161 * pcibios_set_pcie_reset_state - set reset state for device dev 2162 * @dev: the PCIe device reset 2163 * @state: Reset state to enter into 2164 * 2165 * Set the PCIe reset state for the device. This is the default 2166 * implementation. Architecture implementations can override this. 2167 */ 2168 int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev, 2169 enum pcie_reset_state state) 2170 { 2171 return -EINVAL; 2172 } 2173 2174 /** 2175 * pci_set_pcie_reset_state - set reset state for device dev 2176 * @dev: the PCIe device reset 2177 * @state: Reset state to enter into 2178 * 2179 * Sets the PCI reset state for the device. 2180 */ 2181 int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 2182 { 2183 return pcibios_set_pcie_reset_state(dev, state); 2184 } 2185 EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state); 2186 2187 void pcie_clear_device_status(struct pci_dev *dev) 2188 { 2189 u16 sta; 2190 2191 pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta); 2192 pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta); 2193 } 2194 2195 /** 2196 * pcie_clear_root_pme_status - Clear root port PME interrupt status. 2197 * @dev: PCIe root port or event collector. 2198 */ 2199 void pcie_clear_root_pme_status(struct pci_dev *dev) 2200 { 2201 pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME); 2202 } 2203 2204 /** 2205 * pci_check_pme_status - Check if given device has generated PME. 2206 * @dev: Device to check. 2207 * 2208 * Check the PME status of the device and if set, clear it and clear PME enable 2209 * (if set). Return 'true' if PME status and PME enable were both set or 2210 * 'false' otherwise. 2211 */ 2212 bool pci_check_pme_status(struct pci_dev *dev) 2213 { 2214 int pmcsr_pos; 2215 u16 pmcsr; 2216 bool ret = false; 2217 2218 if (!dev->pm_cap) 2219 return false; 2220 2221 pmcsr_pos = dev->pm_cap + PCI_PM_CTRL; 2222 pci_read_config_word(dev, pmcsr_pos, &pmcsr); 2223 if (!(pmcsr & PCI_PM_CTRL_PME_STATUS)) 2224 return false; 2225 2226 /* Clear PME status. */ 2227 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2228 if (pmcsr & PCI_PM_CTRL_PME_ENABLE) { 2229 /* Disable PME to avoid interrupt flood. */ 2230 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2231 ret = true; 2232 } 2233 2234 pci_write_config_word(dev, pmcsr_pos, pmcsr); 2235 2236 return ret; 2237 } 2238 2239 /** 2240 * pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set. 2241 * @dev: Device to handle. 2242 * @pme_poll_reset: Whether or not to reset the device's pme_poll flag. 2243 * 2244 * Check if @dev has generated PME and queue a resume request for it in that 2245 * case. 2246 */ 2247 static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset) 2248 { 2249 if (pme_poll_reset && dev->pme_poll) 2250 dev->pme_poll = false; 2251 2252 if (pci_check_pme_status(dev)) { 2253 pci_wakeup_event(dev); 2254 pm_request_resume(&dev->dev); 2255 } 2256 return 0; 2257 } 2258 2259 /** 2260 * pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary. 2261 * @bus: Top bus of the subtree to walk. 2262 */ 2263 void pci_pme_wakeup_bus(struct pci_bus *bus) 2264 { 2265 if (bus) 2266 pci_walk_bus(bus, pci_pme_wakeup, (void *)true); 2267 } 2268 2269 2270 /** 2271 * pci_pme_capable - check the capability of PCI device to generate PME# 2272 * @dev: PCI device to handle. 2273 * @state: PCI state from which device will issue PME#. 2274 */ 2275 bool pci_pme_capable(struct pci_dev *dev, pci_power_t state) 2276 { 2277 if (!dev->pm_cap) 2278 return false; 2279 2280 return !!(dev->pme_support & (1 << state)); 2281 } 2282 EXPORT_SYMBOL(pci_pme_capable); 2283 2284 static void pci_pme_list_scan(struct work_struct *work) 2285 { 2286 struct pci_pme_device *pme_dev, *n; 2287 2288 mutex_lock(&pci_pme_list_mutex); 2289 list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) { 2290 if (pme_dev->dev->pme_poll) { 2291 struct pci_dev *bridge; 2292 2293 bridge = pme_dev->dev->bus->self; 2294 /* 2295 * If bridge is in low power state, the 2296 * configuration space of subordinate devices 2297 * may be not accessible 2298 */ 2299 if (bridge && bridge->current_state != PCI_D0) 2300 continue; 2301 /* 2302 * If the device is in D3cold it should not be 2303 * polled either. 2304 */ 2305 if (pme_dev->dev->current_state == PCI_D3cold) 2306 continue; 2307 2308 pci_pme_wakeup(pme_dev->dev, NULL); 2309 } else { 2310 list_del(&pme_dev->list); 2311 kfree(pme_dev); 2312 } 2313 } 2314 if (!list_empty(&pci_pme_list)) 2315 queue_delayed_work(system_freezable_wq, &pci_pme_work, 2316 msecs_to_jiffies(PME_TIMEOUT)); 2317 mutex_unlock(&pci_pme_list_mutex); 2318 } 2319 2320 static void __pci_pme_active(struct pci_dev *dev, bool enable) 2321 { 2322 u16 pmcsr; 2323 2324 if (!dev->pme_support) 2325 return; 2326 2327 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2328 /* Clear PME_Status by writing 1 to it and enable PME# */ 2329 pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE; 2330 if (!enable) 2331 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2332 2333 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2334 } 2335 2336 /** 2337 * pci_pme_restore - Restore PME configuration after config space restore. 2338 * @dev: PCI device to update. 2339 */ 2340 void pci_pme_restore(struct pci_dev *dev) 2341 { 2342 u16 pmcsr; 2343 2344 if (!dev->pme_support) 2345 return; 2346 2347 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &pmcsr); 2348 if (dev->wakeup_prepared) { 2349 pmcsr |= PCI_PM_CTRL_PME_ENABLE; 2350 pmcsr &= ~PCI_PM_CTRL_PME_STATUS; 2351 } else { 2352 pmcsr &= ~PCI_PM_CTRL_PME_ENABLE; 2353 pmcsr |= PCI_PM_CTRL_PME_STATUS; 2354 } 2355 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, pmcsr); 2356 } 2357 2358 /** 2359 * pci_pme_active - enable or disable PCI device's PME# function 2360 * @dev: PCI device to handle. 2361 * @enable: 'true' to enable PME# generation; 'false' to disable it. 2362 * 2363 * The caller must verify that the device is capable of generating PME# before 2364 * calling this function with @enable equal to 'true'. 2365 */ 2366 void pci_pme_active(struct pci_dev *dev, bool enable) 2367 { 2368 __pci_pme_active(dev, enable); 2369 2370 /* 2371 * PCI (as opposed to PCIe) PME requires that the device have 2372 * its PME# line hooked up correctly. Not all hardware vendors 2373 * do this, so the PME never gets delivered and the device 2374 * remains asleep. The easiest way around this is to 2375 * periodically walk the list of suspended devices and check 2376 * whether any have their PME flag set. The assumption is that 2377 * we'll wake up often enough anyway that this won't be a huge 2378 * hit, and the power savings from the devices will still be a 2379 * win. 2380 * 2381 * Although PCIe uses in-band PME message instead of PME# line 2382 * to report PME, PME does not work for some PCIe devices in 2383 * reality. For example, there are devices that set their PME 2384 * status bits, but don't really bother to send a PME message; 2385 * there are PCI Express Root Ports that don't bother to 2386 * trigger interrupts when they receive PME messages from the 2387 * devices below. So PME poll is used for PCIe devices too. 2388 */ 2389 2390 if (dev->pme_poll) { 2391 struct pci_pme_device *pme_dev; 2392 if (enable) { 2393 pme_dev = kmalloc(sizeof(struct pci_pme_device), 2394 GFP_KERNEL); 2395 if (!pme_dev) { 2396 pci_warn(dev, "can't enable PME#\n"); 2397 return; 2398 } 2399 pme_dev->dev = dev; 2400 mutex_lock(&pci_pme_list_mutex); 2401 list_add(&pme_dev->list, &pci_pme_list); 2402 if (list_is_singular(&pci_pme_list)) 2403 queue_delayed_work(system_freezable_wq, 2404 &pci_pme_work, 2405 msecs_to_jiffies(PME_TIMEOUT)); 2406 mutex_unlock(&pci_pme_list_mutex); 2407 } else { 2408 mutex_lock(&pci_pme_list_mutex); 2409 list_for_each_entry(pme_dev, &pci_pme_list, list) { 2410 if (pme_dev->dev == dev) { 2411 list_del(&pme_dev->list); 2412 kfree(pme_dev); 2413 break; 2414 } 2415 } 2416 mutex_unlock(&pci_pme_list_mutex); 2417 } 2418 } 2419 2420 pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled"); 2421 } 2422 EXPORT_SYMBOL(pci_pme_active); 2423 2424 /** 2425 * __pci_enable_wake - enable PCI device as wakeup event source 2426 * @dev: PCI device affected 2427 * @state: PCI state from which device will issue wakeup events 2428 * @enable: True to enable event generation; false to disable 2429 * 2430 * This enables the device as a wakeup event source, or disables it. 2431 * When such events involves platform-specific hooks, those hooks are 2432 * called automatically by this routine. 2433 * 2434 * Devices with legacy power management (no standard PCI PM capabilities) 2435 * always require such platform hooks. 2436 * 2437 * RETURN VALUE: 2438 * 0 is returned on success 2439 * -EINVAL is returned if device is not supposed to wake up the system 2440 * Error code depending on the platform is returned if both the platform and 2441 * the native mechanism fail to enable the generation of wake-up events 2442 */ 2443 static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable) 2444 { 2445 int ret = 0; 2446 2447 /* 2448 * Bridges that are not power-manageable directly only signal 2449 * wakeup on behalf of subordinate devices which is set up 2450 * elsewhere, so skip them. However, bridges that are 2451 * power-manageable may signal wakeup for themselves (for example, 2452 * on a hotplug event) and they need to be covered here. 2453 */ 2454 if (!pci_power_manageable(dev)) 2455 return 0; 2456 2457 /* Don't do the same thing twice in a row for one device. */ 2458 if (!!enable == !!dev->wakeup_prepared) 2459 return 0; 2460 2461 /* 2462 * According to "PCI System Architecture" 4th ed. by Tom Shanley & Don 2463 * Anderson we should be doing PME# wake enable followed by ACPI wake 2464 * enable. To disable wake-up we call the platform first, for symmetry. 2465 */ 2466 2467 if (enable) { 2468 int error; 2469 2470 if (pci_pme_capable(dev, state)) 2471 pci_pme_active(dev, true); 2472 else 2473 ret = 1; 2474 error = platform_pci_set_wakeup(dev, true); 2475 if (ret) 2476 ret = error; 2477 if (!ret) 2478 dev->wakeup_prepared = true; 2479 } else { 2480 platform_pci_set_wakeup(dev, false); 2481 pci_pme_active(dev, false); 2482 dev->wakeup_prepared = false; 2483 } 2484 2485 return ret; 2486 } 2487 2488 /** 2489 * pci_enable_wake - change wakeup settings for a PCI device 2490 * @pci_dev: Target device 2491 * @state: PCI state from which device will issue wakeup events 2492 * @enable: Whether or not to enable event generation 2493 * 2494 * If @enable is set, check device_may_wakeup() for the device before calling 2495 * __pci_enable_wake() for it. 2496 */ 2497 int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable) 2498 { 2499 if (enable && !device_may_wakeup(&pci_dev->dev)) 2500 return -EINVAL; 2501 2502 return __pci_enable_wake(pci_dev, state, enable); 2503 } 2504 EXPORT_SYMBOL(pci_enable_wake); 2505 2506 /** 2507 * pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold 2508 * @dev: PCI device to prepare 2509 * @enable: True to enable wake-up event generation; false to disable 2510 * 2511 * Many drivers want the device to wake up the system from D3_hot or D3_cold 2512 * and this function allows them to set that up cleanly - pci_enable_wake() 2513 * should not be called twice in a row to enable wake-up due to PCI PM vs ACPI 2514 * ordering constraints. 2515 * 2516 * This function only returns error code if the device is not allowed to wake 2517 * up the system from sleep or it is not capable of generating PME# from both 2518 * D3_hot and D3_cold and the platform is unable to enable wake-up power for it. 2519 */ 2520 int pci_wake_from_d3(struct pci_dev *dev, bool enable) 2521 { 2522 return pci_pme_capable(dev, PCI_D3cold) ? 2523 pci_enable_wake(dev, PCI_D3cold, enable) : 2524 pci_enable_wake(dev, PCI_D3hot, enable); 2525 } 2526 EXPORT_SYMBOL(pci_wake_from_d3); 2527 2528 /** 2529 * pci_target_state - find an appropriate low power state for a given PCI dev 2530 * @dev: PCI device 2531 * @wakeup: Whether or not wakeup functionality will be enabled for the device. 2532 * 2533 * Use underlying platform code to find a supported low power state for @dev. 2534 * If the platform can't manage @dev, return the deepest state from which it 2535 * can generate wake events, based on any available PME info. 2536 */ 2537 static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup) 2538 { 2539 pci_power_t target_state = PCI_D3hot; 2540 2541 if (platform_pci_power_manageable(dev)) { 2542 /* 2543 * Call the platform to find the target state for the device. 2544 */ 2545 pci_power_t state = platform_pci_choose_state(dev); 2546 2547 switch (state) { 2548 case PCI_POWER_ERROR: 2549 case PCI_UNKNOWN: 2550 break; 2551 case PCI_D1: 2552 case PCI_D2: 2553 if (pci_no_d1d2(dev)) 2554 break; 2555 fallthrough; 2556 default: 2557 target_state = state; 2558 } 2559 2560 return target_state; 2561 } 2562 2563 if (!dev->pm_cap) 2564 target_state = PCI_D0; 2565 2566 /* 2567 * If the device is in D3cold even though it's not power-manageable by 2568 * the platform, it may have been powered down by non-standard means. 2569 * Best to let it slumber. 2570 */ 2571 if (dev->current_state == PCI_D3cold) 2572 target_state = PCI_D3cold; 2573 2574 if (wakeup) { 2575 /* 2576 * Find the deepest state from which the device can generate 2577 * PME#. 2578 */ 2579 if (dev->pme_support) { 2580 while (target_state 2581 && !(dev->pme_support & (1 << target_state))) 2582 target_state--; 2583 } 2584 } 2585 2586 return target_state; 2587 } 2588 2589 /** 2590 * pci_prepare_to_sleep - prepare PCI device for system-wide transition 2591 * into a sleep state 2592 * @dev: Device to handle. 2593 * 2594 * Choose the power state appropriate for the device depending on whether 2595 * it can wake up the system and/or is power manageable by the platform 2596 * (PCI_D3hot is the default) and put the device into that state. 2597 */ 2598 int pci_prepare_to_sleep(struct pci_dev *dev) 2599 { 2600 bool wakeup = device_may_wakeup(&dev->dev); 2601 pci_power_t target_state = pci_target_state(dev, wakeup); 2602 int error; 2603 2604 if (target_state == PCI_POWER_ERROR) 2605 return -EIO; 2606 2607 /* 2608 * There are systems (for example, Intel mobile chips since Coffee 2609 * Lake) where the power drawn while suspended can be significantly 2610 * reduced by disabling PTM on PCIe root ports as this allows the 2611 * port to enter a lower-power PM state and the SoC to reach a 2612 * lower-power idle state as a whole. 2613 */ 2614 if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) 2615 pci_disable_ptm(dev); 2616 2617 pci_enable_wake(dev, target_state, wakeup); 2618 2619 error = pci_set_power_state(dev, target_state); 2620 2621 if (error) { 2622 pci_enable_wake(dev, target_state, false); 2623 pci_restore_ptm_state(dev); 2624 } 2625 2626 return error; 2627 } 2628 EXPORT_SYMBOL(pci_prepare_to_sleep); 2629 2630 /** 2631 * pci_back_from_sleep - turn PCI device on during system-wide transition 2632 * into working state 2633 * @dev: Device to handle. 2634 * 2635 * Disable device's system wake-up capability and put it into D0. 2636 */ 2637 int pci_back_from_sleep(struct pci_dev *dev) 2638 { 2639 pci_enable_wake(dev, PCI_D0, false); 2640 return pci_set_power_state(dev, PCI_D0); 2641 } 2642 EXPORT_SYMBOL(pci_back_from_sleep); 2643 2644 /** 2645 * pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend. 2646 * @dev: PCI device being suspended. 2647 * 2648 * Prepare @dev to generate wake-up events at run time and put it into a low 2649 * power state. 2650 */ 2651 int pci_finish_runtime_suspend(struct pci_dev *dev) 2652 { 2653 pci_power_t target_state; 2654 int error; 2655 2656 target_state = pci_target_state(dev, device_can_wakeup(&dev->dev)); 2657 if (target_state == PCI_POWER_ERROR) 2658 return -EIO; 2659 2660 dev->runtime_d3cold = target_state == PCI_D3cold; 2661 2662 /* 2663 * There are systems (for example, Intel mobile chips since Coffee 2664 * Lake) where the power drawn while suspended can be significantly 2665 * reduced by disabling PTM on PCIe root ports as this allows the 2666 * port to enter a lower-power PM state and the SoC to reach a 2667 * lower-power idle state as a whole. 2668 */ 2669 if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) 2670 pci_disable_ptm(dev); 2671 2672 __pci_enable_wake(dev, target_state, pci_dev_run_wake(dev)); 2673 2674 error = pci_set_power_state(dev, target_state); 2675 2676 if (error) { 2677 pci_enable_wake(dev, target_state, false); 2678 pci_restore_ptm_state(dev); 2679 dev->runtime_d3cold = false; 2680 } 2681 2682 return error; 2683 } 2684 2685 /** 2686 * pci_dev_run_wake - Check if device can generate run-time wake-up events. 2687 * @dev: Device to check. 2688 * 2689 * Return true if the device itself is capable of generating wake-up events 2690 * (through the platform or using the native PCIe PME) or if the device supports 2691 * PME and one of its upstream bridges can generate wake-up events. 2692 */ 2693 bool pci_dev_run_wake(struct pci_dev *dev) 2694 { 2695 struct pci_bus *bus = dev->bus; 2696 2697 if (!dev->pme_support) 2698 return false; 2699 2700 /* PME-capable in principle, but not from the target power state */ 2701 if (!pci_pme_capable(dev, pci_target_state(dev, true))) 2702 return false; 2703 2704 if (device_can_wakeup(&dev->dev)) 2705 return true; 2706 2707 while (bus->parent) { 2708 struct pci_dev *bridge = bus->self; 2709 2710 if (device_can_wakeup(&bridge->dev)) 2711 return true; 2712 2713 bus = bus->parent; 2714 } 2715 2716 /* We have reached the root bus. */ 2717 if (bus->bridge) 2718 return device_can_wakeup(bus->bridge); 2719 2720 return false; 2721 } 2722 EXPORT_SYMBOL_GPL(pci_dev_run_wake); 2723 2724 /** 2725 * pci_dev_need_resume - Check if it is necessary to resume the device. 2726 * @pci_dev: Device to check. 2727 * 2728 * Return 'true' if the device is not runtime-suspended or it has to be 2729 * reconfigured due to wakeup settings difference between system and runtime 2730 * suspend, or the current power state of it is not suitable for the upcoming 2731 * (system-wide) transition. 2732 */ 2733 bool pci_dev_need_resume(struct pci_dev *pci_dev) 2734 { 2735 struct device *dev = &pci_dev->dev; 2736 pci_power_t target_state; 2737 2738 if (!pm_runtime_suspended(dev) || platform_pci_need_resume(pci_dev)) 2739 return true; 2740 2741 target_state = pci_target_state(pci_dev, device_may_wakeup(dev)); 2742 2743 /* 2744 * If the earlier platform check has not triggered, D3cold is just power 2745 * removal on top of D3hot, so no need to resume the device in that 2746 * case. 2747 */ 2748 return target_state != pci_dev->current_state && 2749 target_state != PCI_D3cold && 2750 pci_dev->current_state != PCI_D3hot; 2751 } 2752 2753 /** 2754 * pci_dev_adjust_pme - Adjust PME setting for a suspended device. 2755 * @pci_dev: Device to check. 2756 * 2757 * If the device is suspended and it is not configured for system wakeup, 2758 * disable PME for it to prevent it from waking up the system unnecessarily. 2759 * 2760 * Note that if the device's power state is D3cold and the platform check in 2761 * pci_dev_need_resume() has not triggered, the device's configuration need not 2762 * be changed. 2763 */ 2764 void pci_dev_adjust_pme(struct pci_dev *pci_dev) 2765 { 2766 struct device *dev = &pci_dev->dev; 2767 2768 spin_lock_irq(&dev->power.lock); 2769 2770 if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) && 2771 pci_dev->current_state < PCI_D3cold) 2772 __pci_pme_active(pci_dev, false); 2773 2774 spin_unlock_irq(&dev->power.lock); 2775 } 2776 2777 /** 2778 * pci_dev_complete_resume - Finalize resume from system sleep for a device. 2779 * @pci_dev: Device to handle. 2780 * 2781 * If the device is runtime suspended and wakeup-capable, enable PME for it as 2782 * it might have been disabled during the prepare phase of system suspend if 2783 * the device was not configured for system wakeup. 2784 */ 2785 void pci_dev_complete_resume(struct pci_dev *pci_dev) 2786 { 2787 struct device *dev = &pci_dev->dev; 2788 2789 if (!pci_dev_run_wake(pci_dev)) 2790 return; 2791 2792 spin_lock_irq(&dev->power.lock); 2793 2794 if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold) 2795 __pci_pme_active(pci_dev, true); 2796 2797 spin_unlock_irq(&dev->power.lock); 2798 } 2799 2800 void pci_config_pm_runtime_get(struct pci_dev *pdev) 2801 { 2802 struct device *dev = &pdev->dev; 2803 struct device *parent = dev->parent; 2804 2805 if (parent) 2806 pm_runtime_get_sync(parent); 2807 pm_runtime_get_noresume(dev); 2808 /* 2809 * pdev->current_state is set to PCI_D3cold during suspending, 2810 * so wait until suspending completes 2811 */ 2812 pm_runtime_barrier(dev); 2813 /* 2814 * Only need to resume devices in D3cold, because config 2815 * registers are still accessible for devices suspended but 2816 * not in D3cold. 2817 */ 2818 if (pdev->current_state == PCI_D3cold) 2819 pm_runtime_resume(dev); 2820 } 2821 2822 void pci_config_pm_runtime_put(struct pci_dev *pdev) 2823 { 2824 struct device *dev = &pdev->dev; 2825 struct device *parent = dev->parent; 2826 2827 pm_runtime_put(dev); 2828 if (parent) 2829 pm_runtime_put_sync(parent); 2830 } 2831 2832 static const struct dmi_system_id bridge_d3_blacklist[] = { 2833 #ifdef CONFIG_X86 2834 { 2835 /* 2836 * Gigabyte X299 root port is not marked as hotplug capable 2837 * which allows Linux to power manage it. However, this 2838 * confuses the BIOS SMI handler so don't power manage root 2839 * ports on that system. 2840 */ 2841 .ident = "X299 DESIGNARE EX-CF", 2842 .matches = { 2843 DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."), 2844 DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"), 2845 }, 2846 }, 2847 #endif 2848 { } 2849 }; 2850 2851 /** 2852 * pci_bridge_d3_possible - Is it possible to put the bridge into D3 2853 * @bridge: Bridge to check 2854 * 2855 * This function checks if it is possible to move the bridge to D3. 2856 * Currently we only allow D3 for recent enough PCIe ports and Thunderbolt. 2857 */ 2858 bool pci_bridge_d3_possible(struct pci_dev *bridge) 2859 { 2860 if (!pci_is_pcie(bridge)) 2861 return false; 2862 2863 switch (pci_pcie_type(bridge)) { 2864 case PCI_EXP_TYPE_ROOT_PORT: 2865 case PCI_EXP_TYPE_UPSTREAM: 2866 case PCI_EXP_TYPE_DOWNSTREAM: 2867 if (pci_bridge_d3_disable) 2868 return false; 2869 2870 /* 2871 * Hotplug ports handled by firmware in System Management Mode 2872 * may not be put into D3 by the OS (Thunderbolt on non-Macs). 2873 */ 2874 if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge)) 2875 return false; 2876 2877 if (pci_bridge_d3_force) 2878 return true; 2879 2880 /* Even the oldest 2010 Thunderbolt controller supports D3. */ 2881 if (bridge->is_thunderbolt) 2882 return true; 2883 2884 /* Platform might know better if the bridge supports D3 */ 2885 if (platform_pci_bridge_d3(bridge)) 2886 return true; 2887 2888 /* 2889 * Hotplug ports handled natively by the OS were not validated 2890 * by vendors for runtime D3 at least until 2018 because there 2891 * was no OS support. 2892 */ 2893 if (bridge->is_hotplug_bridge) 2894 return false; 2895 2896 if (dmi_check_system(bridge_d3_blacklist)) 2897 return false; 2898 2899 /* 2900 * It should be safe to put PCIe ports from 2015 or newer 2901 * to D3. 2902 */ 2903 if (dmi_get_bios_year() >= 2015) 2904 return true; 2905 break; 2906 } 2907 2908 return false; 2909 } 2910 2911 static int pci_dev_check_d3cold(struct pci_dev *dev, void *data) 2912 { 2913 bool *d3cold_ok = data; 2914 2915 if (/* The device needs to be allowed to go D3cold ... */ 2916 dev->no_d3cold || !dev->d3cold_allowed || 2917 2918 /* ... and if it is wakeup capable to do so from D3cold. */ 2919 (device_may_wakeup(&dev->dev) && 2920 !pci_pme_capable(dev, PCI_D3cold)) || 2921 2922 /* If it is a bridge it must be allowed to go to D3. */ 2923 !pci_power_manageable(dev)) 2924 2925 *d3cold_ok = false; 2926 2927 return !*d3cold_ok; 2928 } 2929 2930 /* 2931 * pci_bridge_d3_update - Update bridge D3 capabilities 2932 * @dev: PCI device which is changed 2933 * 2934 * Update upstream bridge PM capabilities accordingly depending on if the 2935 * device PM configuration was changed or the device is being removed. The 2936 * change is also propagated upstream. 2937 */ 2938 void pci_bridge_d3_update(struct pci_dev *dev) 2939 { 2940 bool remove = !device_is_registered(&dev->dev); 2941 struct pci_dev *bridge; 2942 bool d3cold_ok = true; 2943 2944 bridge = pci_upstream_bridge(dev); 2945 if (!bridge || !pci_bridge_d3_possible(bridge)) 2946 return; 2947 2948 /* 2949 * If D3 is currently allowed for the bridge, removing one of its 2950 * children won't change that. 2951 */ 2952 if (remove && bridge->bridge_d3) 2953 return; 2954 2955 /* 2956 * If D3 is currently allowed for the bridge and a child is added or 2957 * changed, disallowance of D3 can only be caused by that child, so 2958 * we only need to check that single device, not any of its siblings. 2959 * 2960 * If D3 is currently not allowed for the bridge, checking the device 2961 * first may allow us to skip checking its siblings. 2962 */ 2963 if (!remove) 2964 pci_dev_check_d3cold(dev, &d3cold_ok); 2965 2966 /* 2967 * If D3 is currently not allowed for the bridge, this may be caused 2968 * either by the device being changed/removed or any of its siblings, 2969 * so we need to go through all children to find out if one of them 2970 * continues to block D3. 2971 */ 2972 if (d3cold_ok && !bridge->bridge_d3) 2973 pci_walk_bus(bridge->subordinate, pci_dev_check_d3cold, 2974 &d3cold_ok); 2975 2976 if (bridge->bridge_d3 != d3cold_ok) { 2977 bridge->bridge_d3 = d3cold_ok; 2978 /* Propagate change to upstream bridges */ 2979 pci_bridge_d3_update(bridge); 2980 } 2981 } 2982 2983 /** 2984 * pci_d3cold_enable - Enable D3cold for device 2985 * @dev: PCI device to handle 2986 * 2987 * This function can be used in drivers to enable D3cold from the device 2988 * they handle. It also updates upstream PCI bridge PM capabilities 2989 * accordingly. 2990 */ 2991 void pci_d3cold_enable(struct pci_dev *dev) 2992 { 2993 if (dev->no_d3cold) { 2994 dev->no_d3cold = false; 2995 pci_bridge_d3_update(dev); 2996 } 2997 } 2998 EXPORT_SYMBOL_GPL(pci_d3cold_enable); 2999 3000 /** 3001 * pci_d3cold_disable - Disable D3cold for device 3002 * @dev: PCI device to handle 3003 * 3004 * This function can be used in drivers to disable D3cold from the device 3005 * they handle. It also updates upstream PCI bridge PM capabilities 3006 * accordingly. 3007 */ 3008 void pci_d3cold_disable(struct pci_dev *dev) 3009 { 3010 if (!dev->no_d3cold) { 3011 dev->no_d3cold = true; 3012 pci_bridge_d3_update(dev); 3013 } 3014 } 3015 EXPORT_SYMBOL_GPL(pci_d3cold_disable); 3016 3017 /** 3018 * pci_pm_init - Initialize PM functions of given PCI device 3019 * @dev: PCI device to handle. 3020 */ 3021 void pci_pm_init(struct pci_dev *dev) 3022 { 3023 int pm; 3024 u16 status; 3025 u16 pmc; 3026 3027 pm_runtime_forbid(&dev->dev); 3028 pm_runtime_set_active(&dev->dev); 3029 pm_runtime_enable(&dev->dev); 3030 device_enable_async_suspend(&dev->dev); 3031 dev->wakeup_prepared = false; 3032 3033 dev->pm_cap = 0; 3034 dev->pme_support = 0; 3035 3036 /* find PCI PM capability in list */ 3037 pm = pci_find_capability(dev, PCI_CAP_ID_PM); 3038 if (!pm) 3039 return; 3040 /* Check device's ability to generate PME# */ 3041 pci_read_config_word(dev, pm + PCI_PM_PMC, &pmc); 3042 3043 if ((pmc & PCI_PM_CAP_VER_MASK) > 3) { 3044 pci_err(dev, "unsupported PM cap regs version (%u)\n", 3045 pmc & PCI_PM_CAP_VER_MASK); 3046 return; 3047 } 3048 3049 dev->pm_cap = pm; 3050 dev->d3hot_delay = PCI_PM_D3HOT_WAIT; 3051 dev->d3cold_delay = PCI_PM_D3COLD_WAIT; 3052 dev->bridge_d3 = pci_bridge_d3_possible(dev); 3053 dev->d3cold_allowed = true; 3054 3055 dev->d1_support = false; 3056 dev->d2_support = false; 3057 if (!pci_no_d1d2(dev)) { 3058 if (pmc & PCI_PM_CAP_D1) 3059 dev->d1_support = true; 3060 if (pmc & PCI_PM_CAP_D2) 3061 dev->d2_support = true; 3062 3063 if (dev->d1_support || dev->d2_support) 3064 pci_info(dev, "supports%s%s\n", 3065 dev->d1_support ? " D1" : "", 3066 dev->d2_support ? " D2" : ""); 3067 } 3068 3069 pmc &= PCI_PM_CAP_PME_MASK; 3070 if (pmc) { 3071 pci_info(dev, "PME# supported from%s%s%s%s%s\n", 3072 (pmc & PCI_PM_CAP_PME_D0) ? " D0" : "", 3073 (pmc & PCI_PM_CAP_PME_D1) ? " D1" : "", 3074 (pmc & PCI_PM_CAP_PME_D2) ? " D2" : "", 3075 (pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "", 3076 (pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : ""); 3077 dev->pme_support = pmc >> PCI_PM_CAP_PME_SHIFT; 3078 dev->pme_poll = true; 3079 /* 3080 * Make device's PM flags reflect the wake-up capability, but 3081 * let the user space enable it to wake up the system as needed. 3082 */ 3083 device_set_wakeup_capable(&dev->dev, true); 3084 /* Disable the PME# generation functionality */ 3085 pci_pme_active(dev, false); 3086 } 3087 3088 pci_read_config_word(dev, PCI_STATUS, &status); 3089 if (status & PCI_STATUS_IMM_READY) 3090 dev->imm_ready = 1; 3091 } 3092 3093 static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop) 3094 { 3095 unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI; 3096 3097 switch (prop) { 3098 case PCI_EA_P_MEM: 3099 case PCI_EA_P_VF_MEM: 3100 flags |= IORESOURCE_MEM; 3101 break; 3102 case PCI_EA_P_MEM_PREFETCH: 3103 case PCI_EA_P_VF_MEM_PREFETCH: 3104 flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH; 3105 break; 3106 case PCI_EA_P_IO: 3107 flags |= IORESOURCE_IO; 3108 break; 3109 default: 3110 return 0; 3111 } 3112 3113 return flags; 3114 } 3115 3116 static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei, 3117 u8 prop) 3118 { 3119 if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO) 3120 return &dev->resource[bei]; 3121 #ifdef CONFIG_PCI_IOV 3122 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 && 3123 (prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH)) 3124 return &dev->resource[PCI_IOV_RESOURCES + 3125 bei - PCI_EA_BEI_VF_BAR0]; 3126 #endif 3127 else if (bei == PCI_EA_BEI_ROM) 3128 return &dev->resource[PCI_ROM_RESOURCE]; 3129 else 3130 return NULL; 3131 } 3132 3133 /* Read an Enhanced Allocation (EA) entry */ 3134 static int pci_ea_read(struct pci_dev *dev, int offset) 3135 { 3136 struct resource *res; 3137 int ent_size, ent_offset = offset; 3138 resource_size_t start, end; 3139 unsigned long flags; 3140 u32 dw0, bei, base, max_offset; 3141 u8 prop; 3142 bool support_64 = (sizeof(resource_size_t) >= 8); 3143 3144 pci_read_config_dword(dev, ent_offset, &dw0); 3145 ent_offset += 4; 3146 3147 /* Entry size field indicates DWORDs after 1st */ 3148 ent_size = ((dw0 & PCI_EA_ES) + 1) << 2; 3149 3150 if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */ 3151 goto out; 3152 3153 bei = (dw0 & PCI_EA_BEI) >> 4; 3154 prop = (dw0 & PCI_EA_PP) >> 8; 3155 3156 /* 3157 * If the Property is in the reserved range, try the Secondary 3158 * Property instead. 3159 */ 3160 if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED) 3161 prop = (dw0 & PCI_EA_SP) >> 16; 3162 if (prop > PCI_EA_P_BRIDGE_IO) 3163 goto out; 3164 3165 res = pci_ea_get_resource(dev, bei, prop); 3166 if (!res) { 3167 pci_err(dev, "Unsupported EA entry BEI: %u\n", bei); 3168 goto out; 3169 } 3170 3171 flags = pci_ea_flags(dev, prop); 3172 if (!flags) { 3173 pci_err(dev, "Unsupported EA properties: %#x\n", prop); 3174 goto out; 3175 } 3176 3177 /* Read Base */ 3178 pci_read_config_dword(dev, ent_offset, &base); 3179 start = (base & PCI_EA_FIELD_MASK); 3180 ent_offset += 4; 3181 3182 /* Read MaxOffset */ 3183 pci_read_config_dword(dev, ent_offset, &max_offset); 3184 ent_offset += 4; 3185 3186 /* Read Base MSBs (if 64-bit entry) */ 3187 if (base & PCI_EA_IS_64) { 3188 u32 base_upper; 3189 3190 pci_read_config_dword(dev, ent_offset, &base_upper); 3191 ent_offset += 4; 3192 3193 flags |= IORESOURCE_MEM_64; 3194 3195 /* entry starts above 32-bit boundary, can't use */ 3196 if (!support_64 && base_upper) 3197 goto out; 3198 3199 if (support_64) 3200 start |= ((u64)base_upper << 32); 3201 } 3202 3203 end = start + (max_offset | 0x03); 3204 3205 /* Read MaxOffset MSBs (if 64-bit entry) */ 3206 if (max_offset & PCI_EA_IS_64) { 3207 u32 max_offset_upper; 3208 3209 pci_read_config_dword(dev, ent_offset, &max_offset_upper); 3210 ent_offset += 4; 3211 3212 flags |= IORESOURCE_MEM_64; 3213 3214 /* entry too big, can't use */ 3215 if (!support_64 && max_offset_upper) 3216 goto out; 3217 3218 if (support_64) 3219 end += ((u64)max_offset_upper << 32); 3220 } 3221 3222 if (end < start) { 3223 pci_err(dev, "EA Entry crosses address boundary\n"); 3224 goto out; 3225 } 3226 3227 if (ent_size != ent_offset - offset) { 3228 pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n", 3229 ent_size, ent_offset - offset); 3230 goto out; 3231 } 3232 3233 res->name = pci_name(dev); 3234 res->start = start; 3235 res->end = end; 3236 res->flags = flags; 3237 3238 if (bei <= PCI_EA_BEI_BAR5) 3239 pci_info(dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 3240 bei, res, prop); 3241 else if (bei == PCI_EA_BEI_ROM) 3242 pci_info(dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n", 3243 res, prop); 3244 else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5) 3245 pci_info(dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n", 3246 bei - PCI_EA_BEI_VF_BAR0, res, prop); 3247 else 3248 pci_info(dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n", 3249 bei, res, prop); 3250 3251 out: 3252 return offset + ent_size; 3253 } 3254 3255 /* Enhanced Allocation Initialization */ 3256 void pci_ea_init(struct pci_dev *dev) 3257 { 3258 int ea; 3259 u8 num_ent; 3260 int offset; 3261 int i; 3262 3263 /* find PCI EA capability in list */ 3264 ea = pci_find_capability(dev, PCI_CAP_ID_EA); 3265 if (!ea) 3266 return; 3267 3268 /* determine the number of entries */ 3269 pci_bus_read_config_byte(dev->bus, dev->devfn, ea + PCI_EA_NUM_ENT, 3270 &num_ent); 3271 num_ent &= PCI_EA_NUM_ENT_MASK; 3272 3273 offset = ea + PCI_EA_FIRST_ENT; 3274 3275 /* Skip DWORD 2 for type 1 functions */ 3276 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) 3277 offset += 4; 3278 3279 /* parse each EA entry */ 3280 for (i = 0; i < num_ent; ++i) 3281 offset = pci_ea_read(dev, offset); 3282 } 3283 3284 static void pci_add_saved_cap(struct pci_dev *pci_dev, 3285 struct pci_cap_saved_state *new_cap) 3286 { 3287 hlist_add_head(&new_cap->next, &pci_dev->saved_cap_space); 3288 } 3289 3290 /** 3291 * _pci_add_cap_save_buffer - allocate buffer for saving given 3292 * capability registers 3293 * @dev: the PCI device 3294 * @cap: the capability to allocate the buffer for 3295 * @extended: Standard or Extended capability ID 3296 * @size: requested size of the buffer 3297 */ 3298 static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap, 3299 bool extended, unsigned int size) 3300 { 3301 int pos; 3302 struct pci_cap_saved_state *save_state; 3303 3304 if (extended) 3305 pos = pci_find_ext_capability(dev, cap); 3306 else 3307 pos = pci_find_capability(dev, cap); 3308 3309 if (!pos) 3310 return 0; 3311 3312 save_state = kzalloc(sizeof(*save_state) + size, GFP_KERNEL); 3313 if (!save_state) 3314 return -ENOMEM; 3315 3316 save_state->cap.cap_nr = cap; 3317 save_state->cap.cap_extended = extended; 3318 save_state->cap.size = size; 3319 pci_add_saved_cap(dev, save_state); 3320 3321 return 0; 3322 } 3323 3324 int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size) 3325 { 3326 return _pci_add_cap_save_buffer(dev, cap, false, size); 3327 } 3328 3329 int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size) 3330 { 3331 return _pci_add_cap_save_buffer(dev, cap, true, size); 3332 } 3333 3334 /** 3335 * pci_allocate_cap_save_buffers - allocate buffers for saving capabilities 3336 * @dev: the PCI device 3337 */ 3338 void pci_allocate_cap_save_buffers(struct pci_dev *dev) 3339 { 3340 int error; 3341 3342 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP, 3343 PCI_EXP_SAVE_REGS * sizeof(u16)); 3344 if (error) 3345 pci_err(dev, "unable to preallocate PCI Express save buffer\n"); 3346 3347 error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, sizeof(u16)); 3348 if (error) 3349 pci_err(dev, "unable to preallocate PCI-X save buffer\n"); 3350 3351 error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR, 3352 2 * sizeof(u16)); 3353 if (error) 3354 pci_err(dev, "unable to allocate suspend buffer for LTR\n"); 3355 3356 error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_L1SS, 3357 2 * sizeof(u32)); 3358 if (error) 3359 pci_err(dev, "unable to allocate suspend buffer for ASPM-L1SS\n"); 3360 3361 pci_allocate_vc_save_buffers(dev); 3362 } 3363 3364 void pci_free_cap_save_buffers(struct pci_dev *dev) 3365 { 3366 struct pci_cap_saved_state *tmp; 3367 struct hlist_node *n; 3368 3369 hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next) 3370 kfree(tmp); 3371 } 3372 3373 /** 3374 * pci_configure_ari - enable or disable ARI forwarding 3375 * @dev: the PCI device 3376 * 3377 * If @dev and its upstream bridge both support ARI, enable ARI in the 3378 * bridge. Otherwise, disable ARI in the bridge. 3379 */ 3380 void pci_configure_ari(struct pci_dev *dev) 3381 { 3382 u32 cap; 3383 struct pci_dev *bridge; 3384 3385 if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn) 3386 return; 3387 3388 bridge = dev->bus->self; 3389 if (!bridge) 3390 return; 3391 3392 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3393 if (!(cap & PCI_EXP_DEVCAP2_ARI)) 3394 return; 3395 3396 if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) { 3397 pcie_capability_set_word(bridge, PCI_EXP_DEVCTL2, 3398 PCI_EXP_DEVCTL2_ARI); 3399 bridge->ari_enabled = 1; 3400 } else { 3401 pcie_capability_clear_word(bridge, PCI_EXP_DEVCTL2, 3402 PCI_EXP_DEVCTL2_ARI); 3403 bridge->ari_enabled = 0; 3404 } 3405 } 3406 3407 static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags) 3408 { 3409 int pos; 3410 u16 cap, ctrl; 3411 3412 pos = pdev->acs_cap; 3413 if (!pos) 3414 return false; 3415 3416 /* 3417 * Except for egress control, capabilities are either required 3418 * or only required if controllable. Features missing from the 3419 * capability field can therefore be assumed as hard-wired enabled. 3420 */ 3421 pci_read_config_word(pdev, pos + PCI_ACS_CAP, &cap); 3422 acs_flags &= (cap | PCI_ACS_EC); 3423 3424 pci_read_config_word(pdev, pos + PCI_ACS_CTRL, &ctrl); 3425 return (ctrl & acs_flags) == acs_flags; 3426 } 3427 3428 /** 3429 * pci_acs_enabled - test ACS against required flags for a given device 3430 * @pdev: device to test 3431 * @acs_flags: required PCI ACS flags 3432 * 3433 * Return true if the device supports the provided flags. Automatically 3434 * filters out flags that are not implemented on multifunction devices. 3435 * 3436 * Note that this interface checks the effective ACS capabilities of the 3437 * device rather than the actual capabilities. For instance, most single 3438 * function endpoints are not required to support ACS because they have no 3439 * opportunity for peer-to-peer access. We therefore return 'true' 3440 * regardless of whether the device exposes an ACS capability. This makes 3441 * it much easier for callers of this function to ignore the actual type 3442 * or topology of the device when testing ACS support. 3443 */ 3444 bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags) 3445 { 3446 int ret; 3447 3448 ret = pci_dev_specific_acs_enabled(pdev, acs_flags); 3449 if (ret >= 0) 3450 return ret > 0; 3451 3452 /* 3453 * Conventional PCI and PCI-X devices never support ACS, either 3454 * effectively or actually. The shared bus topology implies that 3455 * any device on the bus can receive or snoop DMA. 3456 */ 3457 if (!pci_is_pcie(pdev)) 3458 return false; 3459 3460 switch (pci_pcie_type(pdev)) { 3461 /* 3462 * PCI/X-to-PCIe bridges are not specifically mentioned by the spec, 3463 * but since their primary interface is PCI/X, we conservatively 3464 * handle them as we would a non-PCIe device. 3465 */ 3466 case PCI_EXP_TYPE_PCIE_BRIDGE: 3467 /* 3468 * PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never 3469 * applicable... must never implement an ACS Extended Capability...". 3470 * This seems arbitrary, but we take a conservative interpretation 3471 * of this statement. 3472 */ 3473 case PCI_EXP_TYPE_PCI_BRIDGE: 3474 case PCI_EXP_TYPE_RC_EC: 3475 return false; 3476 /* 3477 * PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should 3478 * implement ACS in order to indicate their peer-to-peer capabilities, 3479 * regardless of whether they are single- or multi-function devices. 3480 */ 3481 case PCI_EXP_TYPE_DOWNSTREAM: 3482 case PCI_EXP_TYPE_ROOT_PORT: 3483 return pci_acs_flags_enabled(pdev, acs_flags); 3484 /* 3485 * PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be 3486 * implemented by the remaining PCIe types to indicate peer-to-peer 3487 * capabilities, but only when they are part of a multifunction 3488 * device. The footnote for section 6.12 indicates the specific 3489 * PCIe types included here. 3490 */ 3491 case PCI_EXP_TYPE_ENDPOINT: 3492 case PCI_EXP_TYPE_UPSTREAM: 3493 case PCI_EXP_TYPE_LEG_END: 3494 case PCI_EXP_TYPE_RC_END: 3495 if (!pdev->multifunction) 3496 break; 3497 3498 return pci_acs_flags_enabled(pdev, acs_flags); 3499 } 3500 3501 /* 3502 * PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable 3503 * to single function devices with the exception of downstream ports. 3504 */ 3505 return true; 3506 } 3507 3508 /** 3509 * pci_acs_path_enabled - test ACS flags from start to end in a hierarchy 3510 * @start: starting downstream device 3511 * @end: ending upstream device or NULL to search to the root bus 3512 * @acs_flags: required flags 3513 * 3514 * Walk up a device tree from start to end testing PCI ACS support. If 3515 * any step along the way does not support the required flags, return false. 3516 */ 3517 bool pci_acs_path_enabled(struct pci_dev *start, 3518 struct pci_dev *end, u16 acs_flags) 3519 { 3520 struct pci_dev *pdev, *parent = start; 3521 3522 do { 3523 pdev = parent; 3524 3525 if (!pci_acs_enabled(pdev, acs_flags)) 3526 return false; 3527 3528 if (pci_is_root_bus(pdev->bus)) 3529 return (end == NULL); 3530 3531 parent = pdev->bus->self; 3532 } while (pdev != end); 3533 3534 return true; 3535 } 3536 3537 /** 3538 * pci_acs_init - Initialize ACS if hardware supports it 3539 * @dev: the PCI device 3540 */ 3541 void pci_acs_init(struct pci_dev *dev) 3542 { 3543 dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS); 3544 3545 /* 3546 * Attempt to enable ACS regardless of capability because some Root 3547 * Ports (e.g. those quirked with *_intel_pch_acs_*) do not have 3548 * the standard ACS capability but still support ACS via those 3549 * quirks. 3550 */ 3551 pci_enable_acs(dev); 3552 } 3553 3554 /** 3555 * pci_rebar_find_pos - find position of resize ctrl reg for BAR 3556 * @pdev: PCI device 3557 * @bar: BAR to find 3558 * 3559 * Helper to find the position of the ctrl register for a BAR. 3560 * Returns -ENOTSUPP if resizable BARs are not supported at all. 3561 * Returns -ENOENT if no ctrl register for the BAR could be found. 3562 */ 3563 static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) 3564 { 3565 unsigned int pos, nbars, i; 3566 u32 ctrl; 3567 3568 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR); 3569 if (!pos) 3570 return -ENOTSUPP; 3571 3572 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3573 nbars = (ctrl & PCI_REBAR_CTRL_NBAR_MASK) >> 3574 PCI_REBAR_CTRL_NBAR_SHIFT; 3575 3576 for (i = 0; i < nbars; i++, pos += 8) { 3577 int bar_idx; 3578 3579 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3580 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX; 3581 if (bar_idx == bar) 3582 return pos; 3583 } 3584 3585 return -ENOENT; 3586 } 3587 3588 /** 3589 * pci_rebar_get_possible_sizes - get possible sizes for BAR 3590 * @pdev: PCI device 3591 * @bar: BAR to query 3592 * 3593 * Get the possible sizes of a resizable BAR as bitmask defined in the spec 3594 * (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable. 3595 */ 3596 u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar) 3597 { 3598 int pos; 3599 u32 cap; 3600 3601 pos = pci_rebar_find_pos(pdev, bar); 3602 if (pos < 0) 3603 return 0; 3604 3605 pci_read_config_dword(pdev, pos + PCI_REBAR_CAP, &cap); 3606 cap &= PCI_REBAR_CAP_SIZES; 3607 3608 /* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */ 3609 if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f && 3610 bar == 0 && cap == 0x7000) 3611 cap = 0x3f000; 3612 3613 return cap >> 4; 3614 } 3615 EXPORT_SYMBOL(pci_rebar_get_possible_sizes); 3616 3617 /** 3618 * pci_rebar_get_current_size - get the current size of a BAR 3619 * @pdev: PCI device 3620 * @bar: BAR to set size to 3621 * 3622 * Read the size of a BAR from the resizable BAR config. 3623 * Returns size if found or negative error code. 3624 */ 3625 int pci_rebar_get_current_size(struct pci_dev *pdev, int bar) 3626 { 3627 int pos; 3628 u32 ctrl; 3629 3630 pos = pci_rebar_find_pos(pdev, bar); 3631 if (pos < 0) 3632 return pos; 3633 3634 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3635 return (ctrl & PCI_REBAR_CTRL_BAR_SIZE) >> PCI_REBAR_CTRL_BAR_SHIFT; 3636 } 3637 3638 /** 3639 * pci_rebar_set_size - set a new size for a BAR 3640 * @pdev: PCI device 3641 * @bar: BAR to set size to 3642 * @size: new size as defined in the spec (0=1MB, 19=512GB) 3643 * 3644 * Set the new size of a BAR as defined in the spec. 3645 * Returns zero if resizing was successful, error code otherwise. 3646 */ 3647 int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size) 3648 { 3649 int pos; 3650 u32 ctrl; 3651 3652 pos = pci_rebar_find_pos(pdev, bar); 3653 if (pos < 0) 3654 return pos; 3655 3656 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl); 3657 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE; 3658 ctrl |= size << PCI_REBAR_CTRL_BAR_SHIFT; 3659 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl); 3660 return 0; 3661 } 3662 3663 /** 3664 * pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port 3665 * @dev: the PCI device 3666 * @cap_mask: mask of desired AtomicOp sizes, including one or more of: 3667 * PCI_EXP_DEVCAP2_ATOMIC_COMP32 3668 * PCI_EXP_DEVCAP2_ATOMIC_COMP64 3669 * PCI_EXP_DEVCAP2_ATOMIC_COMP128 3670 * 3671 * Return 0 if all upstream bridges support AtomicOp routing, egress 3672 * blocking is disabled on all upstream ports, and the root port supports 3673 * the requested completion capabilities (32-bit, 64-bit and/or 128-bit 3674 * AtomicOp completion), or negative otherwise. 3675 */ 3676 int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask) 3677 { 3678 struct pci_bus *bus = dev->bus; 3679 struct pci_dev *bridge; 3680 u32 cap, ctl2; 3681 3682 if (!pci_is_pcie(dev)) 3683 return -EINVAL; 3684 3685 /* 3686 * Per PCIe r4.0, sec 6.15, endpoints and root ports may be 3687 * AtomicOp requesters. For now, we only support endpoints as 3688 * requesters and root ports as completers. No endpoints as 3689 * completers, and no peer-to-peer. 3690 */ 3691 3692 switch (pci_pcie_type(dev)) { 3693 case PCI_EXP_TYPE_ENDPOINT: 3694 case PCI_EXP_TYPE_LEG_END: 3695 case PCI_EXP_TYPE_RC_END: 3696 break; 3697 default: 3698 return -EINVAL; 3699 } 3700 3701 while (bus->parent) { 3702 bridge = bus->self; 3703 3704 pcie_capability_read_dword(bridge, PCI_EXP_DEVCAP2, &cap); 3705 3706 switch (pci_pcie_type(bridge)) { 3707 /* Ensure switch ports support AtomicOp routing */ 3708 case PCI_EXP_TYPE_UPSTREAM: 3709 case PCI_EXP_TYPE_DOWNSTREAM: 3710 if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE)) 3711 return -EINVAL; 3712 break; 3713 3714 /* Ensure root port supports all the sizes we care about */ 3715 case PCI_EXP_TYPE_ROOT_PORT: 3716 if ((cap & cap_mask) != cap_mask) 3717 return -EINVAL; 3718 break; 3719 } 3720 3721 /* Ensure upstream ports don't block AtomicOps on egress */ 3722 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_UPSTREAM) { 3723 pcie_capability_read_dword(bridge, PCI_EXP_DEVCTL2, 3724 &ctl2); 3725 if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK) 3726 return -EINVAL; 3727 } 3728 3729 bus = bus->parent; 3730 } 3731 3732 pcie_capability_set_word(dev, PCI_EXP_DEVCTL2, 3733 PCI_EXP_DEVCTL2_ATOMIC_REQ); 3734 return 0; 3735 } 3736 EXPORT_SYMBOL(pci_enable_atomic_ops_to_root); 3737 3738 /** 3739 * pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge 3740 * @dev: the PCI device 3741 * @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) 3742 * 3743 * Perform INTx swizzling for a device behind one level of bridge. This is 3744 * required by section 9.1 of the PCI-to-PCI bridge specification for devices 3745 * behind bridges on add-in cards. For devices with ARI enabled, the slot 3746 * number is always 0 (see the Implementation Note in section 2.2.8.1 of 3747 * the PCI Express Base Specification, Revision 2.1) 3748 */ 3749 u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin) 3750 { 3751 int slot; 3752 3753 if (pci_ari_enabled(dev->bus)) 3754 slot = 0; 3755 else 3756 slot = PCI_SLOT(dev->devfn); 3757 3758 return (((pin - 1) + slot) % 4) + 1; 3759 } 3760 3761 int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge) 3762 { 3763 u8 pin; 3764 3765 pin = dev->pin; 3766 if (!pin) 3767 return -1; 3768 3769 while (!pci_is_root_bus(dev->bus)) { 3770 pin = pci_swizzle_interrupt_pin(dev, pin); 3771 dev = dev->bus->self; 3772 } 3773 *bridge = dev; 3774 return pin; 3775 } 3776 3777 /** 3778 * pci_common_swizzle - swizzle INTx all the way to root bridge 3779 * @dev: the PCI device 3780 * @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD) 3781 * 3782 * Perform INTx swizzling for a device. This traverses through all PCI-to-PCI 3783 * bridges all the way up to a PCI root bus. 3784 */ 3785 u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp) 3786 { 3787 u8 pin = *pinp; 3788 3789 while (!pci_is_root_bus(dev->bus)) { 3790 pin = pci_swizzle_interrupt_pin(dev, pin); 3791 dev = dev->bus->self; 3792 } 3793 *pinp = pin; 3794 return PCI_SLOT(dev->devfn); 3795 } 3796 EXPORT_SYMBOL_GPL(pci_common_swizzle); 3797 3798 /** 3799 * pci_release_region - Release a PCI bar 3800 * @pdev: PCI device whose resources were previously reserved by 3801 * pci_request_region() 3802 * @bar: BAR to release 3803 * 3804 * Releases the PCI I/O and memory resources previously reserved by a 3805 * successful call to pci_request_region(). Call this function only 3806 * after all use of the PCI regions has ceased. 3807 */ 3808 void pci_release_region(struct pci_dev *pdev, int bar) 3809 { 3810 struct pci_devres *dr; 3811 3812 if (pci_resource_len(pdev, bar) == 0) 3813 return; 3814 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) 3815 release_region(pci_resource_start(pdev, bar), 3816 pci_resource_len(pdev, bar)); 3817 else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) 3818 release_mem_region(pci_resource_start(pdev, bar), 3819 pci_resource_len(pdev, bar)); 3820 3821 dr = find_pci_dr(pdev); 3822 if (dr) 3823 dr->region_mask &= ~(1 << bar); 3824 } 3825 EXPORT_SYMBOL(pci_release_region); 3826 3827 /** 3828 * __pci_request_region - Reserved PCI I/O and memory resource 3829 * @pdev: PCI device whose resources are to be reserved 3830 * @bar: BAR to be reserved 3831 * @res_name: Name to be associated with resource. 3832 * @exclusive: whether the region access is exclusive or not 3833 * 3834 * Mark the PCI region associated with PCI device @pdev BAR @bar as 3835 * being reserved by owner @res_name. Do not access any 3836 * address inside the PCI regions unless this call returns 3837 * successfully. 3838 * 3839 * If @exclusive is set, then the region is marked so that userspace 3840 * is explicitly not allowed to map the resource via /dev/mem or 3841 * sysfs MMIO access. 3842 * 3843 * Returns 0 on success, or %EBUSY on error. A warning 3844 * message is also printed on failure. 3845 */ 3846 static int __pci_request_region(struct pci_dev *pdev, int bar, 3847 const char *res_name, int exclusive) 3848 { 3849 struct pci_devres *dr; 3850 3851 if (pci_resource_len(pdev, bar) == 0) 3852 return 0; 3853 3854 if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) { 3855 if (!request_region(pci_resource_start(pdev, bar), 3856 pci_resource_len(pdev, bar), res_name)) 3857 goto err_out; 3858 } else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) { 3859 if (!__request_mem_region(pci_resource_start(pdev, bar), 3860 pci_resource_len(pdev, bar), res_name, 3861 exclusive)) 3862 goto err_out; 3863 } 3864 3865 dr = find_pci_dr(pdev); 3866 if (dr) 3867 dr->region_mask |= 1 << bar; 3868 3869 return 0; 3870 3871 err_out: 3872 pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar, 3873 &pdev->resource[bar]); 3874 return -EBUSY; 3875 } 3876 3877 /** 3878 * pci_request_region - Reserve PCI I/O and memory resource 3879 * @pdev: PCI device whose resources are to be reserved 3880 * @bar: BAR to be reserved 3881 * @res_name: Name to be associated with resource 3882 * 3883 * Mark the PCI region associated with PCI device @pdev BAR @bar as 3884 * being reserved by owner @res_name. Do not access any 3885 * address inside the PCI regions unless this call returns 3886 * successfully. 3887 * 3888 * Returns 0 on success, or %EBUSY on error. A warning 3889 * message is also printed on failure. 3890 */ 3891 int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name) 3892 { 3893 return __pci_request_region(pdev, bar, res_name, 0); 3894 } 3895 EXPORT_SYMBOL(pci_request_region); 3896 3897 /** 3898 * pci_release_selected_regions - Release selected PCI I/O and memory resources 3899 * @pdev: PCI device whose resources were previously reserved 3900 * @bars: Bitmask of BARs to be released 3901 * 3902 * Release selected PCI I/O and memory resources previously reserved. 3903 * Call this function only after all use of the PCI regions has ceased. 3904 */ 3905 void pci_release_selected_regions(struct pci_dev *pdev, int bars) 3906 { 3907 int i; 3908 3909 for (i = 0; i < PCI_STD_NUM_BARS; i++) 3910 if (bars & (1 << i)) 3911 pci_release_region(pdev, i); 3912 } 3913 EXPORT_SYMBOL(pci_release_selected_regions); 3914 3915 static int __pci_request_selected_regions(struct pci_dev *pdev, int bars, 3916 const char *res_name, int excl) 3917 { 3918 int i; 3919 3920 for (i = 0; i < PCI_STD_NUM_BARS; i++) 3921 if (bars & (1 << i)) 3922 if (__pci_request_region(pdev, i, res_name, excl)) 3923 goto err_out; 3924 return 0; 3925 3926 err_out: 3927 while (--i >= 0) 3928 if (bars & (1 << i)) 3929 pci_release_region(pdev, i); 3930 3931 return -EBUSY; 3932 } 3933 3934 3935 /** 3936 * pci_request_selected_regions - Reserve selected PCI I/O and memory resources 3937 * @pdev: PCI device whose resources are to be reserved 3938 * @bars: Bitmask of BARs to be requested 3939 * @res_name: Name to be associated with resource 3940 */ 3941 int pci_request_selected_regions(struct pci_dev *pdev, int bars, 3942 const char *res_name) 3943 { 3944 return __pci_request_selected_regions(pdev, bars, res_name, 0); 3945 } 3946 EXPORT_SYMBOL(pci_request_selected_regions); 3947 3948 int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars, 3949 const char *res_name) 3950 { 3951 return __pci_request_selected_regions(pdev, bars, res_name, 3952 IORESOURCE_EXCLUSIVE); 3953 } 3954 EXPORT_SYMBOL(pci_request_selected_regions_exclusive); 3955 3956 /** 3957 * pci_release_regions - Release reserved PCI I/O and memory resources 3958 * @pdev: PCI device whose resources were previously reserved by 3959 * pci_request_regions() 3960 * 3961 * Releases all PCI I/O and memory resources previously reserved by a 3962 * successful call to pci_request_regions(). Call this function only 3963 * after all use of the PCI regions has ceased. 3964 */ 3965 3966 void pci_release_regions(struct pci_dev *pdev) 3967 { 3968 pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1); 3969 } 3970 EXPORT_SYMBOL(pci_release_regions); 3971 3972 /** 3973 * pci_request_regions - Reserve PCI I/O and memory resources 3974 * @pdev: PCI device whose resources are to be reserved 3975 * @res_name: Name to be associated with resource. 3976 * 3977 * Mark all PCI regions associated with PCI device @pdev as 3978 * being reserved by owner @res_name. Do not access any 3979 * address inside the PCI regions unless this call returns 3980 * successfully. 3981 * 3982 * Returns 0 on success, or %EBUSY on error. A warning 3983 * message is also printed on failure. 3984 */ 3985 int pci_request_regions(struct pci_dev *pdev, const char *res_name) 3986 { 3987 return pci_request_selected_regions(pdev, 3988 ((1 << PCI_STD_NUM_BARS) - 1), res_name); 3989 } 3990 EXPORT_SYMBOL(pci_request_regions); 3991 3992 /** 3993 * pci_request_regions_exclusive - Reserve PCI I/O and memory resources 3994 * @pdev: PCI device whose resources are to be reserved 3995 * @res_name: Name to be associated with resource. 3996 * 3997 * Mark all PCI regions associated with PCI device @pdev as being reserved 3998 * by owner @res_name. Do not access any address inside the PCI regions 3999 * unless this call returns successfully. 4000 * 4001 * pci_request_regions_exclusive() will mark the region so that /dev/mem 4002 * and the sysfs MMIO access will not be allowed. 4003 * 4004 * Returns 0 on success, or %EBUSY on error. A warning message is also 4005 * printed on failure. 4006 */ 4007 int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name) 4008 { 4009 return pci_request_selected_regions_exclusive(pdev, 4010 ((1 << PCI_STD_NUM_BARS) - 1), res_name); 4011 } 4012 EXPORT_SYMBOL(pci_request_regions_exclusive); 4013 4014 /* 4015 * Record the PCI IO range (expressed as CPU physical address + size). 4016 * Return a negative value if an error has occurred, zero otherwise 4017 */ 4018 int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr, 4019 resource_size_t size) 4020 { 4021 int ret = 0; 4022 #ifdef PCI_IOBASE 4023 struct logic_pio_hwaddr *range; 4024 4025 if (!size || addr + size < addr) 4026 return -EINVAL; 4027 4028 range = kzalloc(sizeof(*range), GFP_ATOMIC); 4029 if (!range) 4030 return -ENOMEM; 4031 4032 range->fwnode = fwnode; 4033 range->size = size; 4034 range->hw_start = addr; 4035 range->flags = LOGIC_PIO_CPU_MMIO; 4036 4037 ret = logic_pio_register_range(range); 4038 if (ret) 4039 kfree(range); 4040 #endif 4041 4042 return ret; 4043 } 4044 4045 phys_addr_t pci_pio_to_address(unsigned long pio) 4046 { 4047 phys_addr_t address = (phys_addr_t)OF_BAD_ADDR; 4048 4049 #ifdef PCI_IOBASE 4050 if (pio >= MMIO_UPPER_LIMIT) 4051 return address; 4052 4053 address = logic_pio_to_hwaddr(pio); 4054 #endif 4055 4056 return address; 4057 } 4058 4059 unsigned long __weak pci_address_to_pio(phys_addr_t address) 4060 { 4061 #ifdef PCI_IOBASE 4062 return logic_pio_trans_cpuaddr(address); 4063 #else 4064 if (address > IO_SPACE_LIMIT) 4065 return (unsigned long)-1; 4066 4067 return (unsigned long) address; 4068 #endif 4069 } 4070 4071 /** 4072 * pci_remap_iospace - Remap the memory mapped I/O space 4073 * @res: Resource describing the I/O space 4074 * @phys_addr: physical address of range to be mapped 4075 * 4076 * Remap the memory mapped I/O space described by the @res and the CPU 4077 * physical address @phys_addr into virtual address space. Only 4078 * architectures that have memory mapped IO functions defined (and the 4079 * PCI_IOBASE value defined) should call this function. 4080 */ 4081 int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr) 4082 { 4083 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 4084 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4085 4086 if (!(res->flags & IORESOURCE_IO)) 4087 return -EINVAL; 4088 4089 if (res->end > IO_SPACE_LIMIT) 4090 return -EINVAL; 4091 4092 return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr, 4093 pgprot_device(PAGE_KERNEL)); 4094 #else 4095 /* 4096 * This architecture does not have memory mapped I/O space, 4097 * so this function should never be called 4098 */ 4099 WARN_ONCE(1, "This architecture does not support memory mapped I/O\n"); 4100 return -ENODEV; 4101 #endif 4102 } 4103 EXPORT_SYMBOL(pci_remap_iospace); 4104 4105 /** 4106 * pci_unmap_iospace - Unmap the memory mapped I/O space 4107 * @res: resource to be unmapped 4108 * 4109 * Unmap the CPU virtual address @res from virtual address space. Only 4110 * architectures that have memory mapped IO functions defined (and the 4111 * PCI_IOBASE value defined) should call this function. 4112 */ 4113 void pci_unmap_iospace(struct resource *res) 4114 { 4115 #if defined(PCI_IOBASE) && defined(CONFIG_MMU) 4116 unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start; 4117 4118 unmap_kernel_range(vaddr, resource_size(res)); 4119 #endif 4120 } 4121 EXPORT_SYMBOL(pci_unmap_iospace); 4122 4123 static void devm_pci_unmap_iospace(struct device *dev, void *ptr) 4124 { 4125 struct resource **res = ptr; 4126 4127 pci_unmap_iospace(*res); 4128 } 4129 4130 /** 4131 * devm_pci_remap_iospace - Managed pci_remap_iospace() 4132 * @dev: Generic device to remap IO address for 4133 * @res: Resource describing the I/O space 4134 * @phys_addr: physical address of range to be mapped 4135 * 4136 * Managed pci_remap_iospace(). Map is automatically unmapped on driver 4137 * detach. 4138 */ 4139 int devm_pci_remap_iospace(struct device *dev, const struct resource *res, 4140 phys_addr_t phys_addr) 4141 { 4142 const struct resource **ptr; 4143 int error; 4144 4145 ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL); 4146 if (!ptr) 4147 return -ENOMEM; 4148 4149 error = pci_remap_iospace(res, phys_addr); 4150 if (error) { 4151 devres_free(ptr); 4152 } else { 4153 *ptr = res; 4154 devres_add(dev, ptr); 4155 } 4156 4157 return error; 4158 } 4159 EXPORT_SYMBOL(devm_pci_remap_iospace); 4160 4161 /** 4162 * devm_pci_remap_cfgspace - Managed pci_remap_cfgspace() 4163 * @dev: Generic device to remap IO address for 4164 * @offset: Resource address to map 4165 * @size: Size of map 4166 * 4167 * Managed pci_remap_cfgspace(). Map is automatically unmapped on driver 4168 * detach. 4169 */ 4170 void __iomem *devm_pci_remap_cfgspace(struct device *dev, 4171 resource_size_t offset, 4172 resource_size_t size) 4173 { 4174 void __iomem **ptr, *addr; 4175 4176 ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL); 4177 if (!ptr) 4178 return NULL; 4179 4180 addr = pci_remap_cfgspace(offset, size); 4181 if (addr) { 4182 *ptr = addr; 4183 devres_add(dev, ptr); 4184 } else 4185 devres_free(ptr); 4186 4187 return addr; 4188 } 4189 EXPORT_SYMBOL(devm_pci_remap_cfgspace); 4190 4191 /** 4192 * devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource 4193 * @dev: generic device to handle the resource for 4194 * @res: configuration space resource to be handled 4195 * 4196 * Checks that a resource is a valid memory region, requests the memory 4197 * region and ioremaps with pci_remap_cfgspace() API that ensures the 4198 * proper PCI configuration space memory attributes are guaranteed. 4199 * 4200 * All operations are managed and will be undone on driver detach. 4201 * 4202 * Returns a pointer to the remapped memory or an ERR_PTR() encoded error code 4203 * on failure. Usage example:: 4204 * 4205 * res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4206 * base = devm_pci_remap_cfg_resource(&pdev->dev, res); 4207 * if (IS_ERR(base)) 4208 * return PTR_ERR(base); 4209 */ 4210 void __iomem *devm_pci_remap_cfg_resource(struct device *dev, 4211 struct resource *res) 4212 { 4213 resource_size_t size; 4214 const char *name; 4215 void __iomem *dest_ptr; 4216 4217 BUG_ON(!dev); 4218 4219 if (!res || resource_type(res) != IORESOURCE_MEM) { 4220 dev_err(dev, "invalid resource\n"); 4221 return IOMEM_ERR_PTR(-EINVAL); 4222 } 4223 4224 size = resource_size(res); 4225 4226 if (res->name) 4227 name = devm_kasprintf(dev, GFP_KERNEL, "%s %s", dev_name(dev), 4228 res->name); 4229 else 4230 name = devm_kstrdup(dev, dev_name(dev), GFP_KERNEL); 4231 if (!name) 4232 return IOMEM_ERR_PTR(-ENOMEM); 4233 4234 if (!devm_request_mem_region(dev, res->start, size, name)) { 4235 dev_err(dev, "can't request region for resource %pR\n", res); 4236 return IOMEM_ERR_PTR(-EBUSY); 4237 } 4238 4239 dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size); 4240 if (!dest_ptr) { 4241 dev_err(dev, "ioremap failed for resource %pR\n", res); 4242 devm_release_mem_region(dev, res->start, size); 4243 dest_ptr = IOMEM_ERR_PTR(-ENOMEM); 4244 } 4245 4246 return dest_ptr; 4247 } 4248 EXPORT_SYMBOL(devm_pci_remap_cfg_resource); 4249 4250 static void __pci_set_master(struct pci_dev *dev, bool enable) 4251 { 4252 u16 old_cmd, cmd; 4253 4254 pci_read_config_word(dev, PCI_COMMAND, &old_cmd); 4255 if (enable) 4256 cmd = old_cmd | PCI_COMMAND_MASTER; 4257 else 4258 cmd = old_cmd & ~PCI_COMMAND_MASTER; 4259 if (cmd != old_cmd) { 4260 pci_dbg(dev, "%s bus mastering\n", 4261 enable ? "enabling" : "disabling"); 4262 pci_write_config_word(dev, PCI_COMMAND, cmd); 4263 } 4264 dev->is_busmaster = enable; 4265 } 4266 4267 /** 4268 * pcibios_setup - process "pci=" kernel boot arguments 4269 * @str: string used to pass in "pci=" kernel boot arguments 4270 * 4271 * Process kernel boot arguments. This is the default implementation. 4272 * Architecture specific implementations can override this as necessary. 4273 */ 4274 char * __weak __init pcibios_setup(char *str) 4275 { 4276 return str; 4277 } 4278 4279 /** 4280 * pcibios_set_master - enable PCI bus-mastering for device dev 4281 * @dev: the PCI device to enable 4282 * 4283 * Enables PCI bus-mastering for the device. This is the default 4284 * implementation. Architecture specific implementations can override 4285 * this if necessary. 4286 */ 4287 void __weak pcibios_set_master(struct pci_dev *dev) 4288 { 4289 u8 lat; 4290 4291 /* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */ 4292 if (pci_is_pcie(dev)) 4293 return; 4294 4295 pci_read_config_byte(dev, PCI_LATENCY_TIMER, &lat); 4296 if (lat < 16) 4297 lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency; 4298 else if (lat > pcibios_max_latency) 4299 lat = pcibios_max_latency; 4300 else 4301 return; 4302 4303 pci_write_config_byte(dev, PCI_LATENCY_TIMER, lat); 4304 } 4305 4306 /** 4307 * pci_set_master - enables bus-mastering for device dev 4308 * @dev: the PCI device to enable 4309 * 4310 * Enables bus-mastering on the device and calls pcibios_set_master() 4311 * to do the needed arch specific settings. 4312 */ 4313 void pci_set_master(struct pci_dev *dev) 4314 { 4315 __pci_set_master(dev, true); 4316 pcibios_set_master(dev); 4317 } 4318 EXPORT_SYMBOL(pci_set_master); 4319 4320 /** 4321 * pci_clear_master - disables bus-mastering for device dev 4322 * @dev: the PCI device to disable 4323 */ 4324 void pci_clear_master(struct pci_dev *dev) 4325 { 4326 __pci_set_master(dev, false); 4327 } 4328 EXPORT_SYMBOL(pci_clear_master); 4329 4330 /** 4331 * pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed 4332 * @dev: the PCI device for which MWI is to be enabled 4333 * 4334 * Helper function for pci_set_mwi. 4335 * Originally copied from drivers/net/acenic.c. 4336 * Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>. 4337 * 4338 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4339 */ 4340 int pci_set_cacheline_size(struct pci_dev *dev) 4341 { 4342 u8 cacheline_size; 4343 4344 if (!pci_cache_line_size) 4345 return -EINVAL; 4346 4347 /* Validate current setting: the PCI_CACHE_LINE_SIZE must be 4348 equal to or multiple of the right value. */ 4349 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4350 if (cacheline_size >= pci_cache_line_size && 4351 (cacheline_size % pci_cache_line_size) == 0) 4352 return 0; 4353 4354 /* Write the correct value. */ 4355 pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, pci_cache_line_size); 4356 /* Read it back. */ 4357 pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, &cacheline_size); 4358 if (cacheline_size == pci_cache_line_size) 4359 return 0; 4360 4361 pci_dbg(dev, "cache line size of %d is not supported\n", 4362 pci_cache_line_size << 2); 4363 4364 return -EINVAL; 4365 } 4366 EXPORT_SYMBOL_GPL(pci_set_cacheline_size); 4367 4368 /** 4369 * pci_set_mwi - enables memory-write-invalidate PCI transaction 4370 * @dev: the PCI device for which MWI is enabled 4371 * 4372 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4373 * 4374 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4375 */ 4376 int pci_set_mwi(struct pci_dev *dev) 4377 { 4378 #ifdef PCI_DISABLE_MWI 4379 return 0; 4380 #else 4381 int rc; 4382 u16 cmd; 4383 4384 rc = pci_set_cacheline_size(dev); 4385 if (rc) 4386 return rc; 4387 4388 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4389 if (!(cmd & PCI_COMMAND_INVALIDATE)) { 4390 pci_dbg(dev, "enabling Mem-Wr-Inval\n"); 4391 cmd |= PCI_COMMAND_INVALIDATE; 4392 pci_write_config_word(dev, PCI_COMMAND, cmd); 4393 } 4394 return 0; 4395 #endif 4396 } 4397 EXPORT_SYMBOL(pci_set_mwi); 4398 4399 /** 4400 * pcim_set_mwi - a device-managed pci_set_mwi() 4401 * @dev: the PCI device for which MWI is enabled 4402 * 4403 * Managed pci_set_mwi(). 4404 * 4405 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4406 */ 4407 int pcim_set_mwi(struct pci_dev *dev) 4408 { 4409 struct pci_devres *dr; 4410 4411 dr = find_pci_dr(dev); 4412 if (!dr) 4413 return -ENOMEM; 4414 4415 dr->mwi = 1; 4416 return pci_set_mwi(dev); 4417 } 4418 EXPORT_SYMBOL(pcim_set_mwi); 4419 4420 /** 4421 * pci_try_set_mwi - enables memory-write-invalidate PCI transaction 4422 * @dev: the PCI device for which MWI is enabled 4423 * 4424 * Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND. 4425 * Callers are not required to check the return value. 4426 * 4427 * RETURNS: An appropriate -ERRNO error value on error, or zero for success. 4428 */ 4429 int pci_try_set_mwi(struct pci_dev *dev) 4430 { 4431 #ifdef PCI_DISABLE_MWI 4432 return 0; 4433 #else 4434 return pci_set_mwi(dev); 4435 #endif 4436 } 4437 EXPORT_SYMBOL(pci_try_set_mwi); 4438 4439 /** 4440 * pci_clear_mwi - disables Memory-Write-Invalidate for device dev 4441 * @dev: the PCI device to disable 4442 * 4443 * Disables PCI Memory-Write-Invalidate transaction on the device 4444 */ 4445 void pci_clear_mwi(struct pci_dev *dev) 4446 { 4447 #ifndef PCI_DISABLE_MWI 4448 u16 cmd; 4449 4450 pci_read_config_word(dev, PCI_COMMAND, &cmd); 4451 if (cmd & PCI_COMMAND_INVALIDATE) { 4452 cmd &= ~PCI_COMMAND_INVALIDATE; 4453 pci_write_config_word(dev, PCI_COMMAND, cmd); 4454 } 4455 #endif 4456 } 4457 EXPORT_SYMBOL(pci_clear_mwi); 4458 4459 /** 4460 * pci_intx - enables/disables PCI INTx for device dev 4461 * @pdev: the PCI device to operate on 4462 * @enable: boolean: whether to enable or disable PCI INTx 4463 * 4464 * Enables/disables PCI INTx for device @pdev 4465 */ 4466 void pci_intx(struct pci_dev *pdev, int enable) 4467 { 4468 u16 pci_command, new; 4469 4470 pci_read_config_word(pdev, PCI_COMMAND, &pci_command); 4471 4472 if (enable) 4473 new = pci_command & ~PCI_COMMAND_INTX_DISABLE; 4474 else 4475 new = pci_command | PCI_COMMAND_INTX_DISABLE; 4476 4477 if (new != pci_command) { 4478 struct pci_devres *dr; 4479 4480 pci_write_config_word(pdev, PCI_COMMAND, new); 4481 4482 dr = find_pci_dr(pdev); 4483 if (dr && !dr->restore_intx) { 4484 dr->restore_intx = 1; 4485 dr->orig_intx = !enable; 4486 } 4487 } 4488 } 4489 EXPORT_SYMBOL_GPL(pci_intx); 4490 4491 static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask) 4492 { 4493 struct pci_bus *bus = dev->bus; 4494 bool mask_updated = true; 4495 u32 cmd_status_dword; 4496 u16 origcmd, newcmd; 4497 unsigned long flags; 4498 bool irq_pending; 4499 4500 /* 4501 * We do a single dword read to retrieve both command and status. 4502 * Document assumptions that make this possible. 4503 */ 4504 BUILD_BUG_ON(PCI_COMMAND % 4); 4505 BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS); 4506 4507 raw_spin_lock_irqsave(&pci_lock, flags); 4508 4509 bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword); 4510 4511 irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT; 4512 4513 /* 4514 * Check interrupt status register to see whether our device 4515 * triggered the interrupt (when masking) or the next IRQ is 4516 * already pending (when unmasking). 4517 */ 4518 if (mask != irq_pending) { 4519 mask_updated = false; 4520 goto done; 4521 } 4522 4523 origcmd = cmd_status_dword; 4524 newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE; 4525 if (mask) 4526 newcmd |= PCI_COMMAND_INTX_DISABLE; 4527 if (newcmd != origcmd) 4528 bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd); 4529 4530 done: 4531 raw_spin_unlock_irqrestore(&pci_lock, flags); 4532 4533 return mask_updated; 4534 } 4535 4536 /** 4537 * pci_check_and_mask_intx - mask INTx on pending interrupt 4538 * @dev: the PCI device to operate on 4539 * 4540 * Check if the device dev has its INTx line asserted, mask it and return 4541 * true in that case. False is returned if no interrupt was pending. 4542 */ 4543 bool pci_check_and_mask_intx(struct pci_dev *dev) 4544 { 4545 return pci_check_and_set_intx_mask(dev, true); 4546 } 4547 EXPORT_SYMBOL_GPL(pci_check_and_mask_intx); 4548 4549 /** 4550 * pci_check_and_unmask_intx - unmask INTx if no interrupt is pending 4551 * @dev: the PCI device to operate on 4552 * 4553 * Check if the device dev has its INTx line asserted, unmask it if not and 4554 * return true. False is returned and the mask remains active if there was 4555 * still an interrupt pending. 4556 */ 4557 bool pci_check_and_unmask_intx(struct pci_dev *dev) 4558 { 4559 return pci_check_and_set_intx_mask(dev, false); 4560 } 4561 EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx); 4562 4563 /** 4564 * pci_wait_for_pending_transaction - wait for pending transaction 4565 * @dev: the PCI device to operate on 4566 * 4567 * Return 0 if transaction is pending 1 otherwise. 4568 */ 4569 int pci_wait_for_pending_transaction(struct pci_dev *dev) 4570 { 4571 if (!pci_is_pcie(dev)) 4572 return 1; 4573 4574 return pci_wait_for_pending(dev, pci_pcie_cap(dev) + PCI_EXP_DEVSTA, 4575 PCI_EXP_DEVSTA_TRPND); 4576 } 4577 EXPORT_SYMBOL(pci_wait_for_pending_transaction); 4578 4579 /** 4580 * pcie_has_flr - check if a device supports function level resets 4581 * @dev: device to check 4582 * 4583 * Returns true if the device advertises support for PCIe function level 4584 * resets. 4585 */ 4586 bool pcie_has_flr(struct pci_dev *dev) 4587 { 4588 u32 cap; 4589 4590 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4591 return false; 4592 4593 pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, &cap); 4594 return cap & PCI_EXP_DEVCAP_FLR; 4595 } 4596 EXPORT_SYMBOL_GPL(pcie_has_flr); 4597 4598 /** 4599 * pcie_flr - initiate a PCIe function level reset 4600 * @dev: device to reset 4601 * 4602 * Initiate a function level reset on @dev. The caller should ensure the 4603 * device supports FLR before calling this function, e.g. by using the 4604 * pcie_has_flr() helper. 4605 */ 4606 int pcie_flr(struct pci_dev *dev) 4607 { 4608 if (!pci_wait_for_pending_transaction(dev)) 4609 pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n"); 4610 4611 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); 4612 4613 if (dev->imm_ready) 4614 return 0; 4615 4616 /* 4617 * Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within 4618 * 100ms, but may silently discard requests while the FLR is in 4619 * progress. Wait 100ms before trying to access the device. 4620 */ 4621 msleep(100); 4622 4623 return pci_dev_wait(dev, "FLR", PCIE_RESET_READY_POLL_MS); 4624 } 4625 EXPORT_SYMBOL_GPL(pcie_flr); 4626 4627 static int pci_af_flr(struct pci_dev *dev, int probe) 4628 { 4629 int pos; 4630 u8 cap; 4631 4632 pos = pci_find_capability(dev, PCI_CAP_ID_AF); 4633 if (!pos) 4634 return -ENOTTY; 4635 4636 if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) 4637 return -ENOTTY; 4638 4639 pci_read_config_byte(dev, pos + PCI_AF_CAP, &cap); 4640 if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR)) 4641 return -ENOTTY; 4642 4643 if (probe) 4644 return 0; 4645 4646 /* 4647 * Wait for Transaction Pending bit to clear. A word-aligned test 4648 * is used, so we use the control offset rather than status and shift 4649 * the test bit to match. 4650 */ 4651 if (!pci_wait_for_pending(dev, pos + PCI_AF_CTRL, 4652 PCI_AF_STATUS_TP << 8)) 4653 pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n"); 4654 4655 pci_write_config_byte(dev, pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR); 4656 4657 if (dev->imm_ready) 4658 return 0; 4659 4660 /* 4661 * Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006, 4662 * updated 27 July 2006; a device must complete an FLR within 4663 * 100ms, but may silently discard requests while the FLR is in 4664 * progress. Wait 100ms before trying to access the device. 4665 */ 4666 msleep(100); 4667 4668 return pci_dev_wait(dev, "AF_FLR", PCIE_RESET_READY_POLL_MS); 4669 } 4670 4671 /** 4672 * pci_pm_reset - Put device into PCI_D3 and back into PCI_D0. 4673 * @dev: Device to reset. 4674 * @probe: If set, only check if the device can be reset this way. 4675 * 4676 * If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is 4677 * unset, it will be reinitialized internally when going from PCI_D3hot to 4678 * PCI_D0. If that's the case and the device is not in a low-power state 4679 * already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset. 4680 * 4681 * NOTE: This causes the caller to sleep for twice the device power transition 4682 * cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms 4683 * by default (i.e. unless the @dev's d3hot_delay field has a different value). 4684 * Moreover, only devices in D0 can be reset by this function. 4685 */ 4686 static int pci_pm_reset(struct pci_dev *dev, int probe) 4687 { 4688 u16 csr; 4689 4690 if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET) 4691 return -ENOTTY; 4692 4693 pci_read_config_word(dev, dev->pm_cap + PCI_PM_CTRL, &csr); 4694 if (csr & PCI_PM_CTRL_NO_SOFT_RESET) 4695 return -ENOTTY; 4696 4697 if (probe) 4698 return 0; 4699 4700 if (dev->current_state != PCI_D0) 4701 return -EINVAL; 4702 4703 csr &= ~PCI_PM_CTRL_STATE_MASK; 4704 csr |= PCI_D3hot; 4705 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4706 pci_dev_d3_sleep(dev); 4707 4708 csr &= ~PCI_PM_CTRL_STATE_MASK; 4709 csr |= PCI_D0; 4710 pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, csr); 4711 pci_dev_d3_sleep(dev); 4712 4713 return pci_dev_wait(dev, "PM D3hot->D0", PCIE_RESET_READY_POLL_MS); 4714 } 4715 4716 /** 4717 * pcie_wait_for_link_delay - Wait until link is active or inactive 4718 * @pdev: Bridge device 4719 * @active: waiting for active or inactive? 4720 * @delay: Delay to wait after link has become active (in ms) 4721 * 4722 * Use this to wait till link becomes active or inactive. 4723 */ 4724 static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active, 4725 int delay) 4726 { 4727 int timeout = 1000; 4728 bool ret; 4729 u16 lnk_status; 4730 4731 /* 4732 * Some controllers might not implement link active reporting. In this 4733 * case, we wait for 1000 ms + any delay requested by the caller. 4734 */ 4735 if (!pdev->link_active_reporting) { 4736 msleep(timeout + delay); 4737 return true; 4738 } 4739 4740 /* 4741 * PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms, 4742 * after which we should expect an link active if the reset was 4743 * successful. If so, software must wait a minimum 100ms before sending 4744 * configuration requests to devices downstream this port. 4745 * 4746 * If the link fails to activate, either the device was physically 4747 * removed or the link is permanently failed. 4748 */ 4749 if (active) 4750 msleep(20); 4751 for (;;) { 4752 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status); 4753 ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA); 4754 if (ret == active) 4755 break; 4756 if (timeout <= 0) 4757 break; 4758 msleep(10); 4759 timeout -= 10; 4760 } 4761 if (active && ret) 4762 msleep(delay); 4763 4764 return ret == active; 4765 } 4766 4767 /** 4768 * pcie_wait_for_link - Wait until link is active or inactive 4769 * @pdev: Bridge device 4770 * @active: waiting for active or inactive? 4771 * 4772 * Use this to wait till link becomes active or inactive. 4773 */ 4774 bool pcie_wait_for_link(struct pci_dev *pdev, bool active) 4775 { 4776 return pcie_wait_for_link_delay(pdev, active, 100); 4777 } 4778 4779 /* 4780 * Find maximum D3cold delay required by all the devices on the bus. The 4781 * spec says 100 ms, but firmware can lower it and we allow drivers to 4782 * increase it as well. 4783 * 4784 * Called with @pci_bus_sem locked for reading. 4785 */ 4786 static int pci_bus_max_d3cold_delay(const struct pci_bus *bus) 4787 { 4788 const struct pci_dev *pdev; 4789 int min_delay = 100; 4790 int max_delay = 0; 4791 4792 list_for_each_entry(pdev, &bus->devices, bus_list) { 4793 if (pdev->d3cold_delay < min_delay) 4794 min_delay = pdev->d3cold_delay; 4795 if (pdev->d3cold_delay > max_delay) 4796 max_delay = pdev->d3cold_delay; 4797 } 4798 4799 return max(min_delay, max_delay); 4800 } 4801 4802 /** 4803 * pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible 4804 * @dev: PCI bridge 4805 * 4806 * Handle necessary delays before access to the devices on the secondary 4807 * side of the bridge are permitted after D3cold to D0 transition. 4808 * 4809 * For PCIe this means the delays in PCIe 5.0 section 6.6.1. For 4810 * conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section 4811 * 4.3.2. 4812 */ 4813 void pci_bridge_wait_for_secondary_bus(struct pci_dev *dev) 4814 { 4815 struct pci_dev *child; 4816 int delay; 4817 4818 if (pci_dev_is_disconnected(dev)) 4819 return; 4820 4821 if (!pci_is_bridge(dev) || !dev->bridge_d3) 4822 return; 4823 4824 down_read(&pci_bus_sem); 4825 4826 /* 4827 * We only deal with devices that are present currently on the bus. 4828 * For any hot-added devices the access delay is handled in pciehp 4829 * board_added(). In case of ACPI hotplug the firmware is expected 4830 * to configure the devices before OS is notified. 4831 */ 4832 if (!dev->subordinate || list_empty(&dev->subordinate->devices)) { 4833 up_read(&pci_bus_sem); 4834 return; 4835 } 4836 4837 /* Take d3cold_delay requirements into account */ 4838 delay = pci_bus_max_d3cold_delay(dev->subordinate); 4839 if (!delay) { 4840 up_read(&pci_bus_sem); 4841 return; 4842 } 4843 4844 child = list_first_entry(&dev->subordinate->devices, struct pci_dev, 4845 bus_list); 4846 up_read(&pci_bus_sem); 4847 4848 /* 4849 * Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before 4850 * accessing the device after reset (that is 1000 ms + 100 ms). In 4851 * practice this should not be needed because we don't do power 4852 * management for them (see pci_bridge_d3_possible()). 4853 */ 4854 if (!pci_is_pcie(dev)) { 4855 pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay); 4856 msleep(1000 + delay); 4857 return; 4858 } 4859 4860 /* 4861 * For PCIe downstream and root ports that do not support speeds 4862 * greater than 5 GT/s need to wait minimum 100 ms. For higher 4863 * speeds (gen3) we need to wait first for the data link layer to 4864 * become active. 4865 * 4866 * However, 100 ms is the minimum and the PCIe spec says the 4867 * software must allow at least 1s before it can determine that the 4868 * device that did not respond is a broken device. There is 4869 * evidence that 100 ms is not always enough, for example certain 4870 * Titan Ridge xHCI controller does not always respond to 4871 * configuration requests if we only wait for 100 ms (see 4872 * https://bugzilla.kernel.org/show_bug.cgi?id=203885). 4873 * 4874 * Therefore we wait for 100 ms and check for the device presence. 4875 * If it is still not present give it an additional 100 ms. 4876 */ 4877 if (!pcie_downstream_port(dev)) 4878 return; 4879 4880 if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) { 4881 pci_dbg(dev, "waiting %d ms for downstream link\n", delay); 4882 msleep(delay); 4883 } else { 4884 pci_dbg(dev, "waiting %d ms for downstream link, after activation\n", 4885 delay); 4886 if (!pcie_wait_for_link_delay(dev, true, delay)) { 4887 /* Did not train, no need to wait any further */ 4888 pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n"); 4889 return; 4890 } 4891 } 4892 4893 if (!pci_device_is_present(child)) { 4894 pci_dbg(child, "waiting additional %d ms to become accessible\n", delay); 4895 msleep(delay); 4896 } 4897 } 4898 4899 void pci_reset_secondary_bus(struct pci_dev *dev) 4900 { 4901 u16 ctrl; 4902 4903 pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &ctrl); 4904 ctrl |= PCI_BRIDGE_CTL_BUS_RESET; 4905 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 4906 4907 /* 4908 * PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double 4909 * this to 2ms to ensure that we meet the minimum requirement. 4910 */ 4911 msleep(2); 4912 4913 ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; 4914 pci_write_config_word(dev, PCI_BRIDGE_CONTROL, ctrl); 4915 4916 /* 4917 * Trhfa for conventional PCI is 2^25 clock cycles. 4918 * Assuming a minimum 33MHz clock this results in a 1s 4919 * delay before we can consider subordinate devices to 4920 * be re-initialized. PCIe has some ways to shorten this, 4921 * but we don't make use of them yet. 4922 */ 4923 ssleep(1); 4924 } 4925 4926 void __weak pcibios_reset_secondary_bus(struct pci_dev *dev) 4927 { 4928 pci_reset_secondary_bus(dev); 4929 } 4930 4931 /** 4932 * pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge. 4933 * @dev: Bridge device 4934 * 4935 * Use the bridge control register to assert reset on the secondary bus. 4936 * Devices on the secondary bus are left in power-on state. 4937 */ 4938 int pci_bridge_secondary_bus_reset(struct pci_dev *dev) 4939 { 4940 pcibios_reset_secondary_bus(dev); 4941 4942 return pci_dev_wait(dev, "bus reset", PCIE_RESET_READY_POLL_MS); 4943 } 4944 EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset); 4945 4946 static int pci_parent_bus_reset(struct pci_dev *dev, int probe) 4947 { 4948 struct pci_dev *pdev; 4949 4950 if (pci_is_root_bus(dev->bus) || dev->subordinate || 4951 !dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 4952 return -ENOTTY; 4953 4954 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 4955 if (pdev != dev) 4956 return -ENOTTY; 4957 4958 if (probe) 4959 return 0; 4960 4961 return pci_bridge_secondary_bus_reset(dev->bus->self); 4962 } 4963 4964 static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, int probe) 4965 { 4966 int rc = -ENOTTY; 4967 4968 if (!hotplug || !try_module_get(hotplug->owner)) 4969 return rc; 4970 4971 if (hotplug->ops->reset_slot) 4972 rc = hotplug->ops->reset_slot(hotplug, probe); 4973 4974 module_put(hotplug->owner); 4975 4976 return rc; 4977 } 4978 4979 static int pci_dev_reset_slot_function(struct pci_dev *dev, int probe) 4980 { 4981 if (dev->multifunction || dev->subordinate || !dev->slot || 4982 dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET) 4983 return -ENOTTY; 4984 4985 return pci_reset_hotplug_slot(dev->slot->hotplug, probe); 4986 } 4987 4988 static void pci_dev_lock(struct pci_dev *dev) 4989 { 4990 pci_cfg_access_lock(dev); 4991 /* block PM suspend, driver probe, etc. */ 4992 device_lock(&dev->dev); 4993 } 4994 4995 /* Return 1 on successful lock, 0 on contention */ 4996 static int pci_dev_trylock(struct pci_dev *dev) 4997 { 4998 if (pci_cfg_access_trylock(dev)) { 4999 if (device_trylock(&dev->dev)) 5000 return 1; 5001 pci_cfg_access_unlock(dev); 5002 } 5003 5004 return 0; 5005 } 5006 5007 static void pci_dev_unlock(struct pci_dev *dev) 5008 { 5009 device_unlock(&dev->dev); 5010 pci_cfg_access_unlock(dev); 5011 } 5012 5013 static void pci_dev_save_and_disable(struct pci_dev *dev) 5014 { 5015 const struct pci_error_handlers *err_handler = 5016 dev->driver ? dev->driver->err_handler : NULL; 5017 5018 /* 5019 * dev->driver->err_handler->reset_prepare() is protected against 5020 * races with ->remove() by the device lock, which must be held by 5021 * the caller. 5022 */ 5023 if (err_handler && err_handler->reset_prepare) 5024 err_handler->reset_prepare(dev); 5025 5026 /* 5027 * Wake-up device prior to save. PM registers default to D0 after 5028 * reset and a simple register restore doesn't reliably return 5029 * to a non-D0 state anyway. 5030 */ 5031 pci_set_power_state(dev, PCI_D0); 5032 5033 pci_save_state(dev); 5034 /* 5035 * Disable the device by clearing the Command register, except for 5036 * INTx-disable which is set. This not only disables MMIO and I/O port 5037 * BARs, but also prevents the device from being Bus Master, preventing 5038 * DMA from the device including MSI/MSI-X interrupts. For PCI 2.3 5039 * compliant devices, INTx-disable prevents legacy interrupts. 5040 */ 5041 pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 5042 } 5043 5044 static void pci_dev_restore(struct pci_dev *dev) 5045 { 5046 const struct pci_error_handlers *err_handler = 5047 dev->driver ? dev->driver->err_handler : NULL; 5048 5049 pci_restore_state(dev); 5050 5051 /* 5052 * dev->driver->err_handler->reset_done() is protected against 5053 * races with ->remove() by the device lock, which must be held by 5054 * the caller. 5055 */ 5056 if (err_handler && err_handler->reset_done) 5057 err_handler->reset_done(dev); 5058 } 5059 5060 /** 5061 * __pci_reset_function_locked - reset a PCI device function while holding 5062 * the @dev mutex lock. 5063 * @dev: PCI device to reset 5064 * 5065 * Some devices allow an individual function to be reset without affecting 5066 * other functions in the same device. The PCI device must be responsive 5067 * to PCI config space in order to use this function. 5068 * 5069 * The device function is presumed to be unused and the caller is holding 5070 * the device mutex lock when this function is called. 5071 * 5072 * Resetting the device will make the contents of PCI configuration space 5073 * random, so any caller of this must be prepared to reinitialise the 5074 * device including MSI, bus mastering, BARs, decoding IO and memory spaces, 5075 * etc. 5076 * 5077 * Returns 0 if the device function was successfully reset or negative if the 5078 * device doesn't support resetting a single function. 5079 */ 5080 int __pci_reset_function_locked(struct pci_dev *dev) 5081 { 5082 int rc; 5083 5084 might_sleep(); 5085 5086 /* 5087 * A reset method returns -ENOTTY if it doesn't support this device 5088 * and we should try the next method. 5089 * 5090 * If it returns 0 (success), we're finished. If it returns any 5091 * other error, we're also finished: this indicates that further 5092 * reset mechanisms might be broken on the device. 5093 */ 5094 rc = pci_dev_specific_reset(dev, 0); 5095 if (rc != -ENOTTY) 5096 return rc; 5097 if (pcie_has_flr(dev)) { 5098 rc = pcie_flr(dev); 5099 if (rc != -ENOTTY) 5100 return rc; 5101 } 5102 rc = pci_af_flr(dev, 0); 5103 if (rc != -ENOTTY) 5104 return rc; 5105 rc = pci_pm_reset(dev, 0); 5106 if (rc != -ENOTTY) 5107 return rc; 5108 rc = pci_dev_reset_slot_function(dev, 0); 5109 if (rc != -ENOTTY) 5110 return rc; 5111 return pci_parent_bus_reset(dev, 0); 5112 } 5113 EXPORT_SYMBOL_GPL(__pci_reset_function_locked); 5114 5115 /** 5116 * pci_probe_reset_function - check whether the device can be safely reset 5117 * @dev: PCI device to reset 5118 * 5119 * Some devices allow an individual function to be reset without affecting 5120 * other functions in the same device. The PCI device must be responsive 5121 * to PCI config space in order to use this function. 5122 * 5123 * Returns 0 if the device function can be reset or negative if the 5124 * device doesn't support resetting a single function. 5125 */ 5126 int pci_probe_reset_function(struct pci_dev *dev) 5127 { 5128 int rc; 5129 5130 might_sleep(); 5131 5132 rc = pci_dev_specific_reset(dev, 1); 5133 if (rc != -ENOTTY) 5134 return rc; 5135 if (pcie_has_flr(dev)) 5136 return 0; 5137 rc = pci_af_flr(dev, 1); 5138 if (rc != -ENOTTY) 5139 return rc; 5140 rc = pci_pm_reset(dev, 1); 5141 if (rc != -ENOTTY) 5142 return rc; 5143 rc = pci_dev_reset_slot_function(dev, 1); 5144 if (rc != -ENOTTY) 5145 return rc; 5146 5147 return pci_parent_bus_reset(dev, 1); 5148 } 5149 5150 /** 5151 * pci_reset_function - quiesce and reset a PCI device function 5152 * @dev: PCI device to reset 5153 * 5154 * Some devices allow an individual function to be reset without affecting 5155 * other functions in the same device. The PCI device must be responsive 5156 * to PCI config space in order to use this function. 5157 * 5158 * This function does not just reset the PCI portion of a device, but 5159 * clears all the state associated with the device. This function differs 5160 * from __pci_reset_function_locked() in that it saves and restores device state 5161 * over the reset and takes the PCI device lock. 5162 * 5163 * Returns 0 if the device function was successfully reset or negative if the 5164 * device doesn't support resetting a single function. 5165 */ 5166 int pci_reset_function(struct pci_dev *dev) 5167 { 5168 int rc; 5169 5170 if (!dev->reset_fn) 5171 return -ENOTTY; 5172 5173 pci_dev_lock(dev); 5174 pci_dev_save_and_disable(dev); 5175 5176 rc = __pci_reset_function_locked(dev); 5177 5178 pci_dev_restore(dev); 5179 pci_dev_unlock(dev); 5180 5181 return rc; 5182 } 5183 EXPORT_SYMBOL_GPL(pci_reset_function); 5184 5185 /** 5186 * pci_reset_function_locked - quiesce and reset a PCI device function 5187 * @dev: PCI device to reset 5188 * 5189 * Some devices allow an individual function to be reset without affecting 5190 * other functions in the same device. The PCI device must be responsive 5191 * to PCI config space in order to use this function. 5192 * 5193 * This function does not just reset the PCI portion of a device, but 5194 * clears all the state associated with the device. This function differs 5195 * from __pci_reset_function_locked() in that it saves and restores device state 5196 * over the reset. It also differs from pci_reset_function() in that it 5197 * requires the PCI device lock to be held. 5198 * 5199 * Returns 0 if the device function was successfully reset or negative if the 5200 * device doesn't support resetting a single function. 5201 */ 5202 int pci_reset_function_locked(struct pci_dev *dev) 5203 { 5204 int rc; 5205 5206 if (!dev->reset_fn) 5207 return -ENOTTY; 5208 5209 pci_dev_save_and_disable(dev); 5210 5211 rc = __pci_reset_function_locked(dev); 5212 5213 pci_dev_restore(dev); 5214 5215 return rc; 5216 } 5217 EXPORT_SYMBOL_GPL(pci_reset_function_locked); 5218 5219 /** 5220 * pci_try_reset_function - quiesce and reset a PCI device function 5221 * @dev: PCI device to reset 5222 * 5223 * Same as above, except return -EAGAIN if unable to lock device. 5224 */ 5225 int pci_try_reset_function(struct pci_dev *dev) 5226 { 5227 int rc; 5228 5229 if (!dev->reset_fn) 5230 return -ENOTTY; 5231 5232 if (!pci_dev_trylock(dev)) 5233 return -EAGAIN; 5234 5235 pci_dev_save_and_disable(dev); 5236 rc = __pci_reset_function_locked(dev); 5237 pci_dev_restore(dev); 5238 pci_dev_unlock(dev); 5239 5240 return rc; 5241 } 5242 EXPORT_SYMBOL_GPL(pci_try_reset_function); 5243 5244 /* Do any devices on or below this bus prevent a bus reset? */ 5245 static bool pci_bus_resetable(struct pci_bus *bus) 5246 { 5247 struct pci_dev *dev; 5248 5249 5250 if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5251 return false; 5252 5253 list_for_each_entry(dev, &bus->devices, bus_list) { 5254 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5255 (dev->subordinate && !pci_bus_resetable(dev->subordinate))) 5256 return false; 5257 } 5258 5259 return true; 5260 } 5261 5262 /* Lock devices from the top of the tree down */ 5263 static void pci_bus_lock(struct pci_bus *bus) 5264 { 5265 struct pci_dev *dev; 5266 5267 list_for_each_entry(dev, &bus->devices, bus_list) { 5268 pci_dev_lock(dev); 5269 if (dev->subordinate) 5270 pci_bus_lock(dev->subordinate); 5271 } 5272 } 5273 5274 /* Unlock devices from the bottom of the tree up */ 5275 static void pci_bus_unlock(struct pci_bus *bus) 5276 { 5277 struct pci_dev *dev; 5278 5279 list_for_each_entry(dev, &bus->devices, bus_list) { 5280 if (dev->subordinate) 5281 pci_bus_unlock(dev->subordinate); 5282 pci_dev_unlock(dev); 5283 } 5284 } 5285 5286 /* Return 1 on successful lock, 0 on contention */ 5287 static int pci_bus_trylock(struct pci_bus *bus) 5288 { 5289 struct pci_dev *dev; 5290 5291 list_for_each_entry(dev, &bus->devices, bus_list) { 5292 if (!pci_dev_trylock(dev)) 5293 goto unlock; 5294 if (dev->subordinate) { 5295 if (!pci_bus_trylock(dev->subordinate)) { 5296 pci_dev_unlock(dev); 5297 goto unlock; 5298 } 5299 } 5300 } 5301 return 1; 5302 5303 unlock: 5304 list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) { 5305 if (dev->subordinate) 5306 pci_bus_unlock(dev->subordinate); 5307 pci_dev_unlock(dev); 5308 } 5309 return 0; 5310 } 5311 5312 /* Do any devices on or below this slot prevent a bus reset? */ 5313 static bool pci_slot_resetable(struct pci_slot *slot) 5314 { 5315 struct pci_dev *dev; 5316 5317 if (slot->bus->self && 5318 (slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)) 5319 return false; 5320 5321 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5322 if (!dev->slot || dev->slot != slot) 5323 continue; 5324 if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET || 5325 (dev->subordinate && !pci_bus_resetable(dev->subordinate))) 5326 return false; 5327 } 5328 5329 return true; 5330 } 5331 5332 /* Lock devices from the top of the tree down */ 5333 static void pci_slot_lock(struct pci_slot *slot) 5334 { 5335 struct pci_dev *dev; 5336 5337 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5338 if (!dev->slot || dev->slot != slot) 5339 continue; 5340 pci_dev_lock(dev); 5341 if (dev->subordinate) 5342 pci_bus_lock(dev->subordinate); 5343 } 5344 } 5345 5346 /* Unlock devices from the bottom of the tree up */ 5347 static void pci_slot_unlock(struct pci_slot *slot) 5348 { 5349 struct pci_dev *dev; 5350 5351 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5352 if (!dev->slot || dev->slot != slot) 5353 continue; 5354 if (dev->subordinate) 5355 pci_bus_unlock(dev->subordinate); 5356 pci_dev_unlock(dev); 5357 } 5358 } 5359 5360 /* Return 1 on successful lock, 0 on contention */ 5361 static int pci_slot_trylock(struct pci_slot *slot) 5362 { 5363 struct pci_dev *dev; 5364 5365 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5366 if (!dev->slot || dev->slot != slot) 5367 continue; 5368 if (!pci_dev_trylock(dev)) 5369 goto unlock; 5370 if (dev->subordinate) { 5371 if (!pci_bus_trylock(dev->subordinate)) { 5372 pci_dev_unlock(dev); 5373 goto unlock; 5374 } 5375 } 5376 } 5377 return 1; 5378 5379 unlock: 5380 list_for_each_entry_continue_reverse(dev, 5381 &slot->bus->devices, bus_list) { 5382 if (!dev->slot || dev->slot != slot) 5383 continue; 5384 if (dev->subordinate) 5385 pci_bus_unlock(dev->subordinate); 5386 pci_dev_unlock(dev); 5387 } 5388 return 0; 5389 } 5390 5391 /* 5392 * Save and disable devices from the top of the tree down while holding 5393 * the @dev mutex lock for the entire tree. 5394 */ 5395 static void pci_bus_save_and_disable_locked(struct pci_bus *bus) 5396 { 5397 struct pci_dev *dev; 5398 5399 list_for_each_entry(dev, &bus->devices, bus_list) { 5400 pci_dev_save_and_disable(dev); 5401 if (dev->subordinate) 5402 pci_bus_save_and_disable_locked(dev->subordinate); 5403 } 5404 } 5405 5406 /* 5407 * Restore devices from top of the tree down while holding @dev mutex lock 5408 * for the entire tree. Parent bridges need to be restored before we can 5409 * get to subordinate devices. 5410 */ 5411 static void pci_bus_restore_locked(struct pci_bus *bus) 5412 { 5413 struct pci_dev *dev; 5414 5415 list_for_each_entry(dev, &bus->devices, bus_list) { 5416 pci_dev_restore(dev); 5417 if (dev->subordinate) 5418 pci_bus_restore_locked(dev->subordinate); 5419 } 5420 } 5421 5422 /* 5423 * Save and disable devices from the top of the tree down while holding 5424 * the @dev mutex lock for the entire tree. 5425 */ 5426 static void pci_slot_save_and_disable_locked(struct pci_slot *slot) 5427 { 5428 struct pci_dev *dev; 5429 5430 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5431 if (!dev->slot || dev->slot != slot) 5432 continue; 5433 pci_dev_save_and_disable(dev); 5434 if (dev->subordinate) 5435 pci_bus_save_and_disable_locked(dev->subordinate); 5436 } 5437 } 5438 5439 /* 5440 * Restore devices from top of the tree down while holding @dev mutex lock 5441 * for the entire tree. Parent bridges need to be restored before we can 5442 * get to subordinate devices. 5443 */ 5444 static void pci_slot_restore_locked(struct pci_slot *slot) 5445 { 5446 struct pci_dev *dev; 5447 5448 list_for_each_entry(dev, &slot->bus->devices, bus_list) { 5449 if (!dev->slot || dev->slot != slot) 5450 continue; 5451 pci_dev_restore(dev); 5452 if (dev->subordinate) 5453 pci_bus_restore_locked(dev->subordinate); 5454 } 5455 } 5456 5457 static int pci_slot_reset(struct pci_slot *slot, int probe) 5458 { 5459 int rc; 5460 5461 if (!slot || !pci_slot_resetable(slot)) 5462 return -ENOTTY; 5463 5464 if (!probe) 5465 pci_slot_lock(slot); 5466 5467 might_sleep(); 5468 5469 rc = pci_reset_hotplug_slot(slot->hotplug, probe); 5470 5471 if (!probe) 5472 pci_slot_unlock(slot); 5473 5474 return rc; 5475 } 5476 5477 /** 5478 * pci_probe_reset_slot - probe whether a PCI slot can be reset 5479 * @slot: PCI slot to probe 5480 * 5481 * Return 0 if slot can be reset, negative if a slot reset is not supported. 5482 */ 5483 int pci_probe_reset_slot(struct pci_slot *slot) 5484 { 5485 return pci_slot_reset(slot, 1); 5486 } 5487 EXPORT_SYMBOL_GPL(pci_probe_reset_slot); 5488 5489 /** 5490 * __pci_reset_slot - Try to reset a PCI slot 5491 * @slot: PCI slot to reset 5492 * 5493 * A PCI bus may host multiple slots, each slot may support a reset mechanism 5494 * independent of other slots. For instance, some slots may support slot power 5495 * control. In the case of a 1:1 bus to slot architecture, this function may 5496 * wrap the bus reset to avoid spurious slot related events such as hotplug. 5497 * Generally a slot reset should be attempted before a bus reset. All of the 5498 * function of the slot and any subordinate buses behind the slot are reset 5499 * through this function. PCI config space of all devices in the slot and 5500 * behind the slot is saved before and restored after reset. 5501 * 5502 * Same as above except return -EAGAIN if the slot cannot be locked 5503 */ 5504 static int __pci_reset_slot(struct pci_slot *slot) 5505 { 5506 int rc; 5507 5508 rc = pci_slot_reset(slot, 1); 5509 if (rc) 5510 return rc; 5511 5512 if (pci_slot_trylock(slot)) { 5513 pci_slot_save_and_disable_locked(slot); 5514 might_sleep(); 5515 rc = pci_reset_hotplug_slot(slot->hotplug, 0); 5516 pci_slot_restore_locked(slot); 5517 pci_slot_unlock(slot); 5518 } else 5519 rc = -EAGAIN; 5520 5521 return rc; 5522 } 5523 5524 static int pci_bus_reset(struct pci_bus *bus, int probe) 5525 { 5526 int ret; 5527 5528 if (!bus->self || !pci_bus_resetable(bus)) 5529 return -ENOTTY; 5530 5531 if (probe) 5532 return 0; 5533 5534 pci_bus_lock(bus); 5535 5536 might_sleep(); 5537 5538 ret = pci_bridge_secondary_bus_reset(bus->self); 5539 5540 pci_bus_unlock(bus); 5541 5542 return ret; 5543 } 5544 5545 /** 5546 * pci_bus_error_reset - reset the bridge's subordinate bus 5547 * @bridge: The parent device that connects to the bus to reset 5548 * 5549 * This function will first try to reset the slots on this bus if the method is 5550 * available. If slot reset fails or is not available, this will fall back to a 5551 * secondary bus reset. 5552 */ 5553 int pci_bus_error_reset(struct pci_dev *bridge) 5554 { 5555 struct pci_bus *bus = bridge->subordinate; 5556 struct pci_slot *slot; 5557 5558 if (!bus) 5559 return -ENOTTY; 5560 5561 mutex_lock(&pci_slot_mutex); 5562 if (list_empty(&bus->slots)) 5563 goto bus_reset; 5564 5565 list_for_each_entry(slot, &bus->slots, list) 5566 if (pci_probe_reset_slot(slot)) 5567 goto bus_reset; 5568 5569 list_for_each_entry(slot, &bus->slots, list) 5570 if (pci_slot_reset(slot, 0)) 5571 goto bus_reset; 5572 5573 mutex_unlock(&pci_slot_mutex); 5574 return 0; 5575 bus_reset: 5576 mutex_unlock(&pci_slot_mutex); 5577 return pci_bus_reset(bridge->subordinate, 0); 5578 } 5579 5580 /** 5581 * pci_probe_reset_bus - probe whether a PCI bus can be reset 5582 * @bus: PCI bus to probe 5583 * 5584 * Return 0 if bus can be reset, negative if a bus reset is not supported. 5585 */ 5586 int pci_probe_reset_bus(struct pci_bus *bus) 5587 { 5588 return pci_bus_reset(bus, 1); 5589 } 5590 EXPORT_SYMBOL_GPL(pci_probe_reset_bus); 5591 5592 /** 5593 * __pci_reset_bus - Try to reset a PCI bus 5594 * @bus: top level PCI bus to reset 5595 * 5596 * Same as above except return -EAGAIN if the bus cannot be locked 5597 */ 5598 static int __pci_reset_bus(struct pci_bus *bus) 5599 { 5600 int rc; 5601 5602 rc = pci_bus_reset(bus, 1); 5603 if (rc) 5604 return rc; 5605 5606 if (pci_bus_trylock(bus)) { 5607 pci_bus_save_and_disable_locked(bus); 5608 might_sleep(); 5609 rc = pci_bridge_secondary_bus_reset(bus->self); 5610 pci_bus_restore_locked(bus); 5611 pci_bus_unlock(bus); 5612 } else 5613 rc = -EAGAIN; 5614 5615 return rc; 5616 } 5617 5618 /** 5619 * pci_reset_bus - Try to reset a PCI bus 5620 * @pdev: top level PCI device to reset via slot/bus 5621 * 5622 * Same as above except return -EAGAIN if the bus cannot be locked 5623 */ 5624 int pci_reset_bus(struct pci_dev *pdev) 5625 { 5626 return (!pci_probe_reset_slot(pdev->slot)) ? 5627 __pci_reset_slot(pdev->slot) : __pci_reset_bus(pdev->bus); 5628 } 5629 EXPORT_SYMBOL_GPL(pci_reset_bus); 5630 5631 /** 5632 * pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count 5633 * @dev: PCI device to query 5634 * 5635 * Returns mmrbc: maximum designed memory read count in bytes or 5636 * appropriate error value. 5637 */ 5638 int pcix_get_max_mmrbc(struct pci_dev *dev) 5639 { 5640 int cap; 5641 u32 stat; 5642 5643 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5644 if (!cap) 5645 return -EINVAL; 5646 5647 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5648 return -EINVAL; 5649 5650 return 512 << ((stat & PCI_X_STATUS_MAX_READ) >> 21); 5651 } 5652 EXPORT_SYMBOL(pcix_get_max_mmrbc); 5653 5654 /** 5655 * pcix_get_mmrbc - get PCI-X maximum memory read byte count 5656 * @dev: PCI device to query 5657 * 5658 * Returns mmrbc: maximum memory read count in bytes or appropriate error 5659 * value. 5660 */ 5661 int pcix_get_mmrbc(struct pci_dev *dev) 5662 { 5663 int cap; 5664 u16 cmd; 5665 5666 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5667 if (!cap) 5668 return -EINVAL; 5669 5670 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5671 return -EINVAL; 5672 5673 return 512 << ((cmd & PCI_X_CMD_MAX_READ) >> 2); 5674 } 5675 EXPORT_SYMBOL(pcix_get_mmrbc); 5676 5677 /** 5678 * pcix_set_mmrbc - set PCI-X maximum memory read byte count 5679 * @dev: PCI device to query 5680 * @mmrbc: maximum memory read count in bytes 5681 * valid values are 512, 1024, 2048, 4096 5682 * 5683 * If possible sets maximum memory read byte count, some bridges have errata 5684 * that prevent this. 5685 */ 5686 int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc) 5687 { 5688 int cap; 5689 u32 stat, v, o; 5690 u16 cmd; 5691 5692 if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(mmrbc)) 5693 return -EINVAL; 5694 5695 v = ffs(mmrbc) - 10; 5696 5697 cap = pci_find_capability(dev, PCI_CAP_ID_PCIX); 5698 if (!cap) 5699 return -EINVAL; 5700 5701 if (pci_read_config_dword(dev, cap + PCI_X_STATUS, &stat)) 5702 return -EINVAL; 5703 5704 if (v > (stat & PCI_X_STATUS_MAX_READ) >> 21) 5705 return -E2BIG; 5706 5707 if (pci_read_config_word(dev, cap + PCI_X_CMD, &cmd)) 5708 return -EINVAL; 5709 5710 o = (cmd & PCI_X_CMD_MAX_READ) >> 2; 5711 if (o != v) { 5712 if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC)) 5713 return -EIO; 5714 5715 cmd &= ~PCI_X_CMD_MAX_READ; 5716 cmd |= v << 2; 5717 if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd)) 5718 return -EIO; 5719 } 5720 return 0; 5721 } 5722 EXPORT_SYMBOL(pcix_set_mmrbc); 5723 5724 /** 5725 * pcie_get_readrq - get PCI Express read request size 5726 * @dev: PCI device to query 5727 * 5728 * Returns maximum memory read request in bytes or appropriate error value. 5729 */ 5730 int pcie_get_readrq(struct pci_dev *dev) 5731 { 5732 u16 ctl; 5733 5734 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 5735 5736 return 128 << ((ctl & PCI_EXP_DEVCTL_READRQ) >> 12); 5737 } 5738 EXPORT_SYMBOL(pcie_get_readrq); 5739 5740 /** 5741 * pcie_set_readrq - set PCI Express maximum memory read request 5742 * @dev: PCI device to query 5743 * @rq: maximum memory read count in bytes 5744 * valid values are 128, 256, 512, 1024, 2048, 4096 5745 * 5746 * If possible sets maximum memory read request in bytes 5747 */ 5748 int pcie_set_readrq(struct pci_dev *dev, int rq) 5749 { 5750 u16 v; 5751 int ret; 5752 5753 if (rq < 128 || rq > 4096 || !is_power_of_2(rq)) 5754 return -EINVAL; 5755 5756 /* 5757 * If using the "performance" PCIe config, we clamp the read rq 5758 * size to the max packet size to keep the host bridge from 5759 * generating requests larger than we can cope with. 5760 */ 5761 if (pcie_bus_config == PCIE_BUS_PERFORMANCE) { 5762 int mps = pcie_get_mps(dev); 5763 5764 if (mps < rq) 5765 rq = mps; 5766 } 5767 5768 v = (ffs(rq) - 8) << 12; 5769 5770 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 5771 PCI_EXP_DEVCTL_READRQ, v); 5772 5773 return pcibios_err_to_errno(ret); 5774 } 5775 EXPORT_SYMBOL(pcie_set_readrq); 5776 5777 /** 5778 * pcie_get_mps - get PCI Express maximum payload size 5779 * @dev: PCI device to query 5780 * 5781 * Returns maximum payload size in bytes 5782 */ 5783 int pcie_get_mps(struct pci_dev *dev) 5784 { 5785 u16 ctl; 5786 5787 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &ctl); 5788 5789 return 128 << ((ctl & PCI_EXP_DEVCTL_PAYLOAD) >> 5); 5790 } 5791 EXPORT_SYMBOL(pcie_get_mps); 5792 5793 /** 5794 * pcie_set_mps - set PCI Express maximum payload size 5795 * @dev: PCI device to query 5796 * @mps: maximum payload size in bytes 5797 * valid values are 128, 256, 512, 1024, 2048, 4096 5798 * 5799 * If possible sets maximum payload size 5800 */ 5801 int pcie_set_mps(struct pci_dev *dev, int mps) 5802 { 5803 u16 v; 5804 int ret; 5805 5806 if (mps < 128 || mps > 4096 || !is_power_of_2(mps)) 5807 return -EINVAL; 5808 5809 v = ffs(mps) - 8; 5810 if (v > dev->pcie_mpss) 5811 return -EINVAL; 5812 v <<= 5; 5813 5814 ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL, 5815 PCI_EXP_DEVCTL_PAYLOAD, v); 5816 5817 return pcibios_err_to_errno(ret); 5818 } 5819 EXPORT_SYMBOL(pcie_set_mps); 5820 5821 /** 5822 * pcie_bandwidth_available - determine minimum link settings of a PCIe 5823 * device and its bandwidth limitation 5824 * @dev: PCI device to query 5825 * @limiting_dev: storage for device causing the bandwidth limitation 5826 * @speed: storage for speed of limiting device 5827 * @width: storage for width of limiting device 5828 * 5829 * Walk up the PCI device chain and find the point where the minimum 5830 * bandwidth is available. Return the bandwidth available there and (if 5831 * limiting_dev, speed, and width pointers are supplied) information about 5832 * that point. The bandwidth returned is in Mb/s, i.e., megabits/second of 5833 * raw bandwidth. 5834 */ 5835 u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, 5836 enum pci_bus_speed *speed, 5837 enum pcie_link_width *width) 5838 { 5839 u16 lnksta; 5840 enum pci_bus_speed next_speed; 5841 enum pcie_link_width next_width; 5842 u32 bw, next_bw; 5843 5844 if (speed) 5845 *speed = PCI_SPEED_UNKNOWN; 5846 if (width) 5847 *width = PCIE_LNK_WIDTH_UNKNOWN; 5848 5849 bw = 0; 5850 5851 while (dev) { 5852 pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta); 5853 5854 next_speed = pcie_link_speed[lnksta & PCI_EXP_LNKSTA_CLS]; 5855 next_width = (lnksta & PCI_EXP_LNKSTA_NLW) >> 5856 PCI_EXP_LNKSTA_NLW_SHIFT; 5857 5858 next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed); 5859 5860 /* Check if current device limits the total bandwidth */ 5861 if (!bw || next_bw <= bw) { 5862 bw = next_bw; 5863 5864 if (limiting_dev) 5865 *limiting_dev = dev; 5866 if (speed) 5867 *speed = next_speed; 5868 if (width) 5869 *width = next_width; 5870 } 5871 5872 dev = pci_upstream_bridge(dev); 5873 } 5874 5875 return bw; 5876 } 5877 EXPORT_SYMBOL(pcie_bandwidth_available); 5878 5879 /** 5880 * pcie_get_speed_cap - query for the PCI device's link speed capability 5881 * @dev: PCI device to query 5882 * 5883 * Query the PCI device speed capability. Return the maximum link speed 5884 * supported by the device. 5885 */ 5886 enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev) 5887 { 5888 u32 lnkcap2, lnkcap; 5889 5890 /* 5891 * Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The 5892 * implementation note there recommends using the Supported Link 5893 * Speeds Vector in Link Capabilities 2 when supported. 5894 * 5895 * Without Link Capabilities 2, i.e., prior to PCIe r3.0, software 5896 * should use the Supported Link Speeds field in Link Capabilities, 5897 * where only 2.5 GT/s and 5.0 GT/s speeds were defined. 5898 */ 5899 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, &lnkcap2); 5900 5901 /* PCIe r3.0-compliant */ 5902 if (lnkcap2) 5903 return PCIE_LNKCAP2_SLS2SPEED(lnkcap2); 5904 5905 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 5906 if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB) 5907 return PCIE_SPEED_5_0GT; 5908 else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB) 5909 return PCIE_SPEED_2_5GT; 5910 5911 return PCI_SPEED_UNKNOWN; 5912 } 5913 EXPORT_SYMBOL(pcie_get_speed_cap); 5914 5915 /** 5916 * pcie_get_width_cap - query for the PCI device's link width capability 5917 * @dev: PCI device to query 5918 * 5919 * Query the PCI device width capability. Return the maximum link width 5920 * supported by the device. 5921 */ 5922 enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev) 5923 { 5924 u32 lnkcap; 5925 5926 pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap); 5927 if (lnkcap) 5928 return (lnkcap & PCI_EXP_LNKCAP_MLW) >> 4; 5929 5930 return PCIE_LNK_WIDTH_UNKNOWN; 5931 } 5932 EXPORT_SYMBOL(pcie_get_width_cap); 5933 5934 /** 5935 * pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability 5936 * @dev: PCI device 5937 * @speed: storage for link speed 5938 * @width: storage for link width 5939 * 5940 * Calculate a PCI device's link bandwidth by querying for its link speed 5941 * and width, multiplying them, and applying encoding overhead. The result 5942 * is in Mb/s, i.e., megabits/second of raw bandwidth. 5943 */ 5944 u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed, 5945 enum pcie_link_width *width) 5946 { 5947 *speed = pcie_get_speed_cap(dev); 5948 *width = pcie_get_width_cap(dev); 5949 5950 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 5951 return 0; 5952 5953 return *width * PCIE_SPEED2MBS_ENC(*speed); 5954 } 5955 5956 /** 5957 * __pcie_print_link_status - Report the PCI device's link speed and width 5958 * @dev: PCI device to query 5959 * @verbose: Print info even when enough bandwidth is available 5960 * 5961 * If the available bandwidth at the device is less than the device is 5962 * capable of, report the device's maximum possible bandwidth and the 5963 * upstream link that limits its performance. If @verbose, always print 5964 * the available bandwidth, even if the device isn't constrained. 5965 */ 5966 void __pcie_print_link_status(struct pci_dev *dev, bool verbose) 5967 { 5968 enum pcie_link_width width, width_cap; 5969 enum pci_bus_speed speed, speed_cap; 5970 struct pci_dev *limiting_dev = NULL; 5971 u32 bw_avail, bw_cap; 5972 5973 bw_cap = pcie_bandwidth_capable(dev, &speed_cap, &width_cap); 5974 bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width); 5975 5976 if (bw_avail >= bw_cap && verbose) 5977 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n", 5978 bw_cap / 1000, bw_cap % 1000, 5979 pci_speed_string(speed_cap), width_cap); 5980 else if (bw_avail < bw_cap) 5981 pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n", 5982 bw_avail / 1000, bw_avail % 1000, 5983 pci_speed_string(speed), width, 5984 limiting_dev ? pci_name(limiting_dev) : "<unknown>", 5985 bw_cap / 1000, bw_cap % 1000, 5986 pci_speed_string(speed_cap), width_cap); 5987 } 5988 5989 /** 5990 * pcie_print_link_status - Report the PCI device's link speed and width 5991 * @dev: PCI device to query 5992 * 5993 * Report the available bandwidth at the device. 5994 */ 5995 void pcie_print_link_status(struct pci_dev *dev) 5996 { 5997 __pcie_print_link_status(dev, true); 5998 } 5999 EXPORT_SYMBOL(pcie_print_link_status); 6000 6001 /** 6002 * pci_select_bars - Make BAR mask from the type of resource 6003 * @dev: the PCI device for which BAR mask is made 6004 * @flags: resource type mask to be selected 6005 * 6006 * This helper routine makes bar mask from the type of resource. 6007 */ 6008 int pci_select_bars(struct pci_dev *dev, unsigned long flags) 6009 { 6010 int i, bars = 0; 6011 for (i = 0; i < PCI_NUM_RESOURCES; i++) 6012 if (pci_resource_flags(dev, i) & flags) 6013 bars |= (1 << i); 6014 return bars; 6015 } 6016 EXPORT_SYMBOL(pci_select_bars); 6017 6018 /* Some architectures require additional programming to enable VGA */ 6019 static arch_set_vga_state_t arch_set_vga_state; 6020 6021 void __init pci_register_set_vga_state(arch_set_vga_state_t func) 6022 { 6023 arch_set_vga_state = func; /* NULL disables */ 6024 } 6025 6026 static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode, 6027 unsigned int command_bits, u32 flags) 6028 { 6029 if (arch_set_vga_state) 6030 return arch_set_vga_state(dev, decode, command_bits, 6031 flags); 6032 return 0; 6033 } 6034 6035 /** 6036 * pci_set_vga_state - set VGA decode state on device and parents if requested 6037 * @dev: the PCI device 6038 * @decode: true = enable decoding, false = disable decoding 6039 * @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY 6040 * @flags: traverse ancestors and change bridges 6041 * CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE 6042 */ 6043 int pci_set_vga_state(struct pci_dev *dev, bool decode, 6044 unsigned int command_bits, u32 flags) 6045 { 6046 struct pci_bus *bus; 6047 struct pci_dev *bridge; 6048 u16 cmd; 6049 int rc; 6050 6051 WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY))); 6052 6053 /* ARCH specific VGA enables */ 6054 rc = pci_set_vga_state_arch(dev, decode, command_bits, flags); 6055 if (rc) 6056 return rc; 6057 6058 if (flags & PCI_VGA_STATE_CHANGE_DECODES) { 6059 pci_read_config_word(dev, PCI_COMMAND, &cmd); 6060 if (decode) 6061 cmd |= command_bits; 6062 else 6063 cmd &= ~command_bits; 6064 pci_write_config_word(dev, PCI_COMMAND, cmd); 6065 } 6066 6067 if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE)) 6068 return 0; 6069 6070 bus = dev->bus; 6071 while (bus) { 6072 bridge = bus->self; 6073 if (bridge) { 6074 pci_read_config_word(bridge, PCI_BRIDGE_CONTROL, 6075 &cmd); 6076 if (decode) 6077 cmd |= PCI_BRIDGE_CTL_VGA; 6078 else 6079 cmd &= ~PCI_BRIDGE_CTL_VGA; 6080 pci_write_config_word(bridge, PCI_BRIDGE_CONTROL, 6081 cmd); 6082 } 6083 bus = bus->parent; 6084 } 6085 return 0; 6086 } 6087 6088 #ifdef CONFIG_ACPI 6089 bool pci_pr3_present(struct pci_dev *pdev) 6090 { 6091 struct acpi_device *adev; 6092 6093 if (acpi_disabled) 6094 return false; 6095 6096 adev = ACPI_COMPANION(&pdev->dev); 6097 if (!adev) 6098 return false; 6099 6100 return adev->power.flags.power_resources && 6101 acpi_has_method(adev->handle, "_PR3"); 6102 } 6103 EXPORT_SYMBOL_GPL(pci_pr3_present); 6104 #endif 6105 6106 /** 6107 * pci_add_dma_alias - Add a DMA devfn alias for a device 6108 * @dev: the PCI device for which alias is added 6109 * @devfn_from: alias slot and function 6110 * @nr_devfns: number of subsequent devfns to alias 6111 * 6112 * This helper encodes an 8-bit devfn as a bit number in dma_alias_mask 6113 * which is used to program permissible bus-devfn source addresses for DMA 6114 * requests in an IOMMU. These aliases factor into IOMMU group creation 6115 * and are useful for devices generating DMA requests beyond or different 6116 * from their logical bus-devfn. Examples include device quirks where the 6117 * device simply uses the wrong devfn, as well as non-transparent bridges 6118 * where the alias may be a proxy for devices in another domain. 6119 * 6120 * IOMMU group creation is performed during device discovery or addition, 6121 * prior to any potential DMA mapping and therefore prior to driver probing 6122 * (especially for userspace assigned devices where IOMMU group definition 6123 * cannot be left as a userspace activity). DMA aliases should therefore 6124 * be configured via quirks, such as the PCI fixup header quirk. 6125 */ 6126 void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from, unsigned nr_devfns) 6127 { 6128 int devfn_to; 6129 6130 nr_devfns = min(nr_devfns, (unsigned) MAX_NR_DEVFNS - devfn_from); 6131 devfn_to = devfn_from + nr_devfns - 1; 6132 6133 if (!dev->dma_alias_mask) 6134 dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL); 6135 if (!dev->dma_alias_mask) { 6136 pci_warn(dev, "Unable to allocate DMA alias mask\n"); 6137 return; 6138 } 6139 6140 bitmap_set(dev->dma_alias_mask, devfn_from, nr_devfns); 6141 6142 if (nr_devfns == 1) 6143 pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n", 6144 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from)); 6145 else if (nr_devfns > 1) 6146 pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n", 6147 PCI_SLOT(devfn_from), PCI_FUNC(devfn_from), 6148 PCI_SLOT(devfn_to), PCI_FUNC(devfn_to)); 6149 } 6150 6151 bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2) 6152 { 6153 return (dev1->dma_alias_mask && 6154 test_bit(dev2->devfn, dev1->dma_alias_mask)) || 6155 (dev2->dma_alias_mask && 6156 test_bit(dev1->devfn, dev2->dma_alias_mask)) || 6157 pci_real_dma_dev(dev1) == dev2 || 6158 pci_real_dma_dev(dev2) == dev1; 6159 } 6160 6161 bool pci_device_is_present(struct pci_dev *pdev) 6162 { 6163 u32 v; 6164 6165 if (pci_dev_is_disconnected(pdev)) 6166 return false; 6167 return pci_bus_read_dev_vendor_id(pdev->bus, pdev->devfn, &v, 0); 6168 } 6169 EXPORT_SYMBOL_GPL(pci_device_is_present); 6170 6171 void pci_ignore_hotplug(struct pci_dev *dev) 6172 { 6173 struct pci_dev *bridge = dev->bus->self; 6174 6175 dev->ignore_hotplug = 1; 6176 /* Propagate the "ignore hotplug" setting to the parent bridge. */ 6177 if (bridge) 6178 bridge->ignore_hotplug = 1; 6179 } 6180 EXPORT_SYMBOL_GPL(pci_ignore_hotplug); 6181 6182 /** 6183 * pci_real_dma_dev - Get PCI DMA device for PCI device 6184 * @dev: the PCI device that may have a PCI DMA alias 6185 * 6186 * Permits the platform to provide architecture-specific functionality to 6187 * devices needing to alias DMA to another PCI device on another PCI bus. If 6188 * the PCI device is on the same bus, it is recommended to use 6189 * pci_add_dma_alias(). This is the default implementation. Architecture 6190 * implementations can override this. 6191 */ 6192 struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev) 6193 { 6194 return dev; 6195 } 6196 6197 resource_size_t __weak pcibios_default_alignment(void) 6198 { 6199 return 0; 6200 } 6201 6202 /* 6203 * Arches that don't want to expose struct resource to userland as-is in 6204 * sysfs and /proc can implement their own pci_resource_to_user(). 6205 */ 6206 void __weak pci_resource_to_user(const struct pci_dev *dev, int bar, 6207 const struct resource *rsrc, 6208 resource_size_t *start, resource_size_t *end) 6209 { 6210 *start = rsrc->start; 6211 *end = rsrc->end; 6212 } 6213 6214 static char *resource_alignment_param; 6215 static DEFINE_SPINLOCK(resource_alignment_lock); 6216 6217 /** 6218 * pci_specified_resource_alignment - get resource alignment specified by user. 6219 * @dev: the PCI device to get 6220 * @resize: whether or not to change resources' size when reassigning alignment 6221 * 6222 * RETURNS: Resource alignment if it is specified. 6223 * Zero if it is not specified. 6224 */ 6225 static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev, 6226 bool *resize) 6227 { 6228 int align_order, count; 6229 resource_size_t align = pcibios_default_alignment(); 6230 const char *p; 6231 int ret; 6232 6233 spin_lock(&resource_alignment_lock); 6234 p = resource_alignment_param; 6235 if (!p || !*p) 6236 goto out; 6237 if (pci_has_flag(PCI_PROBE_ONLY)) { 6238 align = 0; 6239 pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n"); 6240 goto out; 6241 } 6242 6243 while (*p) { 6244 count = 0; 6245 if (sscanf(p, "%d%n", &align_order, &count) == 1 && 6246 p[count] == '@') { 6247 p += count + 1; 6248 if (align_order > 63) { 6249 pr_err("PCI: Invalid requested alignment (order %d)\n", 6250 align_order); 6251 align_order = PAGE_SHIFT; 6252 } 6253 } else { 6254 align_order = PAGE_SHIFT; 6255 } 6256 6257 ret = pci_dev_str_match(dev, p, &p); 6258 if (ret == 1) { 6259 *resize = true; 6260 align = 1ULL << align_order; 6261 break; 6262 } else if (ret < 0) { 6263 pr_err("PCI: Can't parse resource_alignment parameter: %s\n", 6264 p); 6265 break; 6266 } 6267 6268 if (*p != ';' && *p != ',') { 6269 /* End of param or invalid format */ 6270 break; 6271 } 6272 p++; 6273 } 6274 out: 6275 spin_unlock(&resource_alignment_lock); 6276 return align; 6277 } 6278 6279 static void pci_request_resource_alignment(struct pci_dev *dev, int bar, 6280 resource_size_t align, bool resize) 6281 { 6282 struct resource *r = &dev->resource[bar]; 6283 resource_size_t size; 6284 6285 if (!(r->flags & IORESOURCE_MEM)) 6286 return; 6287 6288 if (r->flags & IORESOURCE_PCI_FIXED) { 6289 pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n", 6290 bar, r, (unsigned long long)align); 6291 return; 6292 } 6293 6294 size = resource_size(r); 6295 if (size >= align) 6296 return; 6297 6298 /* 6299 * Increase the alignment of the resource. There are two ways we 6300 * can do this: 6301 * 6302 * 1) Increase the size of the resource. BARs are aligned on their 6303 * size, so when we reallocate space for this resource, we'll 6304 * allocate it with the larger alignment. This also prevents 6305 * assignment of any other BARs inside the alignment region, so 6306 * if we're requesting page alignment, this means no other BARs 6307 * will share the page. 6308 * 6309 * The disadvantage is that this makes the resource larger than 6310 * the hardware BAR, which may break drivers that compute things 6311 * based on the resource size, e.g., to find registers at a 6312 * fixed offset before the end of the BAR. 6313 * 6314 * 2) Retain the resource size, but use IORESOURCE_STARTALIGN and 6315 * set r->start to the desired alignment. By itself this 6316 * doesn't prevent other BARs being put inside the alignment 6317 * region, but if we realign *every* resource of every device in 6318 * the system, none of them will share an alignment region. 6319 * 6320 * When the user has requested alignment for only some devices via 6321 * the "pci=resource_alignment" argument, "resize" is true and we 6322 * use the first method. Otherwise we assume we're aligning all 6323 * devices and we use the second. 6324 */ 6325 6326 pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n", 6327 bar, r, (unsigned long long)align); 6328 6329 if (resize) { 6330 r->start = 0; 6331 r->end = align - 1; 6332 } else { 6333 r->flags &= ~IORESOURCE_SIZEALIGN; 6334 r->flags |= IORESOURCE_STARTALIGN; 6335 r->start = align; 6336 r->end = r->start + size - 1; 6337 } 6338 r->flags |= IORESOURCE_UNSET; 6339 } 6340 6341 /* 6342 * This function disables memory decoding and releases memory resources 6343 * of the device specified by kernel's boot parameter 'pci=resource_alignment='. 6344 * It also rounds up size to specified alignment. 6345 * Later on, the kernel will assign page-aligned memory resource back 6346 * to the device. 6347 */ 6348 void pci_reassigndev_resource_alignment(struct pci_dev *dev) 6349 { 6350 int i; 6351 struct resource *r; 6352 resource_size_t align; 6353 u16 command; 6354 bool resize = false; 6355 6356 /* 6357 * VF BARs are read-only zero according to SR-IOV spec r1.1, sec 6358 * 3.4.1.11. Their resources are allocated from the space 6359 * described by the VF BARx register in the PF's SR-IOV capability. 6360 * We can't influence their alignment here. 6361 */ 6362 if (dev->is_virtfn) 6363 return; 6364 6365 /* check if specified PCI is target device to reassign */ 6366 align = pci_specified_resource_alignment(dev, &resize); 6367 if (!align) 6368 return; 6369 6370 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL && 6371 (dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) { 6372 pci_warn(dev, "Can't reassign resources to host bridge\n"); 6373 return; 6374 } 6375 6376 pci_read_config_word(dev, PCI_COMMAND, &command); 6377 command &= ~PCI_COMMAND_MEMORY; 6378 pci_write_config_word(dev, PCI_COMMAND, command); 6379 6380 for (i = 0; i <= PCI_ROM_RESOURCE; i++) 6381 pci_request_resource_alignment(dev, i, align, resize); 6382 6383 /* 6384 * Need to disable bridge's resource window, 6385 * to enable the kernel to reassign new resource 6386 * window later on. 6387 */ 6388 if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) { 6389 for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) { 6390 r = &dev->resource[i]; 6391 if (!(r->flags & IORESOURCE_MEM)) 6392 continue; 6393 r->flags |= IORESOURCE_UNSET; 6394 r->end = resource_size(r) - 1; 6395 r->start = 0; 6396 } 6397 pci_disable_bridge_window(dev); 6398 } 6399 } 6400 6401 static ssize_t resource_alignment_show(struct bus_type *bus, char *buf) 6402 { 6403 size_t count = 0; 6404 6405 spin_lock(&resource_alignment_lock); 6406 if (resource_alignment_param) 6407 count = scnprintf(buf, PAGE_SIZE, "%s", resource_alignment_param); 6408 spin_unlock(&resource_alignment_lock); 6409 6410 /* 6411 * When set by the command line, resource_alignment_param will not 6412 * have a trailing line feed, which is ugly. So conditionally add 6413 * it here. 6414 */ 6415 if (count >= 2 && buf[count - 2] != '\n' && count < PAGE_SIZE - 1) { 6416 buf[count - 1] = '\n'; 6417 buf[count++] = 0; 6418 } 6419 6420 return count; 6421 } 6422 6423 static ssize_t resource_alignment_store(struct bus_type *bus, 6424 const char *buf, size_t count) 6425 { 6426 char *param = kstrndup(buf, count, GFP_KERNEL); 6427 6428 if (!param) 6429 return -ENOMEM; 6430 6431 spin_lock(&resource_alignment_lock); 6432 kfree(resource_alignment_param); 6433 resource_alignment_param = param; 6434 spin_unlock(&resource_alignment_lock); 6435 return count; 6436 } 6437 6438 static BUS_ATTR_RW(resource_alignment); 6439 6440 static int __init pci_resource_alignment_sysfs_init(void) 6441 { 6442 return bus_create_file(&pci_bus_type, 6443 &bus_attr_resource_alignment); 6444 } 6445 late_initcall(pci_resource_alignment_sysfs_init); 6446 6447 static void pci_no_domains(void) 6448 { 6449 #ifdef CONFIG_PCI_DOMAINS 6450 pci_domains_supported = 0; 6451 #endif 6452 } 6453 6454 #ifdef CONFIG_PCI_DOMAINS_GENERIC 6455 static atomic_t __domain_nr = ATOMIC_INIT(-1); 6456 6457 static int pci_get_new_domain_nr(void) 6458 { 6459 return atomic_inc_return(&__domain_nr); 6460 } 6461 6462 static int of_pci_bus_find_domain_nr(struct device *parent) 6463 { 6464 static int use_dt_domains = -1; 6465 int domain = -1; 6466 6467 if (parent) 6468 domain = of_get_pci_domain_nr(parent->of_node); 6469 6470 /* 6471 * Check DT domain and use_dt_domains values. 6472 * 6473 * If DT domain property is valid (domain >= 0) and 6474 * use_dt_domains != 0, the DT assignment is valid since this means 6475 * we have not previously allocated a domain number by using 6476 * pci_get_new_domain_nr(); we should also update use_dt_domains to 6477 * 1, to indicate that we have just assigned a domain number from 6478 * DT. 6479 * 6480 * If DT domain property value is not valid (ie domain < 0), and we 6481 * have not previously assigned a domain number from DT 6482 * (use_dt_domains != 1) we should assign a domain number by 6483 * using the: 6484 * 6485 * pci_get_new_domain_nr() 6486 * 6487 * API and update the use_dt_domains value to keep track of method we 6488 * are using to assign domain numbers (use_dt_domains = 0). 6489 * 6490 * All other combinations imply we have a platform that is trying 6491 * to mix domain numbers obtained from DT and pci_get_new_domain_nr(), 6492 * which is a recipe for domain mishandling and it is prevented by 6493 * invalidating the domain value (domain = -1) and printing a 6494 * corresponding error. 6495 */ 6496 if (domain >= 0 && use_dt_domains) { 6497 use_dt_domains = 1; 6498 } else if (domain < 0 && use_dt_domains != 1) { 6499 use_dt_domains = 0; 6500 domain = pci_get_new_domain_nr(); 6501 } else { 6502 if (parent) 6503 pr_err("Node %pOF has ", parent->of_node); 6504 pr_err("Inconsistent \"linux,pci-domain\" property in DT\n"); 6505 domain = -1; 6506 } 6507 6508 return domain; 6509 } 6510 6511 int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent) 6512 { 6513 return acpi_disabled ? of_pci_bus_find_domain_nr(parent) : 6514 acpi_pci_bus_find_domain_nr(bus); 6515 } 6516 #endif 6517 6518 /** 6519 * pci_ext_cfg_avail - can we access extended PCI config space? 6520 * 6521 * Returns 1 if we can access PCI extended config space (offsets 6522 * greater than 0xff). This is the default implementation. Architecture 6523 * implementations can override this. 6524 */ 6525 int __weak pci_ext_cfg_avail(void) 6526 { 6527 return 1; 6528 } 6529 6530 void __weak pci_fixup_cardbus(struct pci_bus *bus) 6531 { 6532 } 6533 EXPORT_SYMBOL(pci_fixup_cardbus); 6534 6535 static int __init pci_setup(char *str) 6536 { 6537 while (str) { 6538 char *k = strchr(str, ','); 6539 if (k) 6540 *k++ = 0; 6541 if (*str && (str = pcibios_setup(str)) && *str) { 6542 if (!strcmp(str, "nomsi")) { 6543 pci_no_msi(); 6544 } else if (!strncmp(str, "noats", 5)) { 6545 pr_info("PCIe: ATS is disabled\n"); 6546 pcie_ats_disabled = true; 6547 } else if (!strcmp(str, "noaer")) { 6548 pci_no_aer(); 6549 } else if (!strcmp(str, "earlydump")) { 6550 pci_early_dump = true; 6551 } else if (!strncmp(str, "realloc=", 8)) { 6552 pci_realloc_get_opt(str + 8); 6553 } else if (!strncmp(str, "realloc", 7)) { 6554 pci_realloc_get_opt("on"); 6555 } else if (!strcmp(str, "nodomains")) { 6556 pci_no_domains(); 6557 } else if (!strncmp(str, "noari", 5)) { 6558 pcie_ari_disabled = true; 6559 } else if (!strncmp(str, "cbiosize=", 9)) { 6560 pci_cardbus_io_size = memparse(str + 9, &str); 6561 } else if (!strncmp(str, "cbmemsize=", 10)) { 6562 pci_cardbus_mem_size = memparse(str + 10, &str); 6563 } else if (!strncmp(str, "resource_alignment=", 19)) { 6564 resource_alignment_param = str + 19; 6565 } else if (!strncmp(str, "ecrc=", 5)) { 6566 pcie_ecrc_get_policy(str + 5); 6567 } else if (!strncmp(str, "hpiosize=", 9)) { 6568 pci_hotplug_io_size = memparse(str + 9, &str); 6569 } else if (!strncmp(str, "hpmmiosize=", 11)) { 6570 pci_hotplug_mmio_size = memparse(str + 11, &str); 6571 } else if (!strncmp(str, "hpmmioprefsize=", 15)) { 6572 pci_hotplug_mmio_pref_size = memparse(str + 15, &str); 6573 } else if (!strncmp(str, "hpmemsize=", 10)) { 6574 pci_hotplug_mmio_size = memparse(str + 10, &str); 6575 pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size; 6576 } else if (!strncmp(str, "hpbussize=", 10)) { 6577 pci_hotplug_bus_size = 6578 simple_strtoul(str + 10, &str, 0); 6579 if (pci_hotplug_bus_size > 0xff) 6580 pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE; 6581 } else if (!strncmp(str, "pcie_bus_tune_off", 17)) { 6582 pcie_bus_config = PCIE_BUS_TUNE_OFF; 6583 } else if (!strncmp(str, "pcie_bus_safe", 13)) { 6584 pcie_bus_config = PCIE_BUS_SAFE; 6585 } else if (!strncmp(str, "pcie_bus_perf", 13)) { 6586 pcie_bus_config = PCIE_BUS_PERFORMANCE; 6587 } else if (!strncmp(str, "pcie_bus_peer2peer", 18)) { 6588 pcie_bus_config = PCIE_BUS_PEER2PEER; 6589 } else if (!strncmp(str, "pcie_scan_all", 13)) { 6590 pci_add_flags(PCI_SCAN_ALL_PCIE_DEVS); 6591 } else if (!strncmp(str, "disable_acs_redir=", 18)) { 6592 disable_acs_redir_param = str + 18; 6593 } else { 6594 pr_err("PCI: Unknown option `%s'\n", str); 6595 } 6596 } 6597 str = k; 6598 } 6599 return 0; 6600 } 6601 early_param("pci", pci_setup); 6602 6603 /* 6604 * 'resource_alignment_param' and 'disable_acs_redir_param' are initialized 6605 * in pci_setup(), above, to point to data in the __initdata section which 6606 * will be freed after the init sequence is complete. We can't allocate memory 6607 * in pci_setup() because some architectures do not have any memory allocation 6608 * service available during an early_param() call. So we allocate memory and 6609 * copy the variable here before the init section is freed. 6610 * 6611 */ 6612 static int __init pci_realloc_setup_params(void) 6613 { 6614 resource_alignment_param = kstrdup(resource_alignment_param, 6615 GFP_KERNEL); 6616 disable_acs_redir_param = kstrdup(disable_acs_redir_param, GFP_KERNEL); 6617 6618 return 0; 6619 } 6620 pure_initcall(pci_realloc_setup_params); 6621