Lines Matching +full:lock +full:- +full:latency +full:- +full:ns
1 // SPDX-License-Identifier: GPL-2.0
55 u32 aspm_capable:7; /* Capable ASPM state with latency */
101 list_for_each_entry(child, &linkbus->devices, bus_list) in pci_function_0()
102 if (PCI_FUNC(child->devfn) == 0) in pci_function_0()
120 return link->aspm_default; in policy_to_aspm_state()
136 return link->clkpm_default; in policy_to_clkpm_state()
144 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_set_clkpm_nocheck()
147 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_set_clkpm_nocheck()
151 link->clkpm_enabled = !!enable; in pcie_set_clkpm_nocheck()
160 if (!link->clkpm_capable || link->clkpm_disable) in pcie_set_clkpm()
163 if (link->clkpm_enabled == enable) in pcie_set_clkpm()
174 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_clkpm_cap_init()
177 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_clkpm_cap_init()
188 link->clkpm_enabled = enabled; in pcie_clkpm_cap_init()
189 link->clkpm_default = enabled; in pcie_clkpm_cap_init()
190 link->clkpm_capable = capable; in pcie_clkpm_cap_init()
191 link->clkpm_disable = blacklist ? 1 : 0; in pcie_clkpm_cap_init()
197 * common clock. That will reduce the ASPM state exit latency.
203 struct pci_dev *child, *parent = link->pdev; in pcie_aspm_configure_common_clock()
204 struct pci_bus *linkbus = parent->subordinate; in pcie_aspm_configure_common_clock()
209 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); in pcie_aspm_configure_common_clock()
228 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_configure_common_clock()
243 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_configure_common_clock()
245 child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC; in pcie_aspm_configure_common_clock()
254 if (pcie_retrain_link(link->pdev, true)) { in pcie_aspm_configure_common_clock()
258 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_aspm_configure_common_clock()
261 child_old_ccc[PCI_FUNC(child->devfn)]); in pcie_aspm_configure_common_clock()
267 /* Convert L0s latency encoding to ns */
277 /* Convert L0s acceptable latency encoding to ns */
281 return -1U; in calc_l0s_acceptable()
285 /* Convert L1 latency encoding to ns */
295 /* Convert L1 acceptable latency encoding to ns */
299 return -1U; in calc_l1_acceptable()
331 * LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max in encode_l12_threshold()
335 *scale = 0; /* Value times 1ns */ in encode_l12_threshold()
338 *scale = 1; /* Value times 32ns */ in encode_l12_threshold()
341 *scale = 2; /* Value times 1024ns */ in encode_l12_threshold()
344 *scale = 3; /* Value times 32768ns */ in encode_l12_threshold()
347 *scale = 4; /* Value times 1048576ns */ in encode_l12_threshold()
350 *scale = 5; /* Value times 33554432ns */ in encode_l12_threshold()
360 u32 latency, encoding, lnkcap_up, lnkcap_dw; in pcie_aspm_check_latency() local
366 /* Device not in D0 doesn't need latency check */ in pcie_aspm_check_latency()
367 if ((endpoint->current_state != PCI_D0) && in pcie_aspm_check_latency()
368 (endpoint->current_state != PCI_UNKNOWN)) in pcie_aspm_check_latency()
371 link = endpoint->bus->self->link_state; in pcie_aspm_check_latency()
373 /* Calculate endpoint L0s acceptable latency */ in pcie_aspm_check_latency()
374 encoding = (endpoint->devcap & PCI_EXP_DEVCAP_L0S) >> 6; in pcie_aspm_check_latency()
377 /* Calculate endpoint L1 acceptable latency */ in pcie_aspm_check_latency()
378 encoding = (endpoint->devcap & PCI_EXP_DEVCAP_L1) >> 9; in pcie_aspm_check_latency()
382 struct pci_dev *dev = pci_function_0(link->pdev->subordinate); in pcie_aspm_check_latency()
385 pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP, in pcie_aspm_check_latency()
394 /* Check upstream direction L0s latency */ in pcie_aspm_check_latency()
395 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && in pcie_aspm_check_latency()
397 link->aspm_capable &= ~ASPM_STATE_L0S_UP; in pcie_aspm_check_latency()
399 /* Check downstream direction L0s latency */ in pcie_aspm_check_latency()
400 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && in pcie_aspm_check_latency()
402 link->aspm_capable &= ~ASPM_STATE_L0S_DW; in pcie_aspm_check_latency()
404 * Check L1 latency. in pcie_aspm_check_latency()
412 * a L1 substate exit latency check. We assume that the in pcie_aspm_check_latency()
416 latency = max_t(u32, latency_up_l1, latency_dw_l1); in pcie_aspm_check_latency()
417 if ((link->aspm_capable & ASPM_STATE_L1) && in pcie_aspm_check_latency()
418 (latency + l1_switch_latency > acceptable_l1)) in pcie_aspm_check_latency()
419 link->aspm_capable &= ~ASPM_STATE_L1; in pcie_aspm_check_latency()
422 link = link->parent; in pcie_aspm_check_latency()
441 struct pci_dev *child = link->downstream, *parent = link->pdev; in aspm_calc_l12_info()
472 * least that much latency. in aspm_calc_l12_info()
474 * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and in aspm_calc_l12_info()
475 * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at in aspm_calc_l12_info()
483 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1); in aspm_calc_l12_info()
484 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2); in aspm_calc_l12_info()
485 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1); in aspm_calc_l12_info()
486 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2); in aspm_calc_l12_info()
497 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
499 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
504 pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2); in aspm_calc_l12_info()
505 pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2); in aspm_calc_l12_info()
508 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
512 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
515 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_calc_l12_info()
520 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0, in aspm_calc_l12_info()
522 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0, in aspm_calc_l12_info()
529 struct pci_dev *child = link->downstream, *parent = link->pdev; in aspm_l1ss_init()
533 if (!parent->l1ss || !child->l1ss) in aspm_l1ss_init()
537 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP, in aspm_l1ss_init()
539 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP, in aspm_l1ss_init()
552 if (!child->ltr_path) in aspm_l1ss_init()
556 link->aspm_support |= ASPM_STATE_L1_1; in aspm_l1ss_init()
558 link->aspm_support |= ASPM_STATE_L1_2; in aspm_l1ss_init()
560 link->aspm_support |= ASPM_STATE_L1_1_PCIPM; in aspm_l1ss_init()
562 link->aspm_support |= ASPM_STATE_L1_2_PCIPM; in aspm_l1ss_init()
565 pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in aspm_l1ss_init()
568 pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, in aspm_l1ss_init()
572 link->aspm_enabled |= ASPM_STATE_L1_1; in aspm_l1ss_init()
574 link->aspm_enabled |= ASPM_STATE_L1_2; in aspm_l1ss_init()
576 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; in aspm_l1ss_init()
578 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; in aspm_l1ss_init()
580 if (link->aspm_support & ASPM_STATE_L1_2_MASK) in aspm_l1ss_init()
586 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_aspm_cap_init()
589 struct pci_bus *linkbus = parent->subordinate; in pcie_aspm_cap_init()
593 link->aspm_enabled = ASPM_STATE_ALL; in pcie_aspm_cap_init()
594 link->aspm_disable = ASPM_STATE_ALL; in pcie_aspm_cap_init()
611 * Re-read upstream/downstream components' register state after in pcie_aspm_cap_init()
613 * read-only Link Capabilities may change depending on common clock in pcie_aspm_cap_init()
629 link->aspm_support |= ASPM_STATE_L0S; in pcie_aspm_cap_init()
632 link->aspm_enabled |= ASPM_STATE_L0S_UP; in pcie_aspm_cap_init()
634 link->aspm_enabled |= ASPM_STATE_L0S_DW; in pcie_aspm_cap_init()
638 link->aspm_support |= ASPM_STATE_L1; in pcie_aspm_cap_init()
641 link->aspm_enabled |= ASPM_STATE_L1; in pcie_aspm_cap_init()
646 link->aspm_default = link->aspm_enabled; in pcie_aspm_cap_init()
649 link->aspm_capable = link->aspm_support; in pcie_aspm_cap_init()
652 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_aspm_cap_init()
665 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_config_aspm_l1ss()
667 enable_req = (link->aspm_enabled ^ state) & state; in pcie_config_aspm_l1ss()
671 * - When enabling L1.x, enable bit at parent first, then at child in pcie_config_aspm_l1ss()
672 * - When disabling L1.x, disable bit at child first, then at parent in pcie_config_aspm_l1ss()
673 * - When enabling ASPM L1.x, need to disable L1 in pcie_config_aspm_l1ss()
675 * - The ASPM/PCIPM L1.2 must be disabled while programming timing in pcie_config_aspm_l1ss()
683 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
685 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
709 pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
711 pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, in pcie_config_aspm_l1ss()
724 struct pci_dev *child = link->downstream, *parent = link->pdev; in pcie_config_aspm_link()
725 struct pci_bus *linkbus = parent->subordinate; in pcie_config_aspm_link()
728 state &= (link->aspm_capable & ~link->aspm_disable); in pcie_config_aspm_link()
735 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { in pcie_config_aspm_link()
737 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); in pcie_config_aspm_link()
741 if (link->aspm_enabled == state) in pcie_config_aspm_link()
753 if (link->aspm_capable & ASPM_STATE_L1SS) in pcie_config_aspm_link()
764 list_for_each_entry(child, &linkbus->devices, bus_list) in pcie_config_aspm_link()
769 link->aspm_enabled = state; in pcie_config_aspm_link()
776 link = link->parent; in pcie_config_aspm_path()
782 link->pdev->link_state = NULL; in free_link_state()
795 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { in pcie_aspm_sanity_check()
797 return -EINVAL; in pcie_aspm_sanity_check()
802 * pre-1.1 device in pcie_aspm_sanity_check()
809 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use in pcie_aspm_sanity_check()
814 …pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\… in pcie_aspm_sanity_check()
815 return -EINVAL; in pcie_aspm_sanity_check()
829 INIT_LIST_HEAD(&link->sibling); in alloc_pcie_link_state()
830 link->pdev = pdev; in alloc_pcie_link_state()
831 link->downstream = pci_function_0(pdev->subordinate); in alloc_pcie_link_state()
834 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe in alloc_pcie_link_state()
842 !pdev->bus->parent->self) { in alloc_pcie_link_state()
843 link->root = link; in alloc_pcie_link_state()
847 parent = pdev->bus->parent->self->link_state; in alloc_pcie_link_state()
853 link->parent = parent; in alloc_pcie_link_state()
854 link->root = link->parent->root; in alloc_pcie_link_state()
857 list_add(&link->sibling, &link_list); in alloc_pcie_link_state()
858 pdev->link_state = link; in alloc_pcie_link_state()
866 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) in pcie_aspm_update_sysfs_visibility()
867 sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group); in pcie_aspm_update_sysfs_visibility()
883 if (pdev->link_state) in pcie_aspm_init_link_state()
896 pdev->bus->self) in pcie_aspm_init_link_state()
900 if (list_empty(&pdev->subordinate->devices)) in pcie_aspm_init_link_state()
943 BUG_ON(root->parent); in pcie_update_aspm_capable()
945 if (link->root != root) in pcie_update_aspm_capable()
947 link->aspm_capable = link->aspm_support; in pcie_update_aspm_capable()
951 struct pci_bus *linkbus = link->pdev->subordinate; in pcie_update_aspm_capable()
952 if (link->root != root) in pcie_update_aspm_capable()
954 list_for_each_entry(child, &linkbus->devices, bus_list) { in pcie_update_aspm_capable()
966 struct pci_dev *parent = pdev->bus->self; in pcie_aspm_exit_link_state()
969 if (!parent || !parent->link_state) in pcie_aspm_exit_link_state()
975 link = parent->link_state; in pcie_aspm_exit_link_state()
976 root = link->root; in pcie_aspm_exit_link_state()
977 parent_link = link->parent; in pcie_aspm_exit_link_state()
980 * link->downstream is a pointer to the pci_dev of function 0. If in pcie_aspm_exit_link_state()
982 * so we can't use link->downstream again. Free the link state to in pcie_aspm_exit_link_state()
985 * If we're removing a non-0 function, it's possible we could in pcie_aspm_exit_link_state()
988 * multi-function devices, so disable ASPM for all of them. in pcie_aspm_exit_link_state()
991 list_del(&link->sibling); in pcie_aspm_exit_link_state()
1010 struct pcie_link_state *link = pdev->link_state; in pcie_aspm_pm_state_change()
1015 * Devices changed PM state, we should recheck if latency in pcie_aspm_pm_state_change()
1021 pcie_update_aspm_capable(link->root); in pcie_aspm_pm_state_change()
1030 struct pcie_link_state *link = pdev->link_state; in pcie_aspm_powersave_config_link()
1058 return bridge->link_state; in pcie_aspm_get_link()
1066 return -EINVAL; in __pci_disable_link_state()
1077 return -EPERM; in __pci_disable_link_state()
1084 link->aspm_disable |= ASPM_STATE_L0S; in __pci_disable_link_state()
1087 link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS; in __pci_disable_link_state()
1089 link->aspm_disable |= ASPM_STATE_L1_1; in __pci_disable_link_state()
1091 link->aspm_disable |= ASPM_STATE_L1_2; in __pci_disable_link_state()
1093 link->aspm_disable |= ASPM_STATE_L1_1_PCIPM; in __pci_disable_link_state()
1095 link->aspm_disable |= ASPM_STATE_L1_2_PCIPM; in __pci_disable_link_state()
1099 link->clkpm_disable = 1; in __pci_disable_link_state()
1115 * pci_disable_link_state - Disable device's link state, so the link will
1134 return -EINVAL; in __pci_enable_link_state()
1143 return -EPERM; in __pci_enable_link_state()
1149 link->aspm_default = 0; in __pci_enable_link_state()
1151 link->aspm_default |= ASPM_STATE_L0S; in __pci_enable_link_state()
1153 link->aspm_default |= ASPM_STATE_L1; in __pci_enable_link_state()
1156 link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1; in __pci_enable_link_state()
1158 link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1; in __pci_enable_link_state()
1160 link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1; in __pci_enable_link_state()
1162 link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1; in __pci_enable_link_state()
1165 link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0; in __pci_enable_link_state()
1175 * pci_enable_link_state - Clear and set the default device link state so that
1191 * pci_enable_link_state_locked - Clear and set the default device link state
1200 * Context: Caller holds pci_bus_sem read lock.
1217 return -EPERM; in pcie_aspm_set_policy()
1252 * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1267 return link->aspm_enabled; in pcie_aspm_enabled()
1278 return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0); in aspm_attr_show_common()
1290 return -EINVAL; in aspm_attr_store_common()
1296 link->aspm_disable &= ~state; in aspm_attr_store_common()
1299 link->aspm_disable &= ~ASPM_STATE_L1; in aspm_attr_store_common()
1301 link->aspm_disable |= state; in aspm_attr_store_common()
1303 link->aspm_disable |= ASPM_STATE_L1SS; in aspm_attr_store_common()
1337 return sysfs_emit(buf, "%d\n", link->clkpm_enabled); in ASPM_ATTR()
1349 return -EINVAL; in clkpm_store()
1354 link->clkpm_disable = !state_enable; in clkpm_store()
1401 return link->clkpm_capable ? a->mode : 0; in aspm_ctrl_attrs_are_visible()
1403 return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0; in aspm_ctrl_attrs_are_visible()