1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * File: drivers/pci/pcie/aspm.c 4 * Enabling PCIe link L0s/L1 state and Clock Power Management 5 * 6 * Copyright (C) 2007 Intel 7 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 8 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/pci.h> 15 #include <linux/pci_regs.h> 16 #include <linux/errno.h> 17 #include <linux/pm.h> 18 #include <linux/init.h> 19 #include <linux/slab.h> 20 #include <linux/jiffies.h> 21 #include <linux/delay.h> 22 #include <linux/pci-aspm.h> 23 #include "../pci.h" 24 25 #ifdef MODULE_PARAM_PREFIX 26 #undef MODULE_PARAM_PREFIX 27 #endif 28 #define MODULE_PARAM_PREFIX "pcie_aspm." 29 30 /* Note: those are not register definitions */ 31 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 32 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 33 #define ASPM_STATE_L1 (4) /* L1 state */ 34 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ 35 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ 36 #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ 37 #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ 38 #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) 39 #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) 40 #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ 41 ASPM_STATE_L1_2_MASK) 42 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 43 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ 44 ASPM_STATE_L1SS) 45 46 /* 47 * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter 48 * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details). 49 * Not sure is there is a way to "calculate" this on the fly, but maybe we 50 * could turn it into a parameter in future. This value has been taken from 51 * the following files from Intel's coreboot (which is the only code I found 52 * to have used this): 53 * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html 54 * https://review.coreboot.org/#/c/8832/ 55 */ 56 #define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30)) 57 58 struct aspm_latency { 59 u32 l0s; /* L0s latency (nsec) */ 60 u32 l1; /* L1 latency (nsec) */ 61 }; 62 63 struct pcie_link_state { 64 struct pci_dev *pdev; /* Upstream component of the Link */ 65 struct pci_dev *downstream; /* Downstream component, function 0 */ 66 struct pcie_link_state *root; /* pointer to the root port link */ 67 struct pcie_link_state *parent; /* pointer to the parent Link state */ 68 struct list_head sibling; /* node in link_list */ 69 struct list_head children; /* list of child link states */ 70 struct list_head link; /* node in parent's children list */ 71 72 /* ASPM state */ 73 u32 aspm_support:7; /* Supported ASPM state */ 74 u32 aspm_enabled:7; /* Enabled ASPM state */ 75 u32 aspm_capable:7; /* Capable ASPM state with latency */ 76 u32 aspm_default:7; /* Default ASPM state by BIOS */ 77 u32 aspm_disable:7; /* Disabled ASPM state */ 78 79 /* Clock PM state */ 80 u32 clkpm_capable:1; /* Clock PM capable? */ 81 u32 clkpm_enabled:1; /* Current Clock PM state */ 82 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 83 84 /* Exit latencies */ 85 struct aspm_latency latency_up; /* Upstream direction exit latency */ 86 struct aspm_latency latency_dw; /* Downstream direction exit latency */ 87 /* 88 * Endpoint acceptable latencies. A pcie downstream port only 89 * has one slot under it, so at most there are 8 functions. 90 */ 91 struct aspm_latency acceptable[8]; 92 93 /* L1 PM Substate info */ 94 struct { 95 u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */ 96 u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */ 97 u32 ctl1; /* value to be programmed in ctl1 */ 98 u32 ctl2; /* value to be programmed in ctl2 */ 99 } l1ss; 100 }; 101 102 static int aspm_disabled, aspm_force; 103 static bool aspm_support_enabled = true; 104 static DEFINE_MUTEX(aspm_lock); 105 static LIST_HEAD(link_list); 106 107 #define POLICY_DEFAULT 0 /* BIOS default setting */ 108 #define POLICY_PERFORMANCE 1 /* high performance */ 109 #define POLICY_POWERSAVE 2 /* high power saving */ 110 #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */ 111 112 #ifdef CONFIG_PCIEASPM_PERFORMANCE 113 static int aspm_policy = POLICY_PERFORMANCE; 114 #elif defined CONFIG_PCIEASPM_POWERSAVE 115 static int aspm_policy = POLICY_POWERSAVE; 116 #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE 117 static int aspm_policy = POLICY_POWER_SUPERSAVE; 118 #else 119 static int aspm_policy; 120 #endif 121 122 static const char *policy_str[] = { 123 [POLICY_DEFAULT] = "default", 124 [POLICY_PERFORMANCE] = "performance", 125 [POLICY_POWERSAVE] = "powersave", 126 [POLICY_POWER_SUPERSAVE] = "powersupersave" 127 }; 128 129 #define LINK_RETRAIN_TIMEOUT HZ 130 131 static int policy_to_aspm_state(struct pcie_link_state *link) 132 { 133 switch (aspm_policy) { 134 case POLICY_PERFORMANCE: 135 /* Disable ASPM and Clock PM */ 136 return 0; 137 case POLICY_POWERSAVE: 138 /* Enable ASPM L0s/L1 */ 139 return (ASPM_STATE_L0S | ASPM_STATE_L1); 140 case POLICY_POWER_SUPERSAVE: 141 /* Enable Everything */ 142 return ASPM_STATE_ALL; 143 case POLICY_DEFAULT: 144 return link->aspm_default; 145 } 146 return 0; 147 } 148 149 static int policy_to_clkpm_state(struct pcie_link_state *link) 150 { 151 switch (aspm_policy) { 152 case POLICY_PERFORMANCE: 153 /* Disable ASPM and Clock PM */ 154 return 0; 155 case POLICY_POWERSAVE: 156 case POLICY_POWER_SUPERSAVE: 157 /* Enable Clock PM */ 158 return 1; 159 case POLICY_DEFAULT: 160 return link->clkpm_default; 161 } 162 return 0; 163 } 164 165 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 166 { 167 struct pci_dev *child; 168 struct pci_bus *linkbus = link->pdev->subordinate; 169 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0; 170 171 list_for_each_entry(child, &linkbus->devices, bus_list) 172 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 173 PCI_EXP_LNKCTL_CLKREQ_EN, 174 val); 175 link->clkpm_enabled = !!enable; 176 } 177 178 static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 179 { 180 /* Don't enable Clock PM if the link is not Clock PM capable */ 181 if (!link->clkpm_capable) 182 enable = 0; 183 /* Need nothing if the specified equals to current state */ 184 if (link->clkpm_enabled == enable) 185 return; 186 pcie_set_clkpm_nocheck(link, enable); 187 } 188 189 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 190 { 191 int capable = 1, enabled = 1; 192 u32 reg32; 193 u16 reg16; 194 struct pci_dev *child; 195 struct pci_bus *linkbus = link->pdev->subordinate; 196 197 /* All functions should have the same cap and state, take the worst */ 198 list_for_each_entry(child, &linkbus->devices, bus_list) { 199 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); 200 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 201 capable = 0; 202 enabled = 0; 203 break; 204 } 205 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 206 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 207 enabled = 0; 208 } 209 link->clkpm_enabled = enabled; 210 link->clkpm_default = enabled; 211 link->clkpm_capable = (blacklist) ? 0 : capable; 212 } 213 214 /* 215 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 216 * could use common clock. If they are, configure them to use the 217 * common clock. That will reduce the ASPM state exit latency. 218 */ 219 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 220 { 221 int same_clock = 1; 222 u16 reg16, parent_reg, child_reg[8]; 223 unsigned long start_jiffies; 224 struct pci_dev *child, *parent = link->pdev; 225 struct pci_bus *linkbus = parent->subordinate; 226 /* 227 * All functions of a slot should have the same Slot Clock 228 * Configuration, so just check one function 229 */ 230 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 231 BUG_ON(!pci_is_pcie(child)); 232 233 /* Check downstream component if bit Slot Clock Configuration is 1 */ 234 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); 235 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 236 same_clock = 0; 237 238 /* Check upstream component if bit Slot Clock Configuration is 1 */ 239 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 240 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 241 same_clock = 0; 242 243 /* Configure downstream component, all functions */ 244 list_for_each_entry(child, &linkbus->devices, bus_list) { 245 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 246 child_reg[PCI_FUNC(child->devfn)] = reg16; 247 if (same_clock) 248 reg16 |= PCI_EXP_LNKCTL_CCC; 249 else 250 reg16 &= ~PCI_EXP_LNKCTL_CCC; 251 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); 252 } 253 254 /* Configure upstream component */ 255 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); 256 parent_reg = reg16; 257 if (same_clock) 258 reg16 |= PCI_EXP_LNKCTL_CCC; 259 else 260 reg16 &= ~PCI_EXP_LNKCTL_CCC; 261 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 262 263 /* Retrain link */ 264 reg16 |= PCI_EXP_LNKCTL_RL; 265 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 266 267 /* Wait for link training end. Break out after waiting for timeout */ 268 start_jiffies = jiffies; 269 for (;;) { 270 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 271 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 272 break; 273 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 274 break; 275 msleep(1); 276 } 277 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 278 return; 279 280 /* Training failed. Restore common clock configurations */ 281 pci_err(parent, "ASPM: Could not configure common clock\n"); 282 list_for_each_entry(child, &linkbus->devices, bus_list) 283 pcie_capability_write_word(child, PCI_EXP_LNKCTL, 284 child_reg[PCI_FUNC(child->devfn)]); 285 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); 286 } 287 288 /* Convert L0s latency encoding to ns */ 289 static u32 calc_l0s_latency(u32 encoding) 290 { 291 if (encoding == 0x7) 292 return (5 * 1000); /* > 4us */ 293 return (64 << encoding); 294 } 295 296 /* Convert L0s acceptable latency encoding to ns */ 297 static u32 calc_l0s_acceptable(u32 encoding) 298 { 299 if (encoding == 0x7) 300 return -1U; 301 return (64 << encoding); 302 } 303 304 /* Convert L1 latency encoding to ns */ 305 static u32 calc_l1_latency(u32 encoding) 306 { 307 if (encoding == 0x7) 308 return (65 * 1000); /* > 64us */ 309 return (1000 << encoding); 310 } 311 312 /* Convert L1 acceptable latency encoding to ns */ 313 static u32 calc_l1_acceptable(u32 encoding) 314 { 315 if (encoding == 0x7) 316 return -1U; 317 return (1000 << encoding); 318 } 319 320 /* Convert L1SS T_pwr encoding to usec */ 321 static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) 322 { 323 switch (scale) { 324 case 0: 325 return val * 2; 326 case 1: 327 return val * 10; 328 case 2: 329 return val * 100; 330 } 331 pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale); 332 return 0; 333 } 334 335 struct aspm_register_info { 336 u32 support:2; 337 u32 enabled:2; 338 u32 latency_encoding_l0s; 339 u32 latency_encoding_l1; 340 341 /* L1 substates */ 342 u32 l1ss_cap_ptr; 343 u32 l1ss_cap; 344 u32 l1ss_ctl1; 345 u32 l1ss_ctl2; 346 }; 347 348 static void pcie_get_aspm_reg(struct pci_dev *pdev, 349 struct aspm_register_info *info) 350 { 351 u16 reg16; 352 u32 reg32; 353 354 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); 355 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 356 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 357 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 358 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); 359 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 360 361 /* Read L1 PM substate capabilities */ 362 info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0; 363 info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); 364 if (!info->l1ss_cap_ptr) 365 return; 366 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP, 367 &info->l1ss_cap); 368 if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) { 369 info->l1ss_cap = 0; 370 return; 371 } 372 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1, 373 &info->l1ss_ctl1); 374 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2, 375 &info->l1ss_ctl2); 376 } 377 378 static void pcie_aspm_check_latency(struct pci_dev *endpoint) 379 { 380 u32 latency, l1_switch_latency = 0; 381 struct aspm_latency *acceptable; 382 struct pcie_link_state *link; 383 384 /* Device not in D0 doesn't need latency check */ 385 if ((endpoint->current_state != PCI_D0) && 386 (endpoint->current_state != PCI_UNKNOWN)) 387 return; 388 389 link = endpoint->bus->self->link_state; 390 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 391 392 while (link) { 393 /* Check upstream direction L0s latency */ 394 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 395 (link->latency_up.l0s > acceptable->l0s)) 396 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 397 398 /* Check downstream direction L0s latency */ 399 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 400 (link->latency_dw.l0s > acceptable->l0s)) 401 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 402 /* 403 * Check L1 latency. 404 * Every switch on the path to root complex need 1 405 * more microsecond for L1. Spec doesn't mention L0s. 406 * 407 * The exit latencies for L1 substates are not advertised 408 * by a device. Since the spec also doesn't mention a way 409 * to determine max latencies introduced by enabling L1 410 * substates on the components, it is not clear how to do 411 * a L1 substate exit latency check. We assume that the 412 * L1 exit latencies advertised by a device include L1 413 * substate latencies (and hence do not do any check). 414 */ 415 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 416 if ((link->aspm_capable & ASPM_STATE_L1) && 417 (latency + l1_switch_latency > acceptable->l1)) 418 link->aspm_capable &= ~ASPM_STATE_L1; 419 l1_switch_latency += 1000; 420 421 link = link->parent; 422 } 423 } 424 425 /* 426 * The L1 PM substate capability is only implemented in function 0 in a 427 * multi function device. 428 */ 429 static struct pci_dev *pci_function_0(struct pci_bus *linkbus) 430 { 431 struct pci_dev *child; 432 433 list_for_each_entry(child, &linkbus->devices, bus_list) 434 if (PCI_FUNC(child->devfn) == 0) 435 return child; 436 return NULL; 437 } 438 439 /* Calculate L1.2 PM substate timing parameters */ 440 static void aspm_calc_l1ss_info(struct pcie_link_state *link, 441 struct aspm_register_info *upreg, 442 struct aspm_register_info *dwreg) 443 { 444 u32 val1, val2, scale1, scale2; 445 446 link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr; 447 link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr; 448 link->l1ss.ctl1 = link->l1ss.ctl2 = 0; 449 450 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) 451 return; 452 453 /* Choose the greater of the two Port Common_Mode_Restore_Times */ 454 val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; 455 val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8; 456 if (val1 > val2) 457 link->l1ss.ctl1 |= val1 << 8; 458 else 459 link->l1ss.ctl1 |= val2 << 8; 460 461 /* 462 * We currently use LTR L1.2 threshold to be fixed constant picked from 463 * Intel's coreboot. 464 */ 465 link->l1ss.ctl1 |= LTR_L1_2_THRESHOLD_BITS; 466 467 /* Choose the greater of the two Port T_POWER_ON times */ 468 val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; 469 scale1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; 470 val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19; 471 scale2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16; 472 473 if (calc_l1ss_pwron(link->pdev, scale1, val1) > 474 calc_l1ss_pwron(link->downstream, scale2, val2)) 475 link->l1ss.ctl2 |= scale1 | (val1 << 3); 476 else 477 link->l1ss.ctl2 |= scale2 | (val2 << 3); 478 } 479 480 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 481 { 482 struct pci_dev *child = link->downstream, *parent = link->pdev; 483 struct pci_bus *linkbus = parent->subordinate; 484 struct aspm_register_info upreg, dwreg; 485 486 if (blacklist) { 487 /* Set enabled/disable so that we will disable ASPM later */ 488 link->aspm_enabled = ASPM_STATE_ALL; 489 link->aspm_disable = ASPM_STATE_ALL; 490 return; 491 } 492 493 /* Get upstream/downstream components' register state */ 494 pcie_get_aspm_reg(parent, &upreg); 495 pcie_get_aspm_reg(child, &dwreg); 496 497 /* 498 * If ASPM not supported, don't mess with the clocks and link, 499 * bail out now. 500 */ 501 if (!(upreg.support & dwreg.support)) 502 return; 503 504 /* Configure common clock before checking latencies */ 505 pcie_aspm_configure_common_clock(link); 506 507 /* 508 * Re-read upstream/downstream components' register state 509 * after clock configuration 510 */ 511 pcie_get_aspm_reg(parent, &upreg); 512 pcie_get_aspm_reg(child, &dwreg); 513 514 /* 515 * Setup L0s state 516 * 517 * Note that we must not enable L0s in either direction on a 518 * given link unless components on both sides of the link each 519 * support L0s. 520 */ 521 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 522 link->aspm_support |= ASPM_STATE_L0S; 523 if (dwreg.enabled & PCIE_LINK_STATE_L0S) 524 link->aspm_enabled |= ASPM_STATE_L0S_UP; 525 if (upreg.enabled & PCIE_LINK_STATE_L0S) 526 link->aspm_enabled |= ASPM_STATE_L0S_DW; 527 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 528 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 529 530 /* Setup L1 state */ 531 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 532 link->aspm_support |= ASPM_STATE_L1; 533 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 534 link->aspm_enabled |= ASPM_STATE_L1; 535 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 536 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 537 538 /* Setup L1 substate */ 539 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) 540 link->aspm_support |= ASPM_STATE_L1_1; 541 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) 542 link->aspm_support |= ASPM_STATE_L1_2; 543 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) 544 link->aspm_support |= ASPM_STATE_L1_1_PCIPM; 545 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) 546 link->aspm_support |= ASPM_STATE_L1_2_PCIPM; 547 548 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) 549 link->aspm_enabled |= ASPM_STATE_L1_1; 550 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) 551 link->aspm_enabled |= ASPM_STATE_L1_2; 552 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) 553 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; 554 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) 555 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; 556 557 if (link->aspm_support & ASPM_STATE_L1SS) 558 aspm_calc_l1ss_info(link, &upreg, &dwreg); 559 560 /* Save default state */ 561 link->aspm_default = link->aspm_enabled; 562 563 /* Setup initial capable state. Will be updated later */ 564 link->aspm_capable = link->aspm_support; 565 /* 566 * If the downstream component has pci bridge function, don't 567 * do ASPM for now. 568 */ 569 list_for_each_entry(child, &linkbus->devices, bus_list) { 570 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { 571 link->aspm_disable = ASPM_STATE_ALL; 572 break; 573 } 574 } 575 576 /* Get and check endpoint acceptable latencies */ 577 list_for_each_entry(child, &linkbus->devices, bus_list) { 578 u32 reg32, encoding; 579 struct aspm_latency *acceptable = 580 &link->acceptable[PCI_FUNC(child->devfn)]; 581 582 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && 583 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) 584 continue; 585 586 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 587 /* Calculate endpoint L0s acceptable latency */ 588 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 589 acceptable->l0s = calc_l0s_acceptable(encoding); 590 /* Calculate endpoint L1 acceptable latency */ 591 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 592 acceptable->l1 = calc_l1_acceptable(encoding); 593 594 pcie_aspm_check_latency(child); 595 } 596 } 597 598 static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, 599 u32 clear, u32 set) 600 { 601 u32 val; 602 603 pci_read_config_dword(pdev, pos, &val); 604 val &= ~clear; 605 val |= set; 606 pci_write_config_dword(pdev, pos, val); 607 } 608 609 /* Configure the ASPM L1 substates */ 610 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) 611 { 612 u32 val, enable_req; 613 struct pci_dev *child = link->downstream, *parent = link->pdev; 614 u32 up_cap_ptr = link->l1ss.up_cap_ptr; 615 u32 dw_cap_ptr = link->l1ss.dw_cap_ptr; 616 617 enable_req = (link->aspm_enabled ^ state) & state; 618 619 /* 620 * Here are the rules specified in the PCIe spec for enabling L1SS: 621 * - When enabling L1.x, enable bit at parent first, then at child 622 * - When disabling L1.x, disable bit at child first, then at parent 623 * - When enabling ASPM L1.x, need to disable L1 624 * (at child followed by parent). 625 * - The ASPM/PCIPM L1.2 must be disabled while programming timing 626 * parameters 627 * 628 * To keep it simple, disable all L1SS bits first, and later enable 629 * what is needed. 630 */ 631 632 /* Disable all L1 substates */ 633 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 634 PCI_L1SS_CTL1_L1SS_MASK, 0); 635 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 636 PCI_L1SS_CTL1_L1SS_MASK, 0); 637 /* 638 * If needed, disable L1, and it gets enabled later 639 * in pcie_config_aspm_link(). 640 */ 641 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { 642 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 643 PCI_EXP_LNKCTL_ASPM_L1, 0); 644 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 645 PCI_EXP_LNKCTL_ASPM_L1, 0); 646 } 647 648 if (enable_req & ASPM_STATE_L1_2_MASK) { 649 650 /* Program T_POWER_ON times in both ports */ 651 pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2, 652 link->l1ss.ctl2); 653 pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2, 654 link->l1ss.ctl2); 655 656 /* Program Common_Mode_Restore_Time in upstream device */ 657 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 658 PCI_L1SS_CTL1_CM_RESTORE_TIME, 659 link->l1ss.ctl1); 660 661 /* Program LTR_L1.2_THRESHOLD time in both ports */ 662 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 663 PCI_L1SS_CTL1_LTR_L12_TH_VALUE | 664 PCI_L1SS_CTL1_LTR_L12_TH_SCALE, 665 link->l1ss.ctl1); 666 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 667 PCI_L1SS_CTL1_LTR_L12_TH_VALUE | 668 PCI_L1SS_CTL1_LTR_L12_TH_SCALE, 669 link->l1ss.ctl1); 670 } 671 672 val = 0; 673 if (state & ASPM_STATE_L1_1) 674 val |= PCI_L1SS_CTL1_ASPM_L1_1; 675 if (state & ASPM_STATE_L1_2) 676 val |= PCI_L1SS_CTL1_ASPM_L1_2; 677 if (state & ASPM_STATE_L1_1_PCIPM) 678 val |= PCI_L1SS_CTL1_PCIPM_L1_1; 679 if (state & ASPM_STATE_L1_2_PCIPM) 680 val |= PCI_L1SS_CTL1_PCIPM_L1_2; 681 682 /* Enable what we need to enable */ 683 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 684 PCI_L1SS_CAP_L1_PM_SS, val); 685 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 686 PCI_L1SS_CAP_L1_PM_SS, val); 687 } 688 689 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 690 { 691 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, 692 PCI_EXP_LNKCTL_ASPMC, val); 693 } 694 695 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 696 { 697 u32 upstream = 0, dwstream = 0; 698 struct pci_dev *child = link->downstream, *parent = link->pdev; 699 struct pci_bus *linkbus = parent->subordinate; 700 701 /* Enable only the states that were not explicitly disabled */ 702 state &= (link->aspm_capable & ~link->aspm_disable); 703 704 /* Can't enable any substates if L1 is not enabled */ 705 if (!(state & ASPM_STATE_L1)) 706 state &= ~ASPM_STATE_L1SS; 707 708 /* Spec says both ports must be in D0 before enabling PCI PM substates*/ 709 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { 710 state &= ~ASPM_STATE_L1_SS_PCIPM; 711 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); 712 } 713 714 /* Nothing to do if the link is already in the requested state */ 715 if (link->aspm_enabled == state) 716 return; 717 /* Convert ASPM state to upstream/downstream ASPM register state */ 718 if (state & ASPM_STATE_L0S_UP) 719 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; 720 if (state & ASPM_STATE_L0S_DW) 721 upstream |= PCI_EXP_LNKCTL_ASPM_L0S; 722 if (state & ASPM_STATE_L1) { 723 upstream |= PCI_EXP_LNKCTL_ASPM_L1; 724 dwstream |= PCI_EXP_LNKCTL_ASPM_L1; 725 } 726 727 if (link->aspm_capable & ASPM_STATE_L1SS) 728 pcie_config_aspm_l1ss(link, state); 729 730 /* 731 * Spec 2.0 suggests all functions should be configured the 732 * same setting for ASPM. Enabling ASPM L1 should be done in 733 * upstream component first and then downstream, and vice 734 * versa for disabling ASPM L1. Spec doesn't mention L0S. 735 */ 736 if (state & ASPM_STATE_L1) 737 pcie_config_aspm_dev(parent, upstream); 738 list_for_each_entry(child, &linkbus->devices, bus_list) 739 pcie_config_aspm_dev(child, dwstream); 740 if (!(state & ASPM_STATE_L1)) 741 pcie_config_aspm_dev(parent, upstream); 742 743 link->aspm_enabled = state; 744 } 745 746 static void pcie_config_aspm_path(struct pcie_link_state *link) 747 { 748 while (link) { 749 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 750 link = link->parent; 751 } 752 } 753 754 static void free_link_state(struct pcie_link_state *link) 755 { 756 link->pdev->link_state = NULL; 757 kfree(link); 758 } 759 760 static int pcie_aspm_sanity_check(struct pci_dev *pdev) 761 { 762 struct pci_dev *child; 763 u32 reg32; 764 765 /* 766 * Some functions in a slot might not all be PCIe functions, 767 * very strange. Disable ASPM for the whole slot 768 */ 769 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 770 if (!pci_is_pcie(child)) 771 return -EINVAL; 772 773 /* 774 * If ASPM is disabled then we're not going to change 775 * the BIOS state. It's safe to continue even if it's a 776 * pre-1.1 device 777 */ 778 779 if (aspm_disabled) 780 continue; 781 782 /* 783 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 784 * RBER bit to determine if a function is 1.1 version device 785 */ 786 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 787 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 788 pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); 789 return -EINVAL; 790 } 791 } 792 return 0; 793 } 794 795 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 796 { 797 struct pcie_link_state *link; 798 799 link = kzalloc(sizeof(*link), GFP_KERNEL); 800 if (!link) 801 return NULL; 802 803 INIT_LIST_HEAD(&link->sibling); 804 INIT_LIST_HEAD(&link->children); 805 INIT_LIST_HEAD(&link->link); 806 link->pdev = pdev; 807 link->downstream = pci_function_0(pdev->subordinate); 808 809 /* 810 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe 811 * hierarchies. Note that some PCIe host implementations omit 812 * the root ports entirely, in which case a downstream port on 813 * a switch may become the root of the link state chain for all 814 * its subordinate endpoints. 815 */ 816 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || 817 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE || 818 !pdev->bus->parent->self) { 819 link->root = link; 820 } else { 821 struct pcie_link_state *parent; 822 823 parent = pdev->bus->parent->self->link_state; 824 if (!parent) { 825 kfree(link); 826 return NULL; 827 } 828 829 link->parent = parent; 830 link->root = link->parent->root; 831 list_add(&link->link, &parent->children); 832 } 833 834 list_add(&link->sibling, &link_list); 835 pdev->link_state = link; 836 return link; 837 } 838 839 /* 840 * pcie_aspm_init_link_state: Initiate PCI express link state. 841 * It is called after the pcie and its children devices are scanned. 842 * @pdev: the root port or switch downstream port 843 */ 844 void pcie_aspm_init_link_state(struct pci_dev *pdev) 845 { 846 struct pcie_link_state *link; 847 int blacklist = !!pcie_aspm_sanity_check(pdev); 848 849 if (!aspm_support_enabled) 850 return; 851 852 if (pdev->link_state) 853 return; 854 855 /* 856 * We allocate pcie_link_state for the component on the upstream 857 * end of a Link, so there's nothing to do unless this device has a 858 * Link on its secondary side. 859 */ 860 if (!pdev->has_secondary_link) 861 return; 862 863 /* VIA has a strange chipset, root port is under a bridge */ 864 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && 865 pdev->bus->self) 866 return; 867 868 down_read(&pci_bus_sem); 869 if (list_empty(&pdev->subordinate->devices)) 870 goto out; 871 872 mutex_lock(&aspm_lock); 873 link = alloc_pcie_link_state(pdev); 874 if (!link) 875 goto unlock; 876 /* 877 * Setup initial ASPM state. Note that we need to configure 878 * upstream links also because capable state of them can be 879 * update through pcie_aspm_cap_init(). 880 */ 881 pcie_aspm_cap_init(link, blacklist); 882 883 /* Setup initial Clock PM state */ 884 pcie_clkpm_cap_init(link, blacklist); 885 886 /* 887 * At this stage drivers haven't had an opportunity to change the 888 * link policy setting. Enabling ASPM on broken hardware can cripple 889 * it even before the driver has had a chance to disable ASPM, so 890 * default to a safe level right now. If we're enabling ASPM beyond 891 * the BIOS's expectation, we'll do so once pci_enable_device() is 892 * called. 893 */ 894 if (aspm_policy != POLICY_POWERSAVE && 895 aspm_policy != POLICY_POWER_SUPERSAVE) { 896 pcie_config_aspm_path(link); 897 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 898 } 899 900 unlock: 901 mutex_unlock(&aspm_lock); 902 out: 903 up_read(&pci_bus_sem); 904 } 905 906 /* Recheck latencies and update aspm_capable for links under the root */ 907 static void pcie_update_aspm_capable(struct pcie_link_state *root) 908 { 909 struct pcie_link_state *link; 910 BUG_ON(root->parent); 911 list_for_each_entry(link, &link_list, sibling) { 912 if (link->root != root) 913 continue; 914 link->aspm_capable = link->aspm_support; 915 } 916 list_for_each_entry(link, &link_list, sibling) { 917 struct pci_dev *child; 918 struct pci_bus *linkbus = link->pdev->subordinate; 919 if (link->root != root) 920 continue; 921 list_for_each_entry(child, &linkbus->devices, bus_list) { 922 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && 923 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) 924 continue; 925 pcie_aspm_check_latency(child); 926 } 927 } 928 } 929 930 /* @pdev: the endpoint device */ 931 void pcie_aspm_exit_link_state(struct pci_dev *pdev) 932 { 933 struct pci_dev *parent = pdev->bus->self; 934 struct pcie_link_state *link, *root, *parent_link; 935 936 if (!parent || !parent->link_state) 937 return; 938 939 down_read(&pci_bus_sem); 940 mutex_lock(&aspm_lock); 941 /* 942 * All PCIe functions are in one slot, remove one function will remove 943 * the whole slot, so just wait until we are the last function left. 944 */ 945 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 946 goto out; 947 948 link = parent->link_state; 949 root = link->root; 950 parent_link = link->parent; 951 952 /* All functions are removed, so just disable ASPM for the link */ 953 pcie_config_aspm_link(link, 0); 954 list_del(&link->sibling); 955 list_del(&link->link); 956 /* Clock PM is for endpoint device */ 957 free_link_state(link); 958 959 /* Recheck latencies and configure upstream links */ 960 if (parent_link) { 961 pcie_update_aspm_capable(root); 962 pcie_config_aspm_path(parent_link); 963 } 964 out: 965 mutex_unlock(&aspm_lock); 966 up_read(&pci_bus_sem); 967 } 968 969 /* @pdev: the root port or switch downstream port */ 970 void pcie_aspm_pm_state_change(struct pci_dev *pdev) 971 { 972 struct pcie_link_state *link = pdev->link_state; 973 974 if (aspm_disabled || !link) 975 return; 976 /* 977 * Devices changed PM state, we should recheck if latency 978 * meets all functions' requirement 979 */ 980 down_read(&pci_bus_sem); 981 mutex_lock(&aspm_lock); 982 pcie_update_aspm_capable(link->root); 983 pcie_config_aspm_path(link); 984 mutex_unlock(&aspm_lock); 985 up_read(&pci_bus_sem); 986 } 987 988 void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 989 { 990 struct pcie_link_state *link = pdev->link_state; 991 992 if (aspm_disabled || !link) 993 return; 994 995 if (aspm_policy != POLICY_POWERSAVE && 996 aspm_policy != POLICY_POWER_SUPERSAVE) 997 return; 998 999 down_read(&pci_bus_sem); 1000 mutex_lock(&aspm_lock); 1001 pcie_config_aspm_path(link); 1002 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1003 mutex_unlock(&aspm_lock); 1004 up_read(&pci_bus_sem); 1005 } 1006 1007 static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) 1008 { 1009 struct pci_dev *parent = pdev->bus->self; 1010 struct pcie_link_state *link; 1011 1012 if (!pci_is_pcie(pdev)) 1013 return; 1014 1015 if (pdev->has_secondary_link) 1016 parent = pdev; 1017 if (!parent || !parent->link_state) 1018 return; 1019 1020 /* 1021 * A driver requested that ASPM be disabled on this device, but 1022 * if we don't have permission to manage ASPM (e.g., on ACPI 1023 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 1024 * the _OSC method), we can't honor that request. Windows has 1025 * a similar mechanism using "PciASPMOptOut", which is also 1026 * ignored in this situation. 1027 */ 1028 if (aspm_disabled) { 1029 pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n"); 1030 return; 1031 } 1032 1033 if (sem) 1034 down_read(&pci_bus_sem); 1035 mutex_lock(&aspm_lock); 1036 link = parent->link_state; 1037 if (state & PCIE_LINK_STATE_L0S) 1038 link->aspm_disable |= ASPM_STATE_L0S; 1039 if (state & PCIE_LINK_STATE_L1) 1040 link->aspm_disable |= ASPM_STATE_L1; 1041 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1042 1043 if (state & PCIE_LINK_STATE_CLKPM) { 1044 link->clkpm_capable = 0; 1045 pcie_set_clkpm(link, 0); 1046 } 1047 mutex_unlock(&aspm_lock); 1048 if (sem) 1049 up_read(&pci_bus_sem); 1050 } 1051 1052 void pci_disable_link_state_locked(struct pci_dev *pdev, int state) 1053 { 1054 __pci_disable_link_state(pdev, state, false); 1055 } 1056 EXPORT_SYMBOL(pci_disable_link_state_locked); 1057 1058 /** 1059 * pci_disable_link_state - Disable device's link state, so the link will 1060 * never enter specific states. Note that if the BIOS didn't grant ASPM 1061 * control to the OS, this does nothing because we can't touch the LNKCTL 1062 * register. 1063 * 1064 * @pdev: PCI device 1065 * @state: ASPM link state to disable 1066 */ 1067 void pci_disable_link_state(struct pci_dev *pdev, int state) 1068 { 1069 __pci_disable_link_state(pdev, state, true); 1070 } 1071 EXPORT_SYMBOL(pci_disable_link_state); 1072 1073 static int pcie_aspm_set_policy(const char *val, 1074 const struct kernel_param *kp) 1075 { 1076 int i; 1077 struct pcie_link_state *link; 1078 1079 if (aspm_disabled) 1080 return -EPERM; 1081 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1082 if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 1083 break; 1084 if (i >= ARRAY_SIZE(policy_str)) 1085 return -EINVAL; 1086 if (i == aspm_policy) 1087 return 0; 1088 1089 down_read(&pci_bus_sem); 1090 mutex_lock(&aspm_lock); 1091 aspm_policy = i; 1092 list_for_each_entry(link, &link_list, sibling) { 1093 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1094 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1095 } 1096 mutex_unlock(&aspm_lock); 1097 up_read(&pci_bus_sem); 1098 return 0; 1099 } 1100 1101 static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp) 1102 { 1103 int i, cnt = 0; 1104 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1105 if (i == aspm_policy) 1106 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 1107 else 1108 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 1109 return cnt; 1110 } 1111 1112 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 1113 NULL, 0644); 1114 1115 #ifdef CONFIG_PCIEASPM_DEBUG 1116 static ssize_t link_state_show(struct device *dev, 1117 struct device_attribute *attr, 1118 char *buf) 1119 { 1120 struct pci_dev *pci_device = to_pci_dev(dev); 1121 struct pcie_link_state *link_state = pci_device->link_state; 1122 1123 return sprintf(buf, "%d\n", link_state->aspm_enabled); 1124 } 1125 1126 static ssize_t link_state_store(struct device *dev, 1127 struct device_attribute *attr, 1128 const char *buf, 1129 size_t n) 1130 { 1131 struct pci_dev *pdev = to_pci_dev(dev); 1132 struct pcie_link_state *link, *root = pdev->link_state->root; 1133 u32 state; 1134 1135 if (aspm_disabled) 1136 return -EPERM; 1137 1138 if (kstrtouint(buf, 10, &state)) 1139 return -EINVAL; 1140 if ((state & ~ASPM_STATE_ALL) != 0) 1141 return -EINVAL; 1142 1143 down_read(&pci_bus_sem); 1144 mutex_lock(&aspm_lock); 1145 list_for_each_entry(link, &link_list, sibling) { 1146 if (link->root != root) 1147 continue; 1148 pcie_config_aspm_link(link, state); 1149 } 1150 mutex_unlock(&aspm_lock); 1151 up_read(&pci_bus_sem); 1152 return n; 1153 } 1154 1155 static ssize_t clk_ctl_show(struct device *dev, 1156 struct device_attribute *attr, 1157 char *buf) 1158 { 1159 struct pci_dev *pci_device = to_pci_dev(dev); 1160 struct pcie_link_state *link_state = pci_device->link_state; 1161 1162 return sprintf(buf, "%d\n", link_state->clkpm_enabled); 1163 } 1164 1165 static ssize_t clk_ctl_store(struct device *dev, 1166 struct device_attribute *attr, 1167 const char *buf, 1168 size_t n) 1169 { 1170 struct pci_dev *pdev = to_pci_dev(dev); 1171 bool state; 1172 1173 if (strtobool(buf, &state)) 1174 return -EINVAL; 1175 1176 down_read(&pci_bus_sem); 1177 mutex_lock(&aspm_lock); 1178 pcie_set_clkpm_nocheck(pdev->link_state, state); 1179 mutex_unlock(&aspm_lock); 1180 up_read(&pci_bus_sem); 1181 1182 return n; 1183 } 1184 1185 static DEVICE_ATTR_RW(link_state); 1186 static DEVICE_ATTR_RW(clk_ctl); 1187 1188 static char power_group[] = "power"; 1189 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 1190 { 1191 struct pcie_link_state *link_state = pdev->link_state; 1192 1193 if (!link_state) 1194 return; 1195 1196 if (link_state->aspm_support) 1197 sysfs_add_file_to_group(&pdev->dev.kobj, 1198 &dev_attr_link_state.attr, power_group); 1199 if (link_state->clkpm_capable) 1200 sysfs_add_file_to_group(&pdev->dev.kobj, 1201 &dev_attr_clk_ctl.attr, power_group); 1202 } 1203 1204 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 1205 { 1206 struct pcie_link_state *link_state = pdev->link_state; 1207 1208 if (!link_state) 1209 return; 1210 1211 if (link_state->aspm_support) 1212 sysfs_remove_file_from_group(&pdev->dev.kobj, 1213 &dev_attr_link_state.attr, power_group); 1214 if (link_state->clkpm_capable) 1215 sysfs_remove_file_from_group(&pdev->dev.kobj, 1216 &dev_attr_clk_ctl.attr, power_group); 1217 } 1218 #endif 1219 1220 static int __init pcie_aspm_disable(char *str) 1221 { 1222 if (!strcmp(str, "off")) { 1223 aspm_policy = POLICY_DEFAULT; 1224 aspm_disabled = 1; 1225 aspm_support_enabled = false; 1226 printk(KERN_INFO "PCIe ASPM is disabled\n"); 1227 } else if (!strcmp(str, "force")) { 1228 aspm_force = 1; 1229 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 1230 } 1231 return 1; 1232 } 1233 1234 __setup("pcie_aspm=", pcie_aspm_disable); 1235 1236 void pcie_no_aspm(void) 1237 { 1238 /* 1239 * Disabling ASPM is intended to prevent the kernel from modifying 1240 * existing hardware state, not to clear existing state. To that end: 1241 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 1242 * (b) prevent userspace from changing policy 1243 */ 1244 if (!aspm_force) { 1245 aspm_policy = POLICY_DEFAULT; 1246 aspm_disabled = 1; 1247 } 1248 } 1249 1250 bool pcie_aspm_support_enabled(void) 1251 { 1252 return aspm_support_enabled; 1253 } 1254 EXPORT_SYMBOL(pcie_aspm_support_enabled); 1255