1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * File: drivers/pci/pcie/aspm.c 4 * Enabling PCIe link L0s/L1 state and Clock Power Management 5 * 6 * Copyright (C) 2007 Intel 7 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com) 8 * Copyright (C) Shaohua Li (shaohua.li@intel.com) 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/pci.h> 15 #include <linux/pci_regs.h> 16 #include <linux/errno.h> 17 #include <linux/pm.h> 18 #include <linux/init.h> 19 #include <linux/slab.h> 20 #include <linux/jiffies.h> 21 #include <linux/delay.h> 22 #include <linux/pci-aspm.h> 23 #include "../pci.h" 24 25 #ifdef MODULE_PARAM_PREFIX 26 #undef MODULE_PARAM_PREFIX 27 #endif 28 #define MODULE_PARAM_PREFIX "pcie_aspm." 29 30 /* Note: those are not register definitions */ 31 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */ 32 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */ 33 #define ASPM_STATE_L1 (4) /* L1 state */ 34 #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */ 35 #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */ 36 #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */ 37 #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */ 38 #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM) 39 #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM) 40 #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\ 41 ASPM_STATE_L1_2_MASK) 42 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW) 43 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \ 44 ASPM_STATE_L1SS) 45 46 /* 47 * When L1 substates are enabled, the LTR L1.2 threshold is a timing parameter 48 * that decides whether L1.1 or L1.2 is entered (Refer PCIe spec for details). 49 * Not sure is there is a way to "calculate" this on the fly, but maybe we 50 * could turn it into a parameter in future. This value has been taken from 51 * the following files from Intel's coreboot (which is the only code I found 52 * to have used this): 53 * https://www.coreboot.org/pipermail/coreboot-gerrit/2015-March/021134.html 54 * https://review.coreboot.org/#/c/8832/ 55 */ 56 #define LTR_L1_2_THRESHOLD_BITS ((1 << 21) | (1 << 23) | (1 << 30)) 57 58 struct aspm_latency { 59 u32 l0s; /* L0s latency (nsec) */ 60 u32 l1; /* L1 latency (nsec) */ 61 }; 62 63 struct pcie_link_state { 64 struct pci_dev *pdev; /* Upstream component of the Link */ 65 struct pci_dev *downstream; /* Downstream component, function 0 */ 66 struct pcie_link_state *root; /* pointer to the root port link */ 67 struct pcie_link_state *parent; /* pointer to the parent Link state */ 68 struct list_head sibling; /* node in link_list */ 69 struct list_head children; /* list of child link states */ 70 struct list_head link; /* node in parent's children list */ 71 72 /* ASPM state */ 73 u32 aspm_support:7; /* Supported ASPM state */ 74 u32 aspm_enabled:7; /* Enabled ASPM state */ 75 u32 aspm_capable:7; /* Capable ASPM state with latency */ 76 u32 aspm_default:7; /* Default ASPM state by BIOS */ 77 u32 aspm_disable:7; /* Disabled ASPM state */ 78 79 /* Clock PM state */ 80 u32 clkpm_capable:1; /* Clock PM capable? */ 81 u32 clkpm_enabled:1; /* Current Clock PM state */ 82 u32 clkpm_default:1; /* Default Clock PM state by BIOS */ 83 84 /* Exit latencies */ 85 struct aspm_latency latency_up; /* Upstream direction exit latency */ 86 struct aspm_latency latency_dw; /* Downstream direction exit latency */ 87 /* 88 * Endpoint acceptable latencies. A pcie downstream port only 89 * has one slot under it, so at most there are 8 functions. 90 */ 91 struct aspm_latency acceptable[8]; 92 93 /* L1 PM Substate info */ 94 struct { 95 u32 up_cap_ptr; /* L1SS cap ptr in upstream dev */ 96 u32 dw_cap_ptr; /* L1SS cap ptr in downstream dev */ 97 u32 ctl1; /* value to be programmed in ctl1 */ 98 u32 ctl2; /* value to be programmed in ctl2 */ 99 } l1ss; 100 }; 101 102 static int aspm_disabled, aspm_force; 103 static bool aspm_support_enabled = true; 104 static DEFINE_MUTEX(aspm_lock); 105 static LIST_HEAD(link_list); 106 107 #define POLICY_DEFAULT 0 /* BIOS default setting */ 108 #define POLICY_PERFORMANCE 1 /* high performance */ 109 #define POLICY_POWERSAVE 2 /* high power saving */ 110 #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */ 111 112 #ifdef CONFIG_PCIEASPM_PERFORMANCE 113 static int aspm_policy = POLICY_PERFORMANCE; 114 #elif defined CONFIG_PCIEASPM_POWERSAVE 115 static int aspm_policy = POLICY_POWERSAVE; 116 #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE 117 static int aspm_policy = POLICY_POWER_SUPERSAVE; 118 #else 119 static int aspm_policy; 120 #endif 121 122 static const char *policy_str[] = { 123 [POLICY_DEFAULT] = "default", 124 [POLICY_PERFORMANCE] = "performance", 125 [POLICY_POWERSAVE] = "powersave", 126 [POLICY_POWER_SUPERSAVE] = "powersupersave" 127 }; 128 129 #define LINK_RETRAIN_TIMEOUT HZ 130 131 static int policy_to_aspm_state(struct pcie_link_state *link) 132 { 133 switch (aspm_policy) { 134 case POLICY_PERFORMANCE: 135 /* Disable ASPM and Clock PM */ 136 return 0; 137 case POLICY_POWERSAVE: 138 /* Enable ASPM L0s/L1 */ 139 return (ASPM_STATE_L0S | ASPM_STATE_L1); 140 case POLICY_POWER_SUPERSAVE: 141 /* Enable Everything */ 142 return ASPM_STATE_ALL; 143 case POLICY_DEFAULT: 144 return link->aspm_default; 145 } 146 return 0; 147 } 148 149 static int policy_to_clkpm_state(struct pcie_link_state *link) 150 { 151 switch (aspm_policy) { 152 case POLICY_PERFORMANCE: 153 /* Disable ASPM and Clock PM */ 154 return 0; 155 case POLICY_POWERSAVE: 156 case POLICY_POWER_SUPERSAVE: 157 /* Enable Clock PM */ 158 return 1; 159 case POLICY_DEFAULT: 160 return link->clkpm_default; 161 } 162 return 0; 163 } 164 165 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable) 166 { 167 struct pci_dev *child; 168 struct pci_bus *linkbus = link->pdev->subordinate; 169 u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0; 170 171 list_for_each_entry(child, &linkbus->devices, bus_list) 172 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 173 PCI_EXP_LNKCTL_CLKREQ_EN, 174 val); 175 link->clkpm_enabled = !!enable; 176 } 177 178 static void pcie_set_clkpm(struct pcie_link_state *link, int enable) 179 { 180 /* Don't enable Clock PM if the link is not Clock PM capable */ 181 if (!link->clkpm_capable) 182 enable = 0; 183 /* Need nothing if the specified equals to current state */ 184 if (link->clkpm_enabled == enable) 185 return; 186 pcie_set_clkpm_nocheck(link, enable); 187 } 188 189 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist) 190 { 191 int capable = 1, enabled = 1; 192 u32 reg32; 193 u16 reg16; 194 struct pci_dev *child; 195 struct pci_bus *linkbus = link->pdev->subordinate; 196 197 /* All functions should have the same cap and state, take the worst */ 198 list_for_each_entry(child, &linkbus->devices, bus_list) { 199 pcie_capability_read_dword(child, PCI_EXP_LNKCAP, ®32); 200 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) { 201 capable = 0; 202 enabled = 0; 203 break; 204 } 205 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 206 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN)) 207 enabled = 0; 208 } 209 link->clkpm_enabled = enabled; 210 link->clkpm_default = enabled; 211 link->clkpm_capable = (blacklist) ? 0 : capable; 212 } 213 214 /* 215 * pcie_aspm_configure_common_clock: check if the 2 ends of a link 216 * could use common clock. If they are, configure them to use the 217 * common clock. That will reduce the ASPM state exit latency. 218 */ 219 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link) 220 { 221 int same_clock = 1; 222 u16 reg16, parent_reg, child_reg[8]; 223 unsigned long start_jiffies; 224 struct pci_dev *child, *parent = link->pdev; 225 struct pci_bus *linkbus = parent->subordinate; 226 /* 227 * All functions of a slot should have the same Slot Clock 228 * Configuration, so just check one function 229 */ 230 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list); 231 BUG_ON(!pci_is_pcie(child)); 232 233 /* Check downstream component if bit Slot Clock Configuration is 1 */ 234 pcie_capability_read_word(child, PCI_EXP_LNKSTA, ®16); 235 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 236 same_clock = 0; 237 238 /* Check upstream component if bit Slot Clock Configuration is 1 */ 239 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 240 if (!(reg16 & PCI_EXP_LNKSTA_SLC)) 241 same_clock = 0; 242 243 /* Configure downstream component, all functions */ 244 list_for_each_entry(child, &linkbus->devices, bus_list) { 245 pcie_capability_read_word(child, PCI_EXP_LNKCTL, ®16); 246 child_reg[PCI_FUNC(child->devfn)] = reg16; 247 if (same_clock) 248 reg16 |= PCI_EXP_LNKCTL_CCC; 249 else 250 reg16 &= ~PCI_EXP_LNKCTL_CCC; 251 pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16); 252 } 253 254 /* Configure upstream component */ 255 pcie_capability_read_word(parent, PCI_EXP_LNKCTL, ®16); 256 parent_reg = reg16; 257 if (same_clock) 258 reg16 |= PCI_EXP_LNKCTL_CCC; 259 else 260 reg16 &= ~PCI_EXP_LNKCTL_CCC; 261 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 262 263 /* Retrain link */ 264 reg16 |= PCI_EXP_LNKCTL_RL; 265 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16); 266 267 /* Wait for link training end. Break out after waiting for timeout */ 268 start_jiffies = jiffies; 269 for (;;) { 270 pcie_capability_read_word(parent, PCI_EXP_LNKSTA, ®16); 271 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 272 break; 273 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) 274 break; 275 msleep(1); 276 } 277 if (!(reg16 & PCI_EXP_LNKSTA_LT)) 278 return; 279 280 /* Training failed. Restore common clock configurations */ 281 dev_err(&parent->dev, "ASPM: Could not configure common clock\n"); 282 list_for_each_entry(child, &linkbus->devices, bus_list) 283 pcie_capability_write_word(child, PCI_EXP_LNKCTL, 284 child_reg[PCI_FUNC(child->devfn)]); 285 pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg); 286 } 287 288 /* Convert L0s latency encoding to ns */ 289 static u32 calc_l0s_latency(u32 encoding) 290 { 291 if (encoding == 0x7) 292 return (5 * 1000); /* > 4us */ 293 return (64 << encoding); 294 } 295 296 /* Convert L0s acceptable latency encoding to ns */ 297 static u32 calc_l0s_acceptable(u32 encoding) 298 { 299 if (encoding == 0x7) 300 return -1U; 301 return (64 << encoding); 302 } 303 304 /* Convert L1 latency encoding to ns */ 305 static u32 calc_l1_latency(u32 encoding) 306 { 307 if (encoding == 0x7) 308 return (65 * 1000); /* > 64us */ 309 return (1000 << encoding); 310 } 311 312 /* Convert L1 acceptable latency encoding to ns */ 313 static u32 calc_l1_acceptable(u32 encoding) 314 { 315 if (encoding == 0x7) 316 return -1U; 317 return (1000 << encoding); 318 } 319 320 /* Convert L1SS T_pwr encoding to usec */ 321 static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val) 322 { 323 switch (scale) { 324 case 0: 325 return val * 2; 326 case 1: 327 return val * 10; 328 case 2: 329 return val * 100; 330 } 331 dev_err(&pdev->dev, "%s: Invalid T_PwrOn scale: %u\n", 332 __func__, scale); 333 return 0; 334 } 335 336 struct aspm_register_info { 337 u32 support:2; 338 u32 enabled:2; 339 u32 latency_encoding_l0s; 340 u32 latency_encoding_l1; 341 342 /* L1 substates */ 343 u32 l1ss_cap_ptr; 344 u32 l1ss_cap; 345 u32 l1ss_ctl1; 346 u32 l1ss_ctl2; 347 }; 348 349 static void pcie_get_aspm_reg(struct pci_dev *pdev, 350 struct aspm_register_info *info) 351 { 352 u16 reg16; 353 u32 reg32; 354 355 pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, ®32); 356 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10; 357 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12; 358 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15; 359 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, ®16); 360 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC; 361 362 /* Read L1 PM substate capabilities */ 363 info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0; 364 info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS); 365 if (!info->l1ss_cap_ptr) 366 return; 367 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP, 368 &info->l1ss_cap); 369 if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) { 370 info->l1ss_cap = 0; 371 return; 372 } 373 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1, 374 &info->l1ss_ctl1); 375 pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2, 376 &info->l1ss_ctl2); 377 } 378 379 static void pcie_aspm_check_latency(struct pci_dev *endpoint) 380 { 381 u32 latency, l1_switch_latency = 0; 382 struct aspm_latency *acceptable; 383 struct pcie_link_state *link; 384 385 /* Device not in D0 doesn't need latency check */ 386 if ((endpoint->current_state != PCI_D0) && 387 (endpoint->current_state != PCI_UNKNOWN)) 388 return; 389 390 link = endpoint->bus->self->link_state; 391 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)]; 392 393 while (link) { 394 /* Check upstream direction L0s latency */ 395 if ((link->aspm_capable & ASPM_STATE_L0S_UP) && 396 (link->latency_up.l0s > acceptable->l0s)) 397 link->aspm_capable &= ~ASPM_STATE_L0S_UP; 398 399 /* Check downstream direction L0s latency */ 400 if ((link->aspm_capable & ASPM_STATE_L0S_DW) && 401 (link->latency_dw.l0s > acceptable->l0s)) 402 link->aspm_capable &= ~ASPM_STATE_L0S_DW; 403 /* 404 * Check L1 latency. 405 * Every switch on the path to root complex need 1 406 * more microsecond for L1. Spec doesn't mention L0s. 407 * 408 * The exit latencies for L1 substates are not advertised 409 * by a device. Since the spec also doesn't mention a way 410 * to determine max latencies introduced by enabling L1 411 * substates on the components, it is not clear how to do 412 * a L1 substate exit latency check. We assume that the 413 * L1 exit latencies advertised by a device include L1 414 * substate latencies (and hence do not do any check). 415 */ 416 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1); 417 if ((link->aspm_capable & ASPM_STATE_L1) && 418 (latency + l1_switch_latency > acceptable->l1)) 419 link->aspm_capable &= ~ASPM_STATE_L1; 420 l1_switch_latency += 1000; 421 422 link = link->parent; 423 } 424 } 425 426 /* 427 * The L1 PM substate capability is only implemented in function 0 in a 428 * multi function device. 429 */ 430 static struct pci_dev *pci_function_0(struct pci_bus *linkbus) 431 { 432 struct pci_dev *child; 433 434 list_for_each_entry(child, &linkbus->devices, bus_list) 435 if (PCI_FUNC(child->devfn) == 0) 436 return child; 437 return NULL; 438 } 439 440 /* Calculate L1.2 PM substate timing parameters */ 441 static void aspm_calc_l1ss_info(struct pcie_link_state *link, 442 struct aspm_register_info *upreg, 443 struct aspm_register_info *dwreg) 444 { 445 u32 val1, val2, scale1, scale2; 446 447 link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr; 448 link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr; 449 link->l1ss.ctl1 = link->l1ss.ctl2 = 0; 450 451 if (!(link->aspm_support & ASPM_STATE_L1_2_MASK)) 452 return; 453 454 /* Choose the greater of the two T_cmn_mode_rstr_time */ 455 val1 = (upreg->l1ss_cap >> 8) & 0xFF; 456 val2 = (upreg->l1ss_cap >> 8) & 0xFF; 457 if (val1 > val2) 458 link->l1ss.ctl1 |= val1 << 8; 459 else 460 link->l1ss.ctl1 |= val2 << 8; 461 /* 462 * We currently use LTR L1.2 threshold to be fixed constant picked from 463 * Intel's coreboot. 464 */ 465 link->l1ss.ctl1 |= LTR_L1_2_THRESHOLD_BITS; 466 467 /* Choose the greater of the two T_pwr_on */ 468 val1 = (upreg->l1ss_cap >> 19) & 0x1F; 469 scale1 = (upreg->l1ss_cap >> 16) & 0x03; 470 val2 = (dwreg->l1ss_cap >> 19) & 0x1F; 471 scale2 = (dwreg->l1ss_cap >> 16) & 0x03; 472 473 if (calc_l1ss_pwron(link->pdev, scale1, val1) > 474 calc_l1ss_pwron(link->downstream, scale2, val2)) 475 link->l1ss.ctl2 |= scale1 | (val1 << 3); 476 else 477 link->l1ss.ctl2 |= scale2 | (val2 << 3); 478 } 479 480 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist) 481 { 482 struct pci_dev *child = link->downstream, *parent = link->pdev; 483 struct pci_bus *linkbus = parent->subordinate; 484 struct aspm_register_info upreg, dwreg; 485 486 if (blacklist) { 487 /* Set enabled/disable so that we will disable ASPM later */ 488 link->aspm_enabled = ASPM_STATE_ALL; 489 link->aspm_disable = ASPM_STATE_ALL; 490 return; 491 } 492 493 /* Get upstream/downstream components' register state */ 494 pcie_get_aspm_reg(parent, &upreg); 495 pcie_get_aspm_reg(child, &dwreg); 496 497 /* 498 * If ASPM not supported, don't mess with the clocks and link, 499 * bail out now. 500 */ 501 if (!(upreg.support & dwreg.support)) 502 return; 503 504 /* Configure common clock before checking latencies */ 505 pcie_aspm_configure_common_clock(link); 506 507 /* 508 * Re-read upstream/downstream components' register state 509 * after clock configuration 510 */ 511 pcie_get_aspm_reg(parent, &upreg); 512 pcie_get_aspm_reg(child, &dwreg); 513 514 /* 515 * Setup L0s state 516 * 517 * Note that we must not enable L0s in either direction on a 518 * given link unless components on both sides of the link each 519 * support L0s. 520 */ 521 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S) 522 link->aspm_support |= ASPM_STATE_L0S; 523 if (dwreg.enabled & PCIE_LINK_STATE_L0S) 524 link->aspm_enabled |= ASPM_STATE_L0S_UP; 525 if (upreg.enabled & PCIE_LINK_STATE_L0S) 526 link->aspm_enabled |= ASPM_STATE_L0S_DW; 527 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s); 528 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s); 529 530 /* Setup L1 state */ 531 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1) 532 link->aspm_support |= ASPM_STATE_L1; 533 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1) 534 link->aspm_enabled |= ASPM_STATE_L1; 535 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1); 536 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1); 537 538 /* Setup L1 substate */ 539 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1) 540 link->aspm_support |= ASPM_STATE_L1_1; 541 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2) 542 link->aspm_support |= ASPM_STATE_L1_2; 543 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1) 544 link->aspm_support |= ASPM_STATE_L1_1_PCIPM; 545 if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2) 546 link->aspm_support |= ASPM_STATE_L1_2_PCIPM; 547 548 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1) 549 link->aspm_enabled |= ASPM_STATE_L1_1; 550 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2) 551 link->aspm_enabled |= ASPM_STATE_L1_2; 552 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1) 553 link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM; 554 if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2) 555 link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM; 556 557 if (link->aspm_support & ASPM_STATE_L1SS) 558 aspm_calc_l1ss_info(link, &upreg, &dwreg); 559 560 /* Save default state */ 561 link->aspm_default = link->aspm_enabled; 562 563 /* Setup initial capable state. Will be updated later */ 564 link->aspm_capable = link->aspm_support; 565 /* 566 * If the downstream component has pci bridge function, don't 567 * do ASPM for now. 568 */ 569 list_for_each_entry(child, &linkbus->devices, bus_list) { 570 if (pci_pcie_type(child) == PCI_EXP_TYPE_PCI_BRIDGE) { 571 link->aspm_disable = ASPM_STATE_ALL; 572 break; 573 } 574 } 575 576 /* Get and check endpoint acceptable latencies */ 577 list_for_each_entry(child, &linkbus->devices, bus_list) { 578 u32 reg32, encoding; 579 struct aspm_latency *acceptable = 580 &link->acceptable[PCI_FUNC(child->devfn)]; 581 582 if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT && 583 pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END) 584 continue; 585 586 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 587 /* Calculate endpoint L0s acceptable latency */ 588 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6; 589 acceptable->l0s = calc_l0s_acceptable(encoding); 590 /* Calculate endpoint L1 acceptable latency */ 591 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9; 592 acceptable->l1 = calc_l1_acceptable(encoding); 593 594 pcie_aspm_check_latency(child); 595 } 596 } 597 598 static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos, 599 u32 clear, u32 set) 600 { 601 u32 val; 602 603 pci_read_config_dword(pdev, pos, &val); 604 val &= ~clear; 605 val |= set; 606 pci_write_config_dword(pdev, pos, val); 607 } 608 609 /* Configure the ASPM L1 substates */ 610 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state) 611 { 612 u32 val, enable_req; 613 struct pci_dev *child = link->downstream, *parent = link->pdev; 614 u32 up_cap_ptr = link->l1ss.up_cap_ptr; 615 u32 dw_cap_ptr = link->l1ss.dw_cap_ptr; 616 617 enable_req = (link->aspm_enabled ^ state) & state; 618 619 /* 620 * Here are the rules specified in the PCIe spec for enabling L1SS: 621 * - When enabling L1.x, enable bit at parent first, then at child 622 * - When disabling L1.x, disable bit at child first, then at parent 623 * - When enabling ASPM L1.x, need to disable L1 624 * (at child followed by parent). 625 * - The ASPM/PCIPM L1.2 must be disabled while programming timing 626 * parameters 627 * 628 * To keep it simple, disable all L1SS bits first, and later enable 629 * what is needed. 630 */ 631 632 /* Disable all L1 substates */ 633 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 634 PCI_L1SS_CTL1_L1SS_MASK, 0); 635 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 636 PCI_L1SS_CTL1_L1SS_MASK, 0); 637 /* 638 * If needed, disable L1, and it gets enabled later 639 * in pcie_config_aspm_link(). 640 */ 641 if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) { 642 pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL, 643 PCI_EXP_LNKCTL_ASPM_L1, 0); 644 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 645 PCI_EXP_LNKCTL_ASPM_L1, 0); 646 } 647 648 if (enable_req & ASPM_STATE_L1_2_MASK) { 649 650 /* Program T_pwr_on in both ports */ 651 pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2, 652 link->l1ss.ctl2); 653 pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2, 654 link->l1ss.ctl2); 655 656 /* Program T_cmn_mode in parent */ 657 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 658 0xFF00, link->l1ss.ctl1); 659 660 /* Program LTR L1.2 threshold in both ports */ 661 pci_clear_and_set_dword(parent, dw_cap_ptr + PCI_L1SS_CTL1, 662 0xE3FF0000, link->l1ss.ctl1); 663 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 664 0xE3FF0000, link->l1ss.ctl1); 665 } 666 667 val = 0; 668 if (state & ASPM_STATE_L1_1) 669 val |= PCI_L1SS_CTL1_ASPM_L1_1; 670 if (state & ASPM_STATE_L1_2) 671 val |= PCI_L1SS_CTL1_ASPM_L1_2; 672 if (state & ASPM_STATE_L1_1_PCIPM) 673 val |= PCI_L1SS_CTL1_PCIPM_L1_1; 674 if (state & ASPM_STATE_L1_2_PCIPM) 675 val |= PCI_L1SS_CTL1_PCIPM_L1_2; 676 677 /* Enable what we need to enable */ 678 pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1, 679 PCI_L1SS_CAP_L1_PM_SS, val); 680 pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1, 681 PCI_L1SS_CAP_L1_PM_SS, val); 682 } 683 684 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val) 685 { 686 pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL, 687 PCI_EXP_LNKCTL_ASPMC, val); 688 } 689 690 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state) 691 { 692 u32 upstream = 0, dwstream = 0; 693 struct pci_dev *child = link->downstream, *parent = link->pdev; 694 struct pci_bus *linkbus = parent->subordinate; 695 696 /* Enable only the states that were not explicitly disabled */ 697 state &= (link->aspm_capable & ~link->aspm_disable); 698 699 /* Can't enable any substates if L1 is not enabled */ 700 if (!(state & ASPM_STATE_L1)) 701 state &= ~ASPM_STATE_L1SS; 702 703 /* Spec says both ports must be in D0 before enabling PCI PM substates*/ 704 if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) { 705 state &= ~ASPM_STATE_L1_SS_PCIPM; 706 state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM); 707 } 708 709 /* Nothing to do if the link is already in the requested state */ 710 if (link->aspm_enabled == state) 711 return; 712 /* Convert ASPM state to upstream/downstream ASPM register state */ 713 if (state & ASPM_STATE_L0S_UP) 714 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S; 715 if (state & ASPM_STATE_L0S_DW) 716 upstream |= PCI_EXP_LNKCTL_ASPM_L0S; 717 if (state & ASPM_STATE_L1) { 718 upstream |= PCI_EXP_LNKCTL_ASPM_L1; 719 dwstream |= PCI_EXP_LNKCTL_ASPM_L1; 720 } 721 722 if (link->aspm_capable & ASPM_STATE_L1SS) 723 pcie_config_aspm_l1ss(link, state); 724 725 /* 726 * Spec 2.0 suggests all functions should be configured the 727 * same setting for ASPM. Enabling ASPM L1 should be done in 728 * upstream component first and then downstream, and vice 729 * versa for disabling ASPM L1. Spec doesn't mention L0S. 730 */ 731 if (state & ASPM_STATE_L1) 732 pcie_config_aspm_dev(parent, upstream); 733 list_for_each_entry(child, &linkbus->devices, bus_list) 734 pcie_config_aspm_dev(child, dwstream); 735 if (!(state & ASPM_STATE_L1)) 736 pcie_config_aspm_dev(parent, upstream); 737 738 link->aspm_enabled = state; 739 } 740 741 static void pcie_config_aspm_path(struct pcie_link_state *link) 742 { 743 while (link) { 744 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 745 link = link->parent; 746 } 747 } 748 749 static void free_link_state(struct pcie_link_state *link) 750 { 751 link->pdev->link_state = NULL; 752 kfree(link); 753 } 754 755 static int pcie_aspm_sanity_check(struct pci_dev *pdev) 756 { 757 struct pci_dev *child; 758 u32 reg32; 759 760 /* 761 * Some functions in a slot might not all be PCIe functions, 762 * very strange. Disable ASPM for the whole slot 763 */ 764 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) { 765 if (!pci_is_pcie(child)) 766 return -EINVAL; 767 768 /* 769 * If ASPM is disabled then we're not going to change 770 * the BIOS state. It's safe to continue even if it's a 771 * pre-1.1 device 772 */ 773 774 if (aspm_disabled) 775 continue; 776 777 /* 778 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use 779 * RBER bit to determine if a function is 1.1 version device 780 */ 781 pcie_capability_read_dword(child, PCI_EXP_DEVCAP, ®32); 782 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) { 783 dev_info(&child->dev, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n"); 784 return -EINVAL; 785 } 786 } 787 return 0; 788 } 789 790 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev) 791 { 792 struct pcie_link_state *link; 793 794 link = kzalloc(sizeof(*link), GFP_KERNEL); 795 if (!link) 796 return NULL; 797 798 INIT_LIST_HEAD(&link->sibling); 799 INIT_LIST_HEAD(&link->children); 800 INIT_LIST_HEAD(&link->link); 801 link->pdev = pdev; 802 link->downstream = pci_function_0(pdev->subordinate); 803 804 /* 805 * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe 806 * hierarchies. 807 */ 808 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT || 809 pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE) { 810 link->root = link; 811 } else { 812 struct pcie_link_state *parent; 813 814 parent = pdev->bus->parent->self->link_state; 815 if (!parent) { 816 kfree(link); 817 return NULL; 818 } 819 820 link->parent = parent; 821 link->root = link->parent->root; 822 list_add(&link->link, &parent->children); 823 } 824 825 list_add(&link->sibling, &link_list); 826 pdev->link_state = link; 827 return link; 828 } 829 830 /* 831 * pcie_aspm_init_link_state: Initiate PCI express link state. 832 * It is called after the pcie and its children devices are scanned. 833 * @pdev: the root port or switch downstream port 834 */ 835 void pcie_aspm_init_link_state(struct pci_dev *pdev) 836 { 837 struct pcie_link_state *link; 838 int blacklist = !!pcie_aspm_sanity_check(pdev); 839 840 if (!aspm_support_enabled) 841 return; 842 843 if (pdev->link_state) 844 return; 845 846 /* 847 * We allocate pcie_link_state for the component on the upstream 848 * end of a Link, so there's nothing to do unless this device has a 849 * Link on its secondary side. 850 */ 851 if (!pdev->has_secondary_link) 852 return; 853 854 /* VIA has a strange chipset, root port is under a bridge */ 855 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT && 856 pdev->bus->self) 857 return; 858 859 down_read(&pci_bus_sem); 860 if (list_empty(&pdev->subordinate->devices)) 861 goto out; 862 863 mutex_lock(&aspm_lock); 864 link = alloc_pcie_link_state(pdev); 865 if (!link) 866 goto unlock; 867 /* 868 * Setup initial ASPM state. Note that we need to configure 869 * upstream links also because capable state of them can be 870 * update through pcie_aspm_cap_init(). 871 */ 872 pcie_aspm_cap_init(link, blacklist); 873 874 /* Setup initial Clock PM state */ 875 pcie_clkpm_cap_init(link, blacklist); 876 877 /* 878 * At this stage drivers haven't had an opportunity to change the 879 * link policy setting. Enabling ASPM on broken hardware can cripple 880 * it even before the driver has had a chance to disable ASPM, so 881 * default to a safe level right now. If we're enabling ASPM beyond 882 * the BIOS's expectation, we'll do so once pci_enable_device() is 883 * called. 884 */ 885 if (aspm_policy != POLICY_POWERSAVE && 886 aspm_policy != POLICY_POWER_SUPERSAVE) { 887 pcie_config_aspm_path(link); 888 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 889 } 890 891 unlock: 892 mutex_unlock(&aspm_lock); 893 out: 894 up_read(&pci_bus_sem); 895 } 896 897 /* Recheck latencies and update aspm_capable for links under the root */ 898 static void pcie_update_aspm_capable(struct pcie_link_state *root) 899 { 900 struct pcie_link_state *link; 901 BUG_ON(root->parent); 902 list_for_each_entry(link, &link_list, sibling) { 903 if (link->root != root) 904 continue; 905 link->aspm_capable = link->aspm_support; 906 } 907 list_for_each_entry(link, &link_list, sibling) { 908 struct pci_dev *child; 909 struct pci_bus *linkbus = link->pdev->subordinate; 910 if (link->root != root) 911 continue; 912 list_for_each_entry(child, &linkbus->devices, bus_list) { 913 if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) && 914 (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)) 915 continue; 916 pcie_aspm_check_latency(child); 917 } 918 } 919 } 920 921 /* @pdev: the endpoint device */ 922 void pcie_aspm_exit_link_state(struct pci_dev *pdev) 923 { 924 struct pci_dev *parent = pdev->bus->self; 925 struct pcie_link_state *link, *root, *parent_link; 926 927 if (!parent || !parent->link_state) 928 return; 929 930 down_read(&pci_bus_sem); 931 mutex_lock(&aspm_lock); 932 /* 933 * All PCIe functions are in one slot, remove one function will remove 934 * the whole slot, so just wait until we are the last function left. 935 */ 936 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices)) 937 goto out; 938 939 link = parent->link_state; 940 root = link->root; 941 parent_link = link->parent; 942 943 /* All functions are removed, so just disable ASPM for the link */ 944 pcie_config_aspm_link(link, 0); 945 list_del(&link->sibling); 946 list_del(&link->link); 947 /* Clock PM is for endpoint device */ 948 free_link_state(link); 949 950 /* Recheck latencies and configure upstream links */ 951 if (parent_link) { 952 pcie_update_aspm_capable(root); 953 pcie_config_aspm_path(parent_link); 954 } 955 out: 956 mutex_unlock(&aspm_lock); 957 up_read(&pci_bus_sem); 958 } 959 960 /* @pdev: the root port or switch downstream port */ 961 void pcie_aspm_pm_state_change(struct pci_dev *pdev) 962 { 963 struct pcie_link_state *link = pdev->link_state; 964 965 if (aspm_disabled || !link) 966 return; 967 /* 968 * Devices changed PM state, we should recheck if latency 969 * meets all functions' requirement 970 */ 971 down_read(&pci_bus_sem); 972 mutex_lock(&aspm_lock); 973 pcie_update_aspm_capable(link->root); 974 pcie_config_aspm_path(link); 975 mutex_unlock(&aspm_lock); 976 up_read(&pci_bus_sem); 977 } 978 979 void pcie_aspm_powersave_config_link(struct pci_dev *pdev) 980 { 981 struct pcie_link_state *link = pdev->link_state; 982 983 if (aspm_disabled || !link) 984 return; 985 986 if (aspm_policy != POLICY_POWERSAVE && 987 aspm_policy != POLICY_POWER_SUPERSAVE) 988 return; 989 990 down_read(&pci_bus_sem); 991 mutex_lock(&aspm_lock); 992 pcie_config_aspm_path(link); 993 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 994 mutex_unlock(&aspm_lock); 995 up_read(&pci_bus_sem); 996 } 997 998 static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem) 999 { 1000 struct pci_dev *parent = pdev->bus->self; 1001 struct pcie_link_state *link; 1002 1003 if (!pci_is_pcie(pdev)) 1004 return; 1005 1006 if (pdev->has_secondary_link) 1007 parent = pdev; 1008 if (!parent || !parent->link_state) 1009 return; 1010 1011 /* 1012 * A driver requested that ASPM be disabled on this device, but 1013 * if we don't have permission to manage ASPM (e.g., on ACPI 1014 * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and 1015 * the _OSC method), we can't honor that request. Windows has 1016 * a similar mechanism using "PciASPMOptOut", which is also 1017 * ignored in this situation. 1018 */ 1019 if (aspm_disabled) { 1020 dev_warn(&pdev->dev, "can't disable ASPM; OS doesn't have ASPM control\n"); 1021 return; 1022 } 1023 1024 if (sem) 1025 down_read(&pci_bus_sem); 1026 mutex_lock(&aspm_lock); 1027 link = parent->link_state; 1028 if (state & PCIE_LINK_STATE_L0S) 1029 link->aspm_disable |= ASPM_STATE_L0S; 1030 if (state & PCIE_LINK_STATE_L1) 1031 link->aspm_disable |= ASPM_STATE_L1; 1032 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1033 1034 if (state & PCIE_LINK_STATE_CLKPM) { 1035 link->clkpm_capable = 0; 1036 pcie_set_clkpm(link, 0); 1037 } 1038 mutex_unlock(&aspm_lock); 1039 if (sem) 1040 up_read(&pci_bus_sem); 1041 } 1042 1043 void pci_disable_link_state_locked(struct pci_dev *pdev, int state) 1044 { 1045 __pci_disable_link_state(pdev, state, false); 1046 } 1047 EXPORT_SYMBOL(pci_disable_link_state_locked); 1048 1049 /** 1050 * pci_disable_link_state - Disable device's link state, so the link will 1051 * never enter specific states. Note that if the BIOS didn't grant ASPM 1052 * control to the OS, this does nothing because we can't touch the LNKCTL 1053 * register. 1054 * 1055 * @pdev: PCI device 1056 * @state: ASPM link state to disable 1057 */ 1058 void pci_disable_link_state(struct pci_dev *pdev, int state) 1059 { 1060 __pci_disable_link_state(pdev, state, true); 1061 } 1062 EXPORT_SYMBOL(pci_disable_link_state); 1063 1064 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp) 1065 { 1066 int i; 1067 struct pcie_link_state *link; 1068 1069 if (aspm_disabled) 1070 return -EPERM; 1071 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1072 if (!strncmp(val, policy_str[i], strlen(policy_str[i]))) 1073 break; 1074 if (i >= ARRAY_SIZE(policy_str)) 1075 return -EINVAL; 1076 if (i == aspm_policy) 1077 return 0; 1078 1079 down_read(&pci_bus_sem); 1080 mutex_lock(&aspm_lock); 1081 aspm_policy = i; 1082 list_for_each_entry(link, &link_list, sibling) { 1083 pcie_config_aspm_link(link, policy_to_aspm_state(link)); 1084 pcie_set_clkpm(link, policy_to_clkpm_state(link)); 1085 } 1086 mutex_unlock(&aspm_lock); 1087 up_read(&pci_bus_sem); 1088 return 0; 1089 } 1090 1091 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp) 1092 { 1093 int i, cnt = 0; 1094 for (i = 0; i < ARRAY_SIZE(policy_str); i++) 1095 if (i == aspm_policy) 1096 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]); 1097 else 1098 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]); 1099 return cnt; 1100 } 1101 1102 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy, 1103 NULL, 0644); 1104 1105 #ifdef CONFIG_PCIEASPM_DEBUG 1106 static ssize_t link_state_show(struct device *dev, 1107 struct device_attribute *attr, 1108 char *buf) 1109 { 1110 struct pci_dev *pci_device = to_pci_dev(dev); 1111 struct pcie_link_state *link_state = pci_device->link_state; 1112 1113 return sprintf(buf, "%d\n", link_state->aspm_enabled); 1114 } 1115 1116 static ssize_t link_state_store(struct device *dev, 1117 struct device_attribute *attr, 1118 const char *buf, 1119 size_t n) 1120 { 1121 struct pci_dev *pdev = to_pci_dev(dev); 1122 struct pcie_link_state *link, *root = pdev->link_state->root; 1123 u32 state; 1124 1125 if (aspm_disabled) 1126 return -EPERM; 1127 1128 if (kstrtouint(buf, 10, &state)) 1129 return -EINVAL; 1130 if ((state & ~ASPM_STATE_ALL) != 0) 1131 return -EINVAL; 1132 1133 down_read(&pci_bus_sem); 1134 mutex_lock(&aspm_lock); 1135 list_for_each_entry(link, &link_list, sibling) { 1136 if (link->root != root) 1137 continue; 1138 pcie_config_aspm_link(link, state); 1139 } 1140 mutex_unlock(&aspm_lock); 1141 up_read(&pci_bus_sem); 1142 return n; 1143 } 1144 1145 static ssize_t clk_ctl_show(struct device *dev, 1146 struct device_attribute *attr, 1147 char *buf) 1148 { 1149 struct pci_dev *pci_device = to_pci_dev(dev); 1150 struct pcie_link_state *link_state = pci_device->link_state; 1151 1152 return sprintf(buf, "%d\n", link_state->clkpm_enabled); 1153 } 1154 1155 static ssize_t clk_ctl_store(struct device *dev, 1156 struct device_attribute *attr, 1157 const char *buf, 1158 size_t n) 1159 { 1160 struct pci_dev *pdev = to_pci_dev(dev); 1161 bool state; 1162 1163 if (strtobool(buf, &state)) 1164 return -EINVAL; 1165 1166 down_read(&pci_bus_sem); 1167 mutex_lock(&aspm_lock); 1168 pcie_set_clkpm_nocheck(pdev->link_state, state); 1169 mutex_unlock(&aspm_lock); 1170 up_read(&pci_bus_sem); 1171 1172 return n; 1173 } 1174 1175 static DEVICE_ATTR_RW(link_state); 1176 static DEVICE_ATTR_RW(clk_ctl); 1177 1178 static char power_group[] = "power"; 1179 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev) 1180 { 1181 struct pcie_link_state *link_state = pdev->link_state; 1182 1183 if (!link_state) 1184 return; 1185 1186 if (link_state->aspm_support) 1187 sysfs_add_file_to_group(&pdev->dev.kobj, 1188 &dev_attr_link_state.attr, power_group); 1189 if (link_state->clkpm_capable) 1190 sysfs_add_file_to_group(&pdev->dev.kobj, 1191 &dev_attr_clk_ctl.attr, power_group); 1192 } 1193 1194 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev) 1195 { 1196 struct pcie_link_state *link_state = pdev->link_state; 1197 1198 if (!link_state) 1199 return; 1200 1201 if (link_state->aspm_support) 1202 sysfs_remove_file_from_group(&pdev->dev.kobj, 1203 &dev_attr_link_state.attr, power_group); 1204 if (link_state->clkpm_capable) 1205 sysfs_remove_file_from_group(&pdev->dev.kobj, 1206 &dev_attr_clk_ctl.attr, power_group); 1207 } 1208 #endif 1209 1210 static int __init pcie_aspm_disable(char *str) 1211 { 1212 if (!strcmp(str, "off")) { 1213 aspm_policy = POLICY_DEFAULT; 1214 aspm_disabled = 1; 1215 aspm_support_enabled = false; 1216 printk(KERN_INFO "PCIe ASPM is disabled\n"); 1217 } else if (!strcmp(str, "force")) { 1218 aspm_force = 1; 1219 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n"); 1220 } 1221 return 1; 1222 } 1223 1224 __setup("pcie_aspm=", pcie_aspm_disable); 1225 1226 void pcie_no_aspm(void) 1227 { 1228 /* 1229 * Disabling ASPM is intended to prevent the kernel from modifying 1230 * existing hardware state, not to clear existing state. To that end: 1231 * (a) set policy to POLICY_DEFAULT in order to avoid changing state 1232 * (b) prevent userspace from changing policy 1233 */ 1234 if (!aspm_force) { 1235 aspm_policy = POLICY_DEFAULT; 1236 aspm_disabled = 1; 1237 } 1238 } 1239 1240 bool pcie_aspm_support_enabled(void) 1241 { 1242 return aspm_support_enabled; 1243 } 1244 EXPORT_SYMBOL(pcie_aspm_support_enabled); 1245