xref: /openbmc/linux/drivers/pci/pcie/aspm.c (revision fd589a8f)
1 /*
2  * File:	drivers/pci/pcie/aspm.c
3  * Enabling PCIE link L0s/L1 state and Clock Power Management
4  *
5  * Copyright (C) 2007 Intel
6  * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7  * Copyright (C) Shaohua Li (shaohua.li@intel.com)
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
16 #include <linux/pm.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
22 #include "../pci.h"
23 
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
26 #endif
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
28 
29 /* Note: those are not register definitions */
30 #define ASPM_STATE_L0S_UP	(1)	/* Upstream direction L0s state */
31 #define ASPM_STATE_L0S_DW	(2)	/* Downstream direction L0s state */
32 #define ASPM_STATE_L1		(4)	/* L1 state */
33 #define ASPM_STATE_L0S		(ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
34 #define ASPM_STATE_ALL		(ASPM_STATE_L0S | ASPM_STATE_L1)
35 
36 struct aspm_latency {
37 	u32 l0s;			/* L0s latency (nsec) */
38 	u32 l1;				/* L1 latency (nsec) */
39 };
40 
41 struct pcie_link_state {
42 	struct pci_dev *pdev;		/* Upstream component of the Link */
43 	struct pcie_link_state *root;	/* pointer to the root port link */
44 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
45 	struct list_head sibling;	/* node in link_list */
46 	struct list_head children;	/* list of child link states */
47 	struct list_head link;		/* node in parent's children list */
48 
49 	/* ASPM state */
50 	u32 aspm_support:3;		/* Supported ASPM state */
51 	u32 aspm_enabled:3;		/* Enabled ASPM state */
52 	u32 aspm_capable:3;		/* Capable ASPM state with latency */
53 	u32 aspm_default:3;		/* Default ASPM state by BIOS */
54 	u32 aspm_disable:3;		/* Disabled ASPM state */
55 
56 	/* Clock PM state */
57 	u32 clkpm_capable:1;		/* Clock PM capable? */
58 	u32 clkpm_enabled:1;		/* Current Clock PM state */
59 	u32 clkpm_default:1;		/* Default Clock PM state by BIOS */
60 
61 	/* Exit latencies */
62 	struct aspm_latency latency_up;	/* Upstream direction exit latency */
63 	struct aspm_latency latency_dw;	/* Downstream direction exit latency */
64 	/*
65 	 * Endpoint acceptable latencies. A pcie downstream port only
66 	 * has one slot under it, so at most there are 8 functions.
67 	 */
68 	struct aspm_latency acceptable[8];
69 };
70 
71 static int aspm_disabled, aspm_force;
72 static DEFINE_MUTEX(aspm_lock);
73 static LIST_HEAD(link_list);
74 
75 #define POLICY_DEFAULT 0	/* BIOS default setting */
76 #define POLICY_PERFORMANCE 1	/* high performance */
77 #define POLICY_POWERSAVE 2	/* high power saving */
78 static int aspm_policy;
79 static const char *policy_str[] = {
80 	[POLICY_DEFAULT] = "default",
81 	[POLICY_PERFORMANCE] = "performance",
82 	[POLICY_POWERSAVE] = "powersave"
83 };
84 
85 #define LINK_RETRAIN_TIMEOUT HZ
86 
87 static int policy_to_aspm_state(struct pcie_link_state *link)
88 {
89 	switch (aspm_policy) {
90 	case POLICY_PERFORMANCE:
91 		/* Disable ASPM and Clock PM */
92 		return 0;
93 	case POLICY_POWERSAVE:
94 		/* Enable ASPM L0s/L1 */
95 		return ASPM_STATE_ALL;
96 	case POLICY_DEFAULT:
97 		return link->aspm_default;
98 	}
99 	return 0;
100 }
101 
102 static int policy_to_clkpm_state(struct pcie_link_state *link)
103 {
104 	switch (aspm_policy) {
105 	case POLICY_PERFORMANCE:
106 		/* Disable ASPM and Clock PM */
107 		return 0;
108 	case POLICY_POWERSAVE:
109 		/* Disable Clock PM */
110 		return 1;
111 	case POLICY_DEFAULT:
112 		return link->clkpm_default;
113 	}
114 	return 0;
115 }
116 
117 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
118 {
119 	int pos;
120 	u16 reg16;
121 	struct pci_dev *child;
122 	struct pci_bus *linkbus = link->pdev->subordinate;
123 
124 	list_for_each_entry(child, &linkbus->devices, bus_list) {
125 		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
126 		if (!pos)
127 			return;
128 		pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
129 		if (enable)
130 			reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
131 		else
132 			reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
133 		pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
134 	}
135 	link->clkpm_enabled = !!enable;
136 }
137 
138 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
139 {
140 	/* Don't enable Clock PM if the link is not Clock PM capable */
141 	if (!link->clkpm_capable && enable)
142 		return;
143 	/* Need nothing if the specified equals to current state */
144 	if (link->clkpm_enabled == enable)
145 		return;
146 	pcie_set_clkpm_nocheck(link, enable);
147 }
148 
149 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
150 {
151 	int pos, capable = 1, enabled = 1;
152 	u32 reg32;
153 	u16 reg16;
154 	struct pci_dev *child;
155 	struct pci_bus *linkbus = link->pdev->subordinate;
156 
157 	/* All functions should have the same cap and state, take the worst */
158 	list_for_each_entry(child, &linkbus->devices, bus_list) {
159 		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
160 		if (!pos)
161 			return;
162 		pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, &reg32);
163 		if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
164 			capable = 0;
165 			enabled = 0;
166 			break;
167 		}
168 		pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
169 		if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
170 			enabled = 0;
171 	}
172 	link->clkpm_enabled = enabled;
173 	link->clkpm_default = enabled;
174 	link->clkpm_capable = (blacklist) ? 0 : capable;
175 }
176 
177 /*
178  * pcie_aspm_configure_common_clock: check if the 2 ends of a link
179  *   could use common clock. If they are, configure them to use the
180  *   common clock. That will reduce the ASPM state exit latency.
181  */
182 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
183 {
184 	int ppos, cpos, same_clock = 1;
185 	u16 reg16, parent_reg, child_reg[8];
186 	unsigned long start_jiffies;
187 	struct pci_dev *child, *parent = link->pdev;
188 	struct pci_bus *linkbus = parent->subordinate;
189 	/*
190 	 * All functions of a slot should have the same Slot Clock
191 	 * Configuration, so just check one function
192 	 */
193 	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
194 	BUG_ON(!child->is_pcie);
195 
196 	/* Check downstream component if bit Slot Clock Configuration is 1 */
197 	cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
198 	pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, &reg16);
199 	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
200 		same_clock = 0;
201 
202 	/* Check upstream component if bit Slot Clock Configuration is 1 */
203 	ppos = pci_find_capability(parent, PCI_CAP_ID_EXP);
204 	pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
205 	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
206 		same_clock = 0;
207 
208 	/* Configure downstream component, all functions */
209 	list_for_each_entry(child, &linkbus->devices, bus_list) {
210 		cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
211 		pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, &reg16);
212 		child_reg[PCI_FUNC(child->devfn)] = reg16;
213 		if (same_clock)
214 			reg16 |= PCI_EXP_LNKCTL_CCC;
215 		else
216 			reg16 &= ~PCI_EXP_LNKCTL_CCC;
217 		pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
218 	}
219 
220 	/* Configure upstream component */
221 	pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, &reg16);
222 	parent_reg = reg16;
223 	if (same_clock)
224 		reg16 |= PCI_EXP_LNKCTL_CCC;
225 	else
226 		reg16 &= ~PCI_EXP_LNKCTL_CCC;
227 	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
228 
229 	/* Retrain link */
230 	reg16 |= PCI_EXP_LNKCTL_RL;
231 	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
232 
233 	/* Wait for link training end. Break out after waiting for timeout */
234 	start_jiffies = jiffies;
235 	for (;;) {
236 		pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
237 		if (!(reg16 & PCI_EXP_LNKSTA_LT))
238 			break;
239 		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
240 			break;
241 		msleep(1);
242 	}
243 	if (!(reg16 & PCI_EXP_LNKSTA_LT))
244 		return;
245 
246 	/* Training failed. Restore common clock configurations */
247 	dev_printk(KERN_ERR, &parent->dev,
248 		   "ASPM: Could not configure common clock\n");
249 	list_for_each_entry(child, &linkbus->devices, bus_list) {
250 		cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
251 		pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
252 				      child_reg[PCI_FUNC(child->devfn)]);
253 	}
254 	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
255 }
256 
257 /* Convert L0s latency encoding to ns */
258 static u32 calc_l0s_latency(u32 encoding)
259 {
260 	if (encoding == 0x7)
261 		return (5 * 1000);	/* > 4us */
262 	return (64 << encoding);
263 }
264 
265 /* Convert L0s acceptable latency encoding to ns */
266 static u32 calc_l0s_acceptable(u32 encoding)
267 {
268 	if (encoding == 0x7)
269 		return -1U;
270 	return (64 << encoding);
271 }
272 
273 /* Convert L1 latency encoding to ns */
274 static u32 calc_l1_latency(u32 encoding)
275 {
276 	if (encoding == 0x7)
277 		return (65 * 1000);	/* > 64us */
278 	return (1000 << encoding);
279 }
280 
281 /* Convert L1 acceptable latency encoding to ns */
282 static u32 calc_l1_acceptable(u32 encoding)
283 {
284 	if (encoding == 0x7)
285 		return -1U;
286 	return (1000 << encoding);
287 }
288 
289 struct aspm_register_info {
290 	u32 support:2;
291 	u32 enabled:2;
292 	u32 latency_encoding_l0s;
293 	u32 latency_encoding_l1;
294 };
295 
296 static void pcie_get_aspm_reg(struct pci_dev *pdev,
297 			      struct aspm_register_info *info)
298 {
299 	int pos;
300 	u16 reg16;
301 	u32 reg32;
302 
303 	pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
304 	pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32);
305 	info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
306 	/* 00b and 10b are defined as "Reserved". */
307 	if (info->support == PCIE_LINK_STATE_L1)
308 		info->support = 0;
309 	info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
310 	info->latency_encoding_l1  = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
311 	pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
312 	info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
313 }
314 
315 static void pcie_aspm_check_latency(struct pci_dev *endpoint)
316 {
317 	u32 latency, l1_switch_latency = 0;
318 	struct aspm_latency *acceptable;
319 	struct pcie_link_state *link;
320 
321 	/* Device not in D0 doesn't need latency check */
322 	if ((endpoint->current_state != PCI_D0) &&
323 	    (endpoint->current_state != PCI_UNKNOWN))
324 		return;
325 
326 	link = endpoint->bus->self->link_state;
327 	acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
328 
329 	while (link) {
330 		/* Check upstream direction L0s latency */
331 		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
332 		    (link->latency_up.l0s > acceptable->l0s))
333 			link->aspm_capable &= ~ASPM_STATE_L0S_UP;
334 
335 		/* Check downstream direction L0s latency */
336 		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
337 		    (link->latency_dw.l0s > acceptable->l0s))
338 			link->aspm_capable &= ~ASPM_STATE_L0S_DW;
339 		/*
340 		 * Check L1 latency.
341 		 * Every switch on the path to root complex need 1
342 		 * more microsecond for L1. Spec doesn't mention L0s.
343 		 */
344 		latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
345 		if ((link->aspm_capable & ASPM_STATE_L1) &&
346 		    (latency + l1_switch_latency > acceptable->l1))
347 			link->aspm_capable &= ~ASPM_STATE_L1;
348 		l1_switch_latency += 1000;
349 
350 		link = link->parent;
351 	}
352 }
353 
354 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
355 {
356 	struct pci_dev *child, *parent = link->pdev;
357 	struct pci_bus *linkbus = parent->subordinate;
358 	struct aspm_register_info upreg, dwreg;
359 
360 	if (blacklist) {
361 		/* Set enabled/disable so that we will disable ASPM later */
362 		link->aspm_enabled = ASPM_STATE_ALL;
363 		link->aspm_disable = ASPM_STATE_ALL;
364 		return;
365 	}
366 
367 	/* Configure common clock before checking latencies */
368 	pcie_aspm_configure_common_clock(link);
369 
370 	/* Get upstream/downstream components' register state */
371 	pcie_get_aspm_reg(parent, &upreg);
372 	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
373 	pcie_get_aspm_reg(child, &dwreg);
374 
375 	/*
376 	 * Setup L0s state
377 	 *
378 	 * Note that we must not enable L0s in either direction on a
379 	 * given link unless components on both sides of the link each
380 	 * support L0s.
381 	 */
382 	if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
383 		link->aspm_support |= ASPM_STATE_L0S;
384 	if (dwreg.enabled & PCIE_LINK_STATE_L0S)
385 		link->aspm_enabled |= ASPM_STATE_L0S_UP;
386 	if (upreg.enabled & PCIE_LINK_STATE_L0S)
387 		link->aspm_enabled |= ASPM_STATE_L0S_DW;
388 	link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
389 	link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
390 
391 	/* Setup L1 state */
392 	if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
393 		link->aspm_support |= ASPM_STATE_L1;
394 	if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
395 		link->aspm_enabled |= ASPM_STATE_L1;
396 	link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
397 	link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
398 
399 	/* Save default state */
400 	link->aspm_default = link->aspm_enabled;
401 
402 	/* Setup initial capable state. Will be updated later */
403 	link->aspm_capable = link->aspm_support;
404 	/*
405 	 * If the downstream component has pci bridge function, don't
406 	 * do ASPM for now.
407 	 */
408 	list_for_each_entry(child, &linkbus->devices, bus_list) {
409 		if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
410 			link->aspm_disable = ASPM_STATE_ALL;
411 			break;
412 		}
413 	}
414 
415 	/* Get and check endpoint acceptable latencies */
416 	list_for_each_entry(child, &linkbus->devices, bus_list) {
417 		int pos;
418 		u32 reg32, encoding;
419 		struct aspm_latency *acceptable =
420 			&link->acceptable[PCI_FUNC(child->devfn)];
421 
422 		if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
423 		    child->pcie_type != PCI_EXP_TYPE_LEG_END)
424 			continue;
425 
426 		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
427 		pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
428 		/* Calculate endpoint L0s acceptable latency */
429 		encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
430 		acceptable->l0s = calc_l0s_acceptable(encoding);
431 		/* Calculate endpoint L1 acceptable latency */
432 		encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
433 		acceptable->l1 = calc_l1_acceptable(encoding);
434 
435 		pcie_aspm_check_latency(child);
436 	}
437 }
438 
439 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
440 {
441 	u16 reg16;
442 	int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
443 
444 	pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
445 	reg16 &= ~0x3;
446 	reg16 |= val;
447 	pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
448 }
449 
450 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
451 {
452 	u32 upstream = 0, dwstream = 0;
453 	struct pci_dev *child, *parent = link->pdev;
454 	struct pci_bus *linkbus = parent->subordinate;
455 
456 	/* Nothing to do if the link is already in the requested state */
457 	state &= (link->aspm_capable & ~link->aspm_disable);
458 	if (link->aspm_enabled == state)
459 		return;
460 	/* Convert ASPM state to upstream/downstream ASPM register state */
461 	if (state & ASPM_STATE_L0S_UP)
462 		dwstream |= PCIE_LINK_STATE_L0S;
463 	if (state & ASPM_STATE_L0S_DW)
464 		upstream |= PCIE_LINK_STATE_L0S;
465 	if (state & ASPM_STATE_L1) {
466 		upstream |= PCIE_LINK_STATE_L1;
467 		dwstream |= PCIE_LINK_STATE_L1;
468 	}
469 	/*
470 	 * Spec 2.0 suggests all functions should be configured the
471 	 * same setting for ASPM. Enabling ASPM L1 should be done in
472 	 * upstream component first and then downstream, and vice
473 	 * versa for disabling ASPM L1. Spec doesn't mention L0S.
474 	 */
475 	if (state & ASPM_STATE_L1)
476 		pcie_config_aspm_dev(parent, upstream);
477 	list_for_each_entry(child, &linkbus->devices, bus_list)
478 		pcie_config_aspm_dev(child, dwstream);
479 	if (!(state & ASPM_STATE_L1))
480 		pcie_config_aspm_dev(parent, upstream);
481 
482 	link->aspm_enabled = state;
483 }
484 
485 static void pcie_config_aspm_path(struct pcie_link_state *link)
486 {
487 	while (link) {
488 		pcie_config_aspm_link(link, policy_to_aspm_state(link));
489 		link = link->parent;
490 	}
491 }
492 
493 static void free_link_state(struct pcie_link_state *link)
494 {
495 	link->pdev->link_state = NULL;
496 	kfree(link);
497 }
498 
499 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
500 {
501 	struct pci_dev *child;
502 	int pos;
503 	u32 reg32;
504 	/*
505 	 * Some functions in a slot might not all be PCIE functions,
506 	 * very strange. Disable ASPM for the whole slot
507 	 */
508 	list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
509 		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
510 		if (!pos)
511 			return -EINVAL;
512 		/*
513 		 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
514 		 * RBER bit to determine if a function is 1.1 version device
515 		 */
516 		pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
517 		if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
518 			dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
519 				" on pre-1.1 PCIe device.  You can enable it"
520 				" with 'pcie_aspm=force'\n");
521 			return -EINVAL;
522 		}
523 	}
524 	return 0;
525 }
526 
527 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
528 {
529 	struct pcie_link_state *link;
530 
531 	link = kzalloc(sizeof(*link), GFP_KERNEL);
532 	if (!link)
533 		return NULL;
534 	INIT_LIST_HEAD(&link->sibling);
535 	INIT_LIST_HEAD(&link->children);
536 	INIT_LIST_HEAD(&link->link);
537 	link->pdev = pdev;
538 	if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
539 		struct pcie_link_state *parent;
540 		parent = pdev->bus->parent->self->link_state;
541 		if (!parent) {
542 			kfree(link);
543 			return NULL;
544 		}
545 		link->parent = parent;
546 		list_add(&link->link, &parent->children);
547 	}
548 	/* Setup a pointer to the root port link */
549 	if (!link->parent)
550 		link->root = link;
551 	else
552 		link->root = link->parent->root;
553 
554 	list_add(&link->sibling, &link_list);
555 	pdev->link_state = link;
556 	return link;
557 }
558 
559 /*
560  * pcie_aspm_init_link_state: Initiate PCI express link state.
561  * It is called after the pcie and its children devices are scaned.
562  * @pdev: the root port or switch downstream port
563  */
564 void pcie_aspm_init_link_state(struct pci_dev *pdev)
565 {
566 	struct pcie_link_state *link;
567 	int blacklist = !!pcie_aspm_sanity_check(pdev);
568 
569 	if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
570 		return;
571 	if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
572 	    pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
573 		return;
574 
575 	/* VIA has a strange chipset, root port is under a bridge */
576 	if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
577 	    pdev->bus->self)
578 		return;
579 
580 	down_read(&pci_bus_sem);
581 	if (list_empty(&pdev->subordinate->devices))
582 		goto out;
583 
584 	mutex_lock(&aspm_lock);
585 	link = alloc_pcie_link_state(pdev);
586 	if (!link)
587 		goto unlock;
588 	/*
589 	 * Setup initial ASPM state. Note that we need to configure
590 	 * upstream links also because capable state of them can be
591 	 * update through pcie_aspm_cap_init().
592 	 */
593 	pcie_aspm_cap_init(link, blacklist);
594 	pcie_config_aspm_path(link);
595 
596 	/* Setup initial Clock PM state */
597 	pcie_clkpm_cap_init(link, blacklist);
598 	pcie_set_clkpm(link, policy_to_clkpm_state(link));
599 unlock:
600 	mutex_unlock(&aspm_lock);
601 out:
602 	up_read(&pci_bus_sem);
603 }
604 
605 /* Recheck latencies and update aspm_capable for links under the root */
606 static void pcie_update_aspm_capable(struct pcie_link_state *root)
607 {
608 	struct pcie_link_state *link;
609 	BUG_ON(root->parent);
610 	list_for_each_entry(link, &link_list, sibling) {
611 		if (link->root != root)
612 			continue;
613 		link->aspm_capable = link->aspm_support;
614 	}
615 	list_for_each_entry(link, &link_list, sibling) {
616 		struct pci_dev *child;
617 		struct pci_bus *linkbus = link->pdev->subordinate;
618 		if (link->root != root)
619 			continue;
620 		list_for_each_entry(child, &linkbus->devices, bus_list) {
621 			if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) &&
622 			    (child->pcie_type != PCI_EXP_TYPE_LEG_END))
623 				continue;
624 			pcie_aspm_check_latency(child);
625 		}
626 	}
627 }
628 
629 /* @pdev: the endpoint device */
630 void pcie_aspm_exit_link_state(struct pci_dev *pdev)
631 {
632 	struct pci_dev *parent = pdev->bus->self;
633 	struct pcie_link_state *link, *root, *parent_link;
634 
635 	if (aspm_disabled || !pdev->is_pcie || !parent || !parent->link_state)
636 		return;
637 	if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
638 	    (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
639 		return;
640 
641 	down_read(&pci_bus_sem);
642 	mutex_lock(&aspm_lock);
643 	/*
644 	 * All PCIe functions are in one slot, remove one function will remove
645 	 * the whole slot, so just wait until we are the last function left.
646 	 */
647 	if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
648 		goto out;
649 
650 	link = parent->link_state;
651 	root = link->root;
652 	parent_link = link->parent;
653 
654 	/* All functions are removed, so just disable ASPM for the link */
655 	pcie_config_aspm_link(link, 0);
656 	list_del(&link->sibling);
657 	list_del(&link->link);
658 	/* Clock PM is for endpoint device */
659 	free_link_state(link);
660 
661 	/* Recheck latencies and configure upstream links */
662 	pcie_update_aspm_capable(root);
663 	pcie_config_aspm_path(parent_link);
664 out:
665 	mutex_unlock(&aspm_lock);
666 	up_read(&pci_bus_sem);
667 }
668 
669 /* @pdev: the root port or switch downstream port */
670 void pcie_aspm_pm_state_change(struct pci_dev *pdev)
671 {
672 	struct pcie_link_state *link = pdev->link_state;
673 
674 	if (aspm_disabled || !pdev->is_pcie || !link)
675 		return;
676 	if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
677 	    (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
678 		return;
679 	/*
680 	 * Devices changed PM state, we should recheck if latency
681 	 * meets all functions' requirement
682 	 */
683 	down_read(&pci_bus_sem);
684 	mutex_lock(&aspm_lock);
685 	pcie_update_aspm_capable(link->root);
686 	pcie_config_aspm_path(link);
687 	mutex_unlock(&aspm_lock);
688 	up_read(&pci_bus_sem);
689 }
690 
691 /*
692  * pci_disable_link_state - disable pci device's link state, so the link will
693  * never enter specific states
694  */
695 void pci_disable_link_state(struct pci_dev *pdev, int state)
696 {
697 	struct pci_dev *parent = pdev->bus->self;
698 	struct pcie_link_state *link;
699 
700 	if (aspm_disabled || !pdev->is_pcie)
701 		return;
702 	if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
703 	    pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
704 		parent = pdev;
705 	if (!parent || !parent->link_state)
706 		return;
707 
708 	down_read(&pci_bus_sem);
709 	mutex_lock(&aspm_lock);
710 	link = parent->link_state;
711 	if (state & PCIE_LINK_STATE_L0S)
712 		link->aspm_disable |= ASPM_STATE_L0S;
713 	if (state & PCIE_LINK_STATE_L1)
714 		link->aspm_disable |= ASPM_STATE_L1;
715 	pcie_config_aspm_link(link, policy_to_aspm_state(link));
716 
717 	if (state & PCIE_LINK_STATE_CLKPM) {
718 		link->clkpm_capable = 0;
719 		pcie_set_clkpm(link, 0);
720 	}
721 	mutex_unlock(&aspm_lock);
722 	up_read(&pci_bus_sem);
723 }
724 EXPORT_SYMBOL(pci_disable_link_state);
725 
726 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
727 {
728 	int i;
729 	struct pcie_link_state *link;
730 
731 	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
732 		if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
733 			break;
734 	if (i >= ARRAY_SIZE(policy_str))
735 		return -EINVAL;
736 	if (i == aspm_policy)
737 		return 0;
738 
739 	down_read(&pci_bus_sem);
740 	mutex_lock(&aspm_lock);
741 	aspm_policy = i;
742 	list_for_each_entry(link, &link_list, sibling) {
743 		pcie_config_aspm_link(link, policy_to_aspm_state(link));
744 		pcie_set_clkpm(link, policy_to_clkpm_state(link));
745 	}
746 	mutex_unlock(&aspm_lock);
747 	up_read(&pci_bus_sem);
748 	return 0;
749 }
750 
751 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
752 {
753 	int i, cnt = 0;
754 	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
755 		if (i == aspm_policy)
756 			cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
757 		else
758 			cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
759 	return cnt;
760 }
761 
762 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
763 	NULL, 0644);
764 
765 #ifdef CONFIG_PCIEASPM_DEBUG
766 static ssize_t link_state_show(struct device *dev,
767 		struct device_attribute *attr,
768 		char *buf)
769 {
770 	struct pci_dev *pci_device = to_pci_dev(dev);
771 	struct pcie_link_state *link_state = pci_device->link_state;
772 
773 	return sprintf(buf, "%d\n", link_state->aspm_enabled);
774 }
775 
776 static ssize_t link_state_store(struct device *dev,
777 		struct device_attribute *attr,
778 		const char *buf,
779 		size_t n)
780 {
781 	struct pci_dev *pdev = to_pci_dev(dev);
782 	struct pcie_link_state *link, *root = pdev->link_state->root;
783 	u32 val = buf[0] - '0', state = 0;
784 
785 	if (n < 1 || val > 3)
786 		return -EINVAL;
787 
788 	/* Convert requested state to ASPM state */
789 	if (val & PCIE_LINK_STATE_L0S)
790 		state |= ASPM_STATE_L0S;
791 	if (val & PCIE_LINK_STATE_L1)
792 		state |= ASPM_STATE_L1;
793 
794 	down_read(&pci_bus_sem);
795 	mutex_lock(&aspm_lock);
796 	list_for_each_entry(link, &link_list, sibling) {
797 		if (link->root != root)
798 			continue;
799 		pcie_config_aspm_link(link, state);
800 	}
801 	mutex_unlock(&aspm_lock);
802 	up_read(&pci_bus_sem);
803 	return n;
804 }
805 
806 static ssize_t clk_ctl_show(struct device *dev,
807 		struct device_attribute *attr,
808 		char *buf)
809 {
810 	struct pci_dev *pci_device = to_pci_dev(dev);
811 	struct pcie_link_state *link_state = pci_device->link_state;
812 
813 	return sprintf(buf, "%d\n", link_state->clkpm_enabled);
814 }
815 
816 static ssize_t clk_ctl_store(struct device *dev,
817 		struct device_attribute *attr,
818 		const char *buf,
819 		size_t n)
820 {
821 	struct pci_dev *pdev = to_pci_dev(dev);
822 	int state;
823 
824 	if (n < 1)
825 		return -EINVAL;
826 	state = buf[0]-'0';
827 
828 	down_read(&pci_bus_sem);
829 	mutex_lock(&aspm_lock);
830 	pcie_set_clkpm_nocheck(pdev->link_state, !!state);
831 	mutex_unlock(&aspm_lock);
832 	up_read(&pci_bus_sem);
833 
834 	return n;
835 }
836 
837 static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
838 static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
839 
840 static char power_group[] = "power";
841 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
842 {
843 	struct pcie_link_state *link_state = pdev->link_state;
844 
845 	if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
846 		pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
847 		return;
848 
849 	if (link_state->aspm_support)
850 		sysfs_add_file_to_group(&pdev->dev.kobj,
851 			&dev_attr_link_state.attr, power_group);
852 	if (link_state->clkpm_capable)
853 		sysfs_add_file_to_group(&pdev->dev.kobj,
854 			&dev_attr_clk_ctl.attr, power_group);
855 }
856 
857 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
858 {
859 	struct pcie_link_state *link_state = pdev->link_state;
860 
861 	if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
862 		pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
863 		return;
864 
865 	if (link_state->aspm_support)
866 		sysfs_remove_file_from_group(&pdev->dev.kobj,
867 			&dev_attr_link_state.attr, power_group);
868 	if (link_state->clkpm_capable)
869 		sysfs_remove_file_from_group(&pdev->dev.kobj,
870 			&dev_attr_clk_ctl.attr, power_group);
871 }
872 #endif
873 
874 static int __init pcie_aspm_disable(char *str)
875 {
876 	if (!strcmp(str, "off")) {
877 		aspm_disabled = 1;
878 		printk(KERN_INFO "PCIe ASPM is disabled\n");
879 	} else if (!strcmp(str, "force")) {
880 		aspm_force = 1;
881 		printk(KERN_INFO "PCIe ASPM is forcedly enabled\n");
882 	}
883 	return 1;
884 }
885 
886 __setup("pcie_aspm=", pcie_aspm_disable);
887 
888 void pcie_no_aspm(void)
889 {
890 	if (!aspm_force)
891 		aspm_disabled = 1;
892 }
893 
894 /**
895  * pcie_aspm_enabled - is PCIe ASPM enabled?
896  *
897  * Returns true if ASPM has not been disabled by the command-line option
898  * pcie_aspm=off.
899  **/
900 int pcie_aspm_enabled(void)
901 {
902        return !aspm_disabled;
903 }
904 EXPORT_SYMBOL(pcie_aspm_enabled);
905 
906