1bf3b1e0cSMichael J. Ruhl // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2bf3b1e0cSMichael J. Ruhl /*
3bf3b1e0cSMichael J. Ruhl * Copyright(c) 2019 Intel Corporation.
4bf3b1e0cSMichael J. Ruhl *
5bf3b1e0cSMichael J. Ruhl */
6bf3b1e0cSMichael J. Ruhl
7bf3b1e0cSMichael J. Ruhl #include "aspm.h"
8bf3b1e0cSMichael J. Ruhl
9bf3b1e0cSMichael J. Ruhl /* Time after which the timer interrupt will re-enable ASPM */
10bf3b1e0cSMichael J. Ruhl #define ASPM_TIMER_MS 1000
11bf3b1e0cSMichael J. Ruhl /* Time for which interrupts are ignored after a timer has been scheduled */
12bf3b1e0cSMichael J. Ruhl #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
13bf3b1e0cSMichael J. Ruhl /* Two interrupts within this time trigger ASPM disable */
14bf3b1e0cSMichael J. Ruhl #define ASPM_TRIGGER_MS 1
15bf3b1e0cSMichael J. Ruhl #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
16bf3b1e0cSMichael J. Ruhl #define ASPM_L1_SUPPORTED(reg) \
17bf3b1e0cSMichael J. Ruhl ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
18bf3b1e0cSMichael J. Ruhl
19bf3b1e0cSMichael J. Ruhl uint aspm_mode = ASPM_MODE_DISABLED;
20bf3b1e0cSMichael J. Ruhl module_param_named(aspm, aspm_mode, uint, 0444);
21bf3b1e0cSMichael J. Ruhl MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
22bf3b1e0cSMichael J. Ruhl
aspm_hw_l1_supported(struct hfi1_devdata * dd)23bf3b1e0cSMichael J. Ruhl static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
24bf3b1e0cSMichael J. Ruhl {
25bf3b1e0cSMichael J. Ruhl struct pci_dev *parent = dd->pcidev->bus->self;
26bf3b1e0cSMichael J. Ruhl u32 up, dn;
27bf3b1e0cSMichael J. Ruhl
28bf3b1e0cSMichael J. Ruhl /*
29bf3b1e0cSMichael J. Ruhl * If the driver does not have access to the upstream component,
30bf3b1e0cSMichael J. Ruhl * it cannot support ASPM L1 at all.
31bf3b1e0cSMichael J. Ruhl */
32bf3b1e0cSMichael J. Ruhl if (!parent)
33bf3b1e0cSMichael J. Ruhl return false;
34bf3b1e0cSMichael J. Ruhl
35bf3b1e0cSMichael J. Ruhl pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
36bf3b1e0cSMichael J. Ruhl dn = ASPM_L1_SUPPORTED(dn);
37bf3b1e0cSMichael J. Ruhl
38bf3b1e0cSMichael J. Ruhl pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
39bf3b1e0cSMichael J. Ruhl up = ASPM_L1_SUPPORTED(up);
40bf3b1e0cSMichael J. Ruhl
41bf3b1e0cSMichael J. Ruhl /* ASPM works on A-step but is reported as not supported */
42bf3b1e0cSMichael J. Ruhl return (!!dn || is_ax(dd)) && !!up;
43bf3b1e0cSMichael J. Ruhl }
44bf3b1e0cSMichael J. Ruhl
45bf3b1e0cSMichael J. Ruhl /* Set L1 entrance latency for slower entry to L1 */
aspm_hw_set_l1_ent_latency(struct hfi1_devdata * dd)46bf3b1e0cSMichael J. Ruhl static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
47bf3b1e0cSMichael J. Ruhl {
48bf3b1e0cSMichael J. Ruhl u32 l1_ent_lat = 0x4u;
49bf3b1e0cSMichael J. Ruhl u32 reg32;
50bf3b1e0cSMichael J. Ruhl
51bf3b1e0cSMichael J. Ruhl pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32);
52bf3b1e0cSMichael J. Ruhl reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
53bf3b1e0cSMichael J. Ruhl reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
54bf3b1e0cSMichael J. Ruhl pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
55bf3b1e0cSMichael J. Ruhl }
56bf3b1e0cSMichael J. Ruhl
aspm_hw_enable_l1(struct hfi1_devdata * dd)57bf3b1e0cSMichael J. Ruhl static void aspm_hw_enable_l1(struct hfi1_devdata *dd)
58bf3b1e0cSMichael J. Ruhl {
59bf3b1e0cSMichael J. Ruhl struct pci_dev *parent = dd->pcidev->bus->self;
60bf3b1e0cSMichael J. Ruhl
61bf3b1e0cSMichael J. Ruhl /*
62bf3b1e0cSMichael J. Ruhl * If the driver does not have access to the upstream component,
63bf3b1e0cSMichael J. Ruhl * it cannot support ASPM L1 at all.
64bf3b1e0cSMichael J. Ruhl */
65bf3b1e0cSMichael J. Ruhl if (!parent)
66bf3b1e0cSMichael J. Ruhl return;
67bf3b1e0cSMichael J. Ruhl
68bf3b1e0cSMichael J. Ruhl /* Enable ASPM L1 first in upstream component and then downstream */
69bf3b1e0cSMichael J. Ruhl pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
70bf3b1e0cSMichael J. Ruhl PCI_EXP_LNKCTL_ASPMC,
71bf3b1e0cSMichael J. Ruhl PCI_EXP_LNKCTL_ASPM_L1);
72bf3b1e0cSMichael J. Ruhl pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
73bf3b1e0cSMichael J. Ruhl PCI_EXP_LNKCTL_ASPMC,
74bf3b1e0cSMichael J. Ruhl PCI_EXP_LNKCTL_ASPM_L1);
75bf3b1e0cSMichael J. Ruhl }
76bf3b1e0cSMichael J. Ruhl
aspm_hw_disable_l1(struct hfi1_devdata * dd)77bf3b1e0cSMichael J. Ruhl void aspm_hw_disable_l1(struct hfi1_devdata *dd)
78bf3b1e0cSMichael J. Ruhl {
79bf3b1e0cSMichael J. Ruhl struct pci_dev *parent = dd->pcidev->bus->self;
80bf3b1e0cSMichael J. Ruhl
81bf3b1e0cSMichael J. Ruhl /* Disable ASPM L1 first in downstream component and then upstream */
82bf3b1e0cSMichael J. Ruhl pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
83bf3b1e0cSMichael J. Ruhl PCI_EXP_LNKCTL_ASPMC, 0x0);
84bf3b1e0cSMichael J. Ruhl if (parent)
85bf3b1e0cSMichael J. Ruhl pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
86bf3b1e0cSMichael J. Ruhl PCI_EXP_LNKCTL_ASPMC, 0x0);
87bf3b1e0cSMichael J. Ruhl }
88bf3b1e0cSMichael J. Ruhl
aspm_enable(struct hfi1_devdata * dd)89bf3b1e0cSMichael J. Ruhl static void aspm_enable(struct hfi1_devdata *dd)
90bf3b1e0cSMichael J. Ruhl {
91bf3b1e0cSMichael J. Ruhl if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
92bf3b1e0cSMichael J. Ruhl !dd->aspm_supported)
93bf3b1e0cSMichael J. Ruhl return;
94bf3b1e0cSMichael J. Ruhl
95bf3b1e0cSMichael J. Ruhl aspm_hw_enable_l1(dd);
96bf3b1e0cSMichael J. Ruhl dd->aspm_enabled = true;
97bf3b1e0cSMichael J. Ruhl }
98bf3b1e0cSMichael J. Ruhl
aspm_disable(struct hfi1_devdata * dd)99bf3b1e0cSMichael J. Ruhl static void aspm_disable(struct hfi1_devdata *dd)
100bf3b1e0cSMichael J. Ruhl {
101bf3b1e0cSMichael J. Ruhl if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
102bf3b1e0cSMichael J. Ruhl return;
103bf3b1e0cSMichael J. Ruhl
104bf3b1e0cSMichael J. Ruhl aspm_hw_disable_l1(dd);
105bf3b1e0cSMichael J. Ruhl dd->aspm_enabled = false;
106bf3b1e0cSMichael J. Ruhl }
107bf3b1e0cSMichael J. Ruhl
aspm_disable_inc(struct hfi1_devdata * dd)108bf3b1e0cSMichael J. Ruhl static void aspm_disable_inc(struct hfi1_devdata *dd)
109bf3b1e0cSMichael J. Ruhl {
110bf3b1e0cSMichael J. Ruhl unsigned long flags;
111bf3b1e0cSMichael J. Ruhl
112bf3b1e0cSMichael J. Ruhl spin_lock_irqsave(&dd->aspm_lock, flags);
113bf3b1e0cSMichael J. Ruhl aspm_disable(dd);
114bf3b1e0cSMichael J. Ruhl atomic_inc(&dd->aspm_disabled_cnt);
115bf3b1e0cSMichael J. Ruhl spin_unlock_irqrestore(&dd->aspm_lock, flags);
116bf3b1e0cSMichael J. Ruhl }
117bf3b1e0cSMichael J. Ruhl
aspm_enable_dec(struct hfi1_devdata * dd)118bf3b1e0cSMichael J. Ruhl static void aspm_enable_dec(struct hfi1_devdata *dd)
119bf3b1e0cSMichael J. Ruhl {
120bf3b1e0cSMichael J. Ruhl unsigned long flags;
121bf3b1e0cSMichael J. Ruhl
122bf3b1e0cSMichael J. Ruhl spin_lock_irqsave(&dd->aspm_lock, flags);
123bf3b1e0cSMichael J. Ruhl if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
124bf3b1e0cSMichael J. Ruhl aspm_enable(dd);
125bf3b1e0cSMichael J. Ruhl spin_unlock_irqrestore(&dd->aspm_lock, flags);
126bf3b1e0cSMichael J. Ruhl }
127bf3b1e0cSMichael J. Ruhl
128bf3b1e0cSMichael J. Ruhl /* ASPM processing for each receive context interrupt */
__aspm_ctx_disable(struct hfi1_ctxtdata * rcd)129bf3b1e0cSMichael J. Ruhl void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
130bf3b1e0cSMichael J. Ruhl {
131bf3b1e0cSMichael J. Ruhl bool restart_timer;
132bf3b1e0cSMichael J. Ruhl bool close_interrupts;
133bf3b1e0cSMichael J. Ruhl unsigned long flags;
134bf3b1e0cSMichael J. Ruhl ktime_t now, prev;
135bf3b1e0cSMichael J. Ruhl
136bf3b1e0cSMichael J. Ruhl spin_lock_irqsave(&rcd->aspm_lock, flags);
137bf3b1e0cSMichael J. Ruhl /* PSM contexts are open */
138bf3b1e0cSMichael J. Ruhl if (!rcd->aspm_intr_enable)
139bf3b1e0cSMichael J. Ruhl goto unlock;
140bf3b1e0cSMichael J. Ruhl
141bf3b1e0cSMichael J. Ruhl prev = rcd->aspm_ts_last_intr;
142bf3b1e0cSMichael J. Ruhl now = ktime_get();
143bf3b1e0cSMichael J. Ruhl rcd->aspm_ts_last_intr = now;
144bf3b1e0cSMichael J. Ruhl
145bf3b1e0cSMichael J. Ruhl /* An interrupt pair close together in time */
146bf3b1e0cSMichael J. Ruhl close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
147bf3b1e0cSMichael J. Ruhl
148bf3b1e0cSMichael J. Ruhl /* Don't push out our timer till this much time has elapsed */
149bf3b1e0cSMichael J. Ruhl restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
150bf3b1e0cSMichael J. Ruhl ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
151bf3b1e0cSMichael J. Ruhl restart_timer = restart_timer && close_interrupts;
152bf3b1e0cSMichael J. Ruhl
153bf3b1e0cSMichael J. Ruhl /* Disable ASPM and schedule timer */
154bf3b1e0cSMichael J. Ruhl if (rcd->aspm_enabled && close_interrupts) {
155bf3b1e0cSMichael J. Ruhl aspm_disable_inc(rcd->dd);
156bf3b1e0cSMichael J. Ruhl rcd->aspm_enabled = false;
157bf3b1e0cSMichael J. Ruhl restart_timer = true;
158bf3b1e0cSMichael J. Ruhl }
159bf3b1e0cSMichael J. Ruhl
160bf3b1e0cSMichael J. Ruhl if (restart_timer) {
161bf3b1e0cSMichael J. Ruhl mod_timer(&rcd->aspm_timer,
162bf3b1e0cSMichael J. Ruhl jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
163bf3b1e0cSMichael J. Ruhl rcd->aspm_ts_timer_sched = now;
164bf3b1e0cSMichael J. Ruhl }
165bf3b1e0cSMichael J. Ruhl unlock:
166bf3b1e0cSMichael J. Ruhl spin_unlock_irqrestore(&rcd->aspm_lock, flags);
167bf3b1e0cSMichael J. Ruhl }
168bf3b1e0cSMichael J. Ruhl
169bf3b1e0cSMichael J. Ruhl /* Timer function for re-enabling ASPM in the absence of interrupt activity */
aspm_ctx_timer_function(struct timer_list * t)170bf3b1e0cSMichael J. Ruhl static void aspm_ctx_timer_function(struct timer_list *t)
171bf3b1e0cSMichael J. Ruhl {
172bf3b1e0cSMichael J. Ruhl struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
173bf3b1e0cSMichael J. Ruhl unsigned long flags;
174bf3b1e0cSMichael J. Ruhl
175bf3b1e0cSMichael J. Ruhl spin_lock_irqsave(&rcd->aspm_lock, flags);
176bf3b1e0cSMichael J. Ruhl aspm_enable_dec(rcd->dd);
177bf3b1e0cSMichael J. Ruhl rcd->aspm_enabled = true;
178bf3b1e0cSMichael J. Ruhl spin_unlock_irqrestore(&rcd->aspm_lock, flags);
179bf3b1e0cSMichael J. Ruhl }
180bf3b1e0cSMichael J. Ruhl
181bf3b1e0cSMichael J. Ruhl /*
182bf3b1e0cSMichael J. Ruhl * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
183bf3b1e0cSMichael J. Ruhl * are open.
184bf3b1e0cSMichael J. Ruhl */
aspm_disable_all(struct hfi1_devdata * dd)185bf3b1e0cSMichael J. Ruhl void aspm_disable_all(struct hfi1_devdata *dd)
186bf3b1e0cSMichael J. Ruhl {
187bf3b1e0cSMichael J. Ruhl struct hfi1_ctxtdata *rcd;
188bf3b1e0cSMichael J. Ruhl unsigned long flags;
189bf3b1e0cSMichael J. Ruhl u16 i;
190bf3b1e0cSMichael J. Ruhl
191bf3b1e0cSMichael J. Ruhl for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
192bf3b1e0cSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
193bf3b1e0cSMichael J. Ruhl if (rcd) {
194bf3b1e0cSMichael J. Ruhl del_timer_sync(&rcd->aspm_timer);
195bf3b1e0cSMichael J. Ruhl spin_lock_irqsave(&rcd->aspm_lock, flags);
196bf3b1e0cSMichael J. Ruhl rcd->aspm_intr_enable = false;
197bf3b1e0cSMichael J. Ruhl spin_unlock_irqrestore(&rcd->aspm_lock, flags);
198bf3b1e0cSMichael J. Ruhl hfi1_rcd_put(rcd);
199bf3b1e0cSMichael J. Ruhl }
200bf3b1e0cSMichael J. Ruhl }
201bf3b1e0cSMichael J. Ruhl
202bf3b1e0cSMichael J. Ruhl aspm_disable(dd);
203bf3b1e0cSMichael J. Ruhl atomic_set(&dd->aspm_disabled_cnt, 0);
204bf3b1e0cSMichael J. Ruhl }
205bf3b1e0cSMichael J. Ruhl
206bf3b1e0cSMichael J. Ruhl /* Re-enable interrupt processing for verbs contexts */
aspm_enable_all(struct hfi1_devdata * dd)207bf3b1e0cSMichael J. Ruhl void aspm_enable_all(struct hfi1_devdata *dd)
208bf3b1e0cSMichael J. Ruhl {
209bf3b1e0cSMichael J. Ruhl struct hfi1_ctxtdata *rcd;
210bf3b1e0cSMichael J. Ruhl unsigned long flags;
211bf3b1e0cSMichael J. Ruhl u16 i;
212bf3b1e0cSMichael J. Ruhl
213bf3b1e0cSMichael J. Ruhl aspm_enable(dd);
214bf3b1e0cSMichael J. Ruhl
215bf3b1e0cSMichael J. Ruhl if (aspm_mode != ASPM_MODE_DYNAMIC)
216bf3b1e0cSMichael J. Ruhl return;
217bf3b1e0cSMichael J. Ruhl
218bf3b1e0cSMichael J. Ruhl for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
219bf3b1e0cSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
220bf3b1e0cSMichael J. Ruhl if (rcd) {
221bf3b1e0cSMichael J. Ruhl spin_lock_irqsave(&rcd->aspm_lock, flags);
222bf3b1e0cSMichael J. Ruhl rcd->aspm_intr_enable = true;
223bf3b1e0cSMichael J. Ruhl rcd->aspm_enabled = true;
224bf3b1e0cSMichael J. Ruhl spin_unlock_irqrestore(&rcd->aspm_lock, flags);
225bf3b1e0cSMichael J. Ruhl hfi1_rcd_put(rcd);
226bf3b1e0cSMichael J. Ruhl }
227bf3b1e0cSMichael J. Ruhl }
228bf3b1e0cSMichael J. Ruhl }
229bf3b1e0cSMichael J. Ruhl
aspm_ctx_init(struct hfi1_ctxtdata * rcd)230bf3b1e0cSMichael J. Ruhl static void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
231bf3b1e0cSMichael J. Ruhl {
232bf3b1e0cSMichael J. Ruhl spin_lock_init(&rcd->aspm_lock);
233bf3b1e0cSMichael J. Ruhl timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
234bf3b1e0cSMichael J. Ruhl rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
235bf3b1e0cSMichael J. Ruhl aspm_mode == ASPM_MODE_DYNAMIC &&
236bf3b1e0cSMichael J. Ruhl rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
237bf3b1e0cSMichael J. Ruhl }
238bf3b1e0cSMichael J. Ruhl
aspm_init(struct hfi1_devdata * dd)239bf3b1e0cSMichael J. Ruhl void aspm_init(struct hfi1_devdata *dd)
240bf3b1e0cSMichael J. Ruhl {
241bf3b1e0cSMichael J. Ruhl struct hfi1_ctxtdata *rcd;
242bf3b1e0cSMichael J. Ruhl u16 i;
243bf3b1e0cSMichael J. Ruhl
244bf3b1e0cSMichael J. Ruhl spin_lock_init(&dd->aspm_lock);
245bf3b1e0cSMichael J. Ruhl dd->aspm_supported = aspm_hw_l1_supported(dd);
246bf3b1e0cSMichael J. Ruhl
247bf3b1e0cSMichael J. Ruhl for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
248bf3b1e0cSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
249bf3b1e0cSMichael J. Ruhl if (rcd)
250bf3b1e0cSMichael J. Ruhl aspm_ctx_init(rcd);
251bf3b1e0cSMichael J. Ruhl hfi1_rcd_put(rcd);
252bf3b1e0cSMichael J. Ruhl }
253bf3b1e0cSMichael J. Ruhl
254bf3b1e0cSMichael J. Ruhl /* Start with ASPM disabled */
255bf3b1e0cSMichael J. Ruhl aspm_hw_set_l1_ent_latency(dd);
256bf3b1e0cSMichael J. Ruhl dd->aspm_enabled = false;
257bf3b1e0cSMichael J. Ruhl aspm_hw_disable_l1(dd);
258bf3b1e0cSMichael J. Ruhl
259bf3b1e0cSMichael J. Ruhl /* Now turn on ASPM if configured */
260bf3b1e0cSMichael J. Ruhl aspm_enable_all(dd);
261bf3b1e0cSMichael J. Ruhl }
262bf3b1e0cSMichael J. Ruhl
aspm_exit(struct hfi1_devdata * dd)263bf3b1e0cSMichael J. Ruhl void aspm_exit(struct hfi1_devdata *dd)
264bf3b1e0cSMichael J. Ruhl {
265bf3b1e0cSMichael J. Ruhl aspm_disable_all(dd);
266bf3b1e0cSMichael J. Ruhl
267bf3b1e0cSMichael J. Ruhl /* Turn on ASPM on exit to conserve power */
268bf3b1e0cSMichael J. Ruhl aspm_enable(dd);
269bf3b1e0cSMichael J. Ruhl }
270bf3b1e0cSMichael J. Ruhl
271