1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2019 Intel Corporation. 4 * 5 */ 6 7 #include "aspm.h" 8 9 /* Time after which the timer interrupt will re-enable ASPM */ 10 #define ASPM_TIMER_MS 1000 11 /* Time for which interrupts are ignored after a timer has been scheduled */ 12 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) 13 /* Two interrupts within this time trigger ASPM disable */ 14 #define ASPM_TRIGGER_MS 1 15 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) 16 #define ASPM_L1_SUPPORTED(reg) \ 17 ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) 18 19 uint aspm_mode = ASPM_MODE_DISABLED; 20 module_param_named(aspm, aspm_mode, uint, 0444); 21 MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic"); 22 23 static bool aspm_hw_l1_supported(struct hfi1_devdata *dd) 24 { 25 struct pci_dev *parent = dd->pcidev->bus->self; 26 u32 up, dn; 27 28 /* 29 * If the driver does not have access to the upstream component, 30 * it cannot support ASPM L1 at all. 31 */ 32 if (!parent) 33 return false; 34 35 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); 36 dn = ASPM_L1_SUPPORTED(dn); 37 38 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); 39 up = ASPM_L1_SUPPORTED(up); 40 41 /* ASPM works on A-step but is reported as not supported */ 42 return (!!dn || is_ax(dd)) && !!up; 43 } 44 45 /* Set L1 entrance latency for slower entry to L1 */ 46 static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) 47 { 48 u32 l1_ent_lat = 0x4u; 49 u32 reg32; 50 51 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); 52 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; 53 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; 54 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); 55 } 56 57 static void aspm_hw_enable_l1(struct hfi1_devdata *dd) 58 { 59 struct pci_dev *parent = dd->pcidev->bus->self; 60 61 /* 62 * If the driver does not have access to the upstream component, 63 * it cannot support ASPM L1 at all. 64 */ 65 if (!parent) 66 return; 67 68 /* Enable ASPM L1 first in upstream component and then downstream */ 69 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 70 PCI_EXP_LNKCTL_ASPMC, 71 PCI_EXP_LNKCTL_ASPM_L1); 72 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, 73 PCI_EXP_LNKCTL_ASPMC, 74 PCI_EXP_LNKCTL_ASPM_L1); 75 } 76 77 void aspm_hw_disable_l1(struct hfi1_devdata *dd) 78 { 79 struct pci_dev *parent = dd->pcidev->bus->self; 80 81 /* Disable ASPM L1 first in downstream component and then upstream */ 82 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, 83 PCI_EXP_LNKCTL_ASPMC, 0x0); 84 if (parent) 85 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 86 PCI_EXP_LNKCTL_ASPMC, 0x0); 87 } 88 89 static void aspm_enable(struct hfi1_devdata *dd) 90 { 91 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || 92 !dd->aspm_supported) 93 return; 94 95 aspm_hw_enable_l1(dd); 96 dd->aspm_enabled = true; 97 } 98 99 static void aspm_disable(struct hfi1_devdata *dd) 100 { 101 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) 102 return; 103 104 aspm_hw_disable_l1(dd); 105 dd->aspm_enabled = false; 106 } 107 108 static void aspm_disable_inc(struct hfi1_devdata *dd) 109 { 110 unsigned long flags; 111 112 spin_lock_irqsave(&dd->aspm_lock, flags); 113 aspm_disable(dd); 114 atomic_inc(&dd->aspm_disabled_cnt); 115 spin_unlock_irqrestore(&dd->aspm_lock, flags); 116 } 117 118 static void aspm_enable_dec(struct hfi1_devdata *dd) 119 { 120 unsigned long flags; 121 122 spin_lock_irqsave(&dd->aspm_lock, flags); 123 if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) 124 aspm_enable(dd); 125 spin_unlock_irqrestore(&dd->aspm_lock, flags); 126 } 127 128 /* ASPM processing for each receive context interrupt */ 129 void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd) 130 { 131 bool restart_timer; 132 bool close_interrupts; 133 unsigned long flags; 134 ktime_t now, prev; 135 136 spin_lock_irqsave(&rcd->aspm_lock, flags); 137 /* PSM contexts are open */ 138 if (!rcd->aspm_intr_enable) 139 goto unlock; 140 141 prev = rcd->aspm_ts_last_intr; 142 now = ktime_get(); 143 rcd->aspm_ts_last_intr = now; 144 145 /* An interrupt pair close together in time */ 146 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; 147 148 /* Don't push out our timer till this much time has elapsed */ 149 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > 150 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; 151 restart_timer = restart_timer && close_interrupts; 152 153 /* Disable ASPM and schedule timer */ 154 if (rcd->aspm_enabled && close_interrupts) { 155 aspm_disable_inc(rcd->dd); 156 rcd->aspm_enabled = false; 157 restart_timer = true; 158 } 159 160 if (restart_timer) { 161 mod_timer(&rcd->aspm_timer, 162 jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); 163 rcd->aspm_ts_timer_sched = now; 164 } 165 unlock: 166 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 167 } 168 169 /* Timer function for re-enabling ASPM in the absence of interrupt activity */ 170 static void aspm_ctx_timer_function(struct timer_list *t) 171 { 172 struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); 173 unsigned long flags; 174 175 spin_lock_irqsave(&rcd->aspm_lock, flags); 176 aspm_enable_dec(rcd->dd); 177 rcd->aspm_enabled = true; 178 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 179 } 180 181 /* 182 * Disable interrupt processing for verbs contexts when PSM or VNIC contexts 183 * are open. 184 */ 185 void aspm_disable_all(struct hfi1_devdata *dd) 186 { 187 struct hfi1_ctxtdata *rcd; 188 unsigned long flags; 189 u16 i; 190 191 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { 192 rcd = hfi1_rcd_get_by_index(dd, i); 193 if (rcd) { 194 del_timer_sync(&rcd->aspm_timer); 195 spin_lock_irqsave(&rcd->aspm_lock, flags); 196 rcd->aspm_intr_enable = false; 197 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 198 hfi1_rcd_put(rcd); 199 } 200 } 201 202 aspm_disable(dd); 203 atomic_set(&dd->aspm_disabled_cnt, 0); 204 } 205 206 /* Re-enable interrupt processing for verbs contexts */ 207 void aspm_enable_all(struct hfi1_devdata *dd) 208 { 209 struct hfi1_ctxtdata *rcd; 210 unsigned long flags; 211 u16 i; 212 213 aspm_enable(dd); 214 215 if (aspm_mode != ASPM_MODE_DYNAMIC) 216 return; 217 218 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { 219 rcd = hfi1_rcd_get_by_index(dd, i); 220 if (rcd) { 221 spin_lock_irqsave(&rcd->aspm_lock, flags); 222 rcd->aspm_intr_enable = true; 223 rcd->aspm_enabled = true; 224 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 225 hfi1_rcd_put(rcd); 226 } 227 } 228 } 229 230 static void aspm_ctx_init(struct hfi1_ctxtdata *rcd) 231 { 232 spin_lock_init(&rcd->aspm_lock); 233 timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); 234 rcd->aspm_intr_supported = rcd->dd->aspm_supported && 235 aspm_mode == ASPM_MODE_DYNAMIC && 236 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; 237 } 238 239 void aspm_init(struct hfi1_devdata *dd) 240 { 241 struct hfi1_ctxtdata *rcd; 242 u16 i; 243 244 spin_lock_init(&dd->aspm_lock); 245 dd->aspm_supported = aspm_hw_l1_supported(dd); 246 247 for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { 248 rcd = hfi1_rcd_get_by_index(dd, i); 249 if (rcd) 250 aspm_ctx_init(rcd); 251 hfi1_rcd_put(rcd); 252 } 253 254 /* Start with ASPM disabled */ 255 aspm_hw_set_l1_ent_latency(dd); 256 dd->aspm_enabled = false; 257 aspm_hw_disable_l1(dd); 258 259 /* Now turn on ASPM if configured */ 260 aspm_enable_all(dd); 261 } 262 263 void aspm_exit(struct hfi1_devdata *dd) 264 { 265 aspm_disable_all(dd); 266 267 /* Turn on ASPM on exit to conserve power */ 268 aspm_enable(dd); 269 } 270 271