1 /* 2 * Copyright(c) 2015, 2016 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 #ifndef _ASPM_H 48 #define _ASPM_H 49 50 #include "hfi.h" 51 52 extern uint aspm_mode; 53 54 enum aspm_mode { 55 ASPM_MODE_DISABLED = 0, /* ASPM always disabled, performance mode */ 56 ASPM_MODE_ENABLED = 1, /* ASPM always enabled, power saving mode */ 57 ASPM_MODE_DYNAMIC = 2, /* ASPM enabled/disabled dynamically */ 58 }; 59 60 /* Time after which the timer interrupt will re-enable ASPM */ 61 #define ASPM_TIMER_MS 1000 62 /* Time for which interrupts are ignored after a timer has been scheduled */ 63 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) 64 /* Two interrupts within this time trigger ASPM disable */ 65 #define ASPM_TRIGGER_MS 1 66 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) 67 #define ASPM_L1_SUPPORTED(reg) \ 68 (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) 69 70 static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd) 71 { 72 struct pci_dev *parent = dd->pcidev->bus->self; 73 u32 up, dn; 74 75 /* 76 * If the driver does not have access to the upstream component, 77 * it cannot support ASPM L1 at all. 78 */ 79 if (!parent) 80 return false; 81 82 pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); 83 dn = ASPM_L1_SUPPORTED(dn); 84 85 pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); 86 up = ASPM_L1_SUPPORTED(up); 87 88 /* ASPM works on A-step but is reported as not supported */ 89 return (!!dn || is_ax(dd)) && !!up; 90 } 91 92 /* Set L1 entrance latency for slower entry to L1 */ 93 static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) 94 { 95 u32 l1_ent_lat = 0x4u; 96 u32 reg32; 97 98 pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); 99 reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; 100 reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; 101 pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); 102 } 103 104 static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd) 105 { 106 struct pci_dev *parent = dd->pcidev->bus->self; 107 108 /* 109 * If the driver does not have access to the upstream component, 110 * it cannot support ASPM L1 at all. 111 */ 112 if (!parent) 113 return; 114 115 /* Enable ASPM L1 first in upstream component and then downstream */ 116 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 117 PCI_EXP_LNKCTL_ASPMC, 118 PCI_EXP_LNKCTL_ASPM_L1); 119 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, 120 PCI_EXP_LNKCTL_ASPMC, 121 PCI_EXP_LNKCTL_ASPM_L1); 122 } 123 124 static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd) 125 { 126 struct pci_dev *parent = dd->pcidev->bus->self; 127 128 /* Disable ASPM L1 first in downstream component and then upstream */ 129 pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, 130 PCI_EXP_LNKCTL_ASPMC, 0x0); 131 if (parent) 132 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, 133 PCI_EXP_LNKCTL_ASPMC, 0x0); 134 } 135 136 static inline void aspm_enable(struct hfi1_devdata *dd) 137 { 138 if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || 139 !dd->aspm_supported) 140 return; 141 142 aspm_hw_enable_l1(dd); 143 dd->aspm_enabled = true; 144 } 145 146 static inline void aspm_disable(struct hfi1_devdata *dd) 147 { 148 if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) 149 return; 150 151 aspm_hw_disable_l1(dd); 152 dd->aspm_enabled = false; 153 } 154 155 static inline void aspm_disable_inc(struct hfi1_devdata *dd) 156 { 157 unsigned long flags; 158 159 spin_lock_irqsave(&dd->aspm_lock, flags); 160 aspm_disable(dd); 161 atomic_inc(&dd->aspm_disabled_cnt); 162 spin_unlock_irqrestore(&dd->aspm_lock, flags); 163 } 164 165 static inline void aspm_enable_dec(struct hfi1_devdata *dd) 166 { 167 unsigned long flags; 168 169 spin_lock_irqsave(&dd->aspm_lock, flags); 170 if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) 171 aspm_enable(dd); 172 spin_unlock_irqrestore(&dd->aspm_lock, flags); 173 } 174 175 /* ASPM processing for each receive context interrupt */ 176 static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd) 177 { 178 bool restart_timer; 179 bool close_interrupts; 180 unsigned long flags; 181 ktime_t now, prev; 182 183 /* Quickest exit for minimum impact */ 184 if (!rcd->aspm_intr_supported) 185 return; 186 187 spin_lock_irqsave(&rcd->aspm_lock, flags); 188 /* PSM contexts are open */ 189 if (!rcd->aspm_intr_enable) 190 goto unlock; 191 192 prev = rcd->aspm_ts_last_intr; 193 now = ktime_get(); 194 rcd->aspm_ts_last_intr = now; 195 196 /* An interrupt pair close together in time */ 197 close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; 198 199 /* Don't push out our timer till this much time has elapsed */ 200 restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > 201 ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; 202 restart_timer = restart_timer && close_interrupts; 203 204 /* Disable ASPM and schedule timer */ 205 if (rcd->aspm_enabled && close_interrupts) { 206 aspm_disable_inc(rcd->dd); 207 rcd->aspm_enabled = false; 208 restart_timer = true; 209 } 210 211 if (restart_timer) { 212 mod_timer(&rcd->aspm_timer, 213 jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); 214 rcd->aspm_ts_timer_sched = now; 215 } 216 unlock: 217 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 218 } 219 220 /* Timer function for re-enabling ASPM in the absence of interrupt activity */ 221 static inline void aspm_ctx_timer_function(unsigned long data) 222 { 223 struct hfi1_ctxtdata *rcd = (struct hfi1_ctxtdata *)data; 224 unsigned long flags; 225 226 spin_lock_irqsave(&rcd->aspm_lock, flags); 227 aspm_enable_dec(rcd->dd); 228 rcd->aspm_enabled = true; 229 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 230 } 231 232 /* Disable interrupt processing for verbs contexts when PSM contexts are open */ 233 static inline void aspm_disable_all(struct hfi1_devdata *dd) 234 { 235 struct hfi1_ctxtdata *rcd; 236 unsigned long flags; 237 unsigned i; 238 239 for (i = 0; i < dd->first_user_ctxt; i++) { 240 rcd = dd->rcd[i]; 241 del_timer_sync(&rcd->aspm_timer); 242 spin_lock_irqsave(&rcd->aspm_lock, flags); 243 rcd->aspm_intr_enable = false; 244 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 245 } 246 247 aspm_disable(dd); 248 atomic_set(&dd->aspm_disabled_cnt, 0); 249 } 250 251 /* Re-enable interrupt processing for verbs contexts */ 252 static inline void aspm_enable_all(struct hfi1_devdata *dd) 253 { 254 struct hfi1_ctxtdata *rcd; 255 unsigned long flags; 256 unsigned i; 257 258 aspm_enable(dd); 259 260 if (aspm_mode != ASPM_MODE_DYNAMIC) 261 return; 262 263 for (i = 0; i < dd->first_user_ctxt; i++) { 264 rcd = dd->rcd[i]; 265 spin_lock_irqsave(&rcd->aspm_lock, flags); 266 rcd->aspm_intr_enable = true; 267 rcd->aspm_enabled = true; 268 spin_unlock_irqrestore(&rcd->aspm_lock, flags); 269 } 270 } 271 272 static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd) 273 { 274 spin_lock_init(&rcd->aspm_lock); 275 setup_timer(&rcd->aspm_timer, aspm_ctx_timer_function, 276 (unsigned long)rcd); 277 rcd->aspm_intr_supported = rcd->dd->aspm_supported && 278 aspm_mode == ASPM_MODE_DYNAMIC && 279 rcd->ctxt < rcd->dd->first_user_ctxt; 280 } 281 282 static inline void aspm_init(struct hfi1_devdata *dd) 283 { 284 unsigned i; 285 286 spin_lock_init(&dd->aspm_lock); 287 dd->aspm_supported = aspm_hw_l1_supported(dd); 288 289 for (i = 0; i < dd->first_user_ctxt; i++) 290 aspm_ctx_init(dd->rcd[i]); 291 292 /* Start with ASPM disabled */ 293 aspm_hw_set_l1_ent_latency(dd); 294 dd->aspm_enabled = false; 295 aspm_hw_disable_l1(dd); 296 297 /* Now turn on ASPM if configured */ 298 aspm_enable_all(dd); 299 } 300 301 static inline void aspm_exit(struct hfi1_devdata *dd) 302 { 303 aspm_disable_all(dd); 304 305 /* Turn on ASPM on exit to conserve power */ 306 aspm_enable(dd); 307 } 308 309 #endif /* _ASPM_H */ 310