1 /* 2 * Copyright(c) 2015, 2016 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 /* 49 * This file contains all of the code that is specific to the HFI chip 50 */ 51 52 #include <linux/pci.h> 53 #include <linux/delay.h> 54 #include <linux/interrupt.h> 55 #include <linux/module.h> 56 57 #include "hfi.h" 58 #include "trace.h" 59 #include "mad.h" 60 #include "pio.h" 61 #include "sdma.h" 62 #include "eprom.h" 63 #include "efivar.h" 64 #include "platform.h" 65 #include "aspm.h" 66 67 #define NUM_IB_PORTS 1 68 69 uint kdeth_qp; 70 module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO); 71 MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix"); 72 73 uint num_vls = HFI1_MAX_VLS_SUPPORTED; 74 module_param(num_vls, uint, S_IRUGO); 75 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)"); 76 77 /* 78 * Default time to aggregate two 10K packets from the idle state 79 * (timer not running). The timer starts at the end of the first packet, 80 * so only the time for one 10K packet and header plus a bit extra is needed. 81 * 10 * 1024 + 64 header byte = 10304 byte 82 * 10304 byte / 12.5 GB/s = 824.32ns 83 */ 84 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */ 85 module_param(rcv_intr_timeout, uint, S_IRUGO); 86 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns"); 87 88 uint rcv_intr_count = 16; /* same as qib */ 89 module_param(rcv_intr_count, uint, S_IRUGO); 90 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count"); 91 92 ushort link_crc_mask = SUPPORTED_CRCS; 93 module_param(link_crc_mask, ushort, S_IRUGO); 94 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link"); 95 96 uint loopback; 97 module_param_named(loopback, loopback, uint, S_IRUGO); 98 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable"); 99 100 /* Other driver tunables */ 101 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/ 102 static ushort crc_14b_sideband = 1; 103 static uint use_flr = 1; 104 uint quick_linkup; /* skip LNI */ 105 106 struct flag_table { 107 u64 flag; /* the flag */ 108 char *str; /* description string */ 109 u16 extra; /* extra information */ 110 u16 unused0; 111 u32 unused1; 112 }; 113 114 /* str must be a string constant */ 115 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra} 116 #define FLAG_ENTRY0(str, flag) {flag, str, 0} 117 118 /* Send Error Consequences */ 119 #define SEC_WRITE_DROPPED 0x1 120 #define SEC_PACKET_DROPPED 0x2 121 #define SEC_SC_HALTED 0x4 /* per-context only */ 122 #define SEC_SPC_FREEZE 0x8 /* per-HFI only */ 123 124 #define MIN_KERNEL_KCTXTS 2 125 #define FIRST_KERNEL_KCTXT 1 126 /* sizes for both the QP and RSM map tables */ 127 #define NUM_MAP_ENTRIES 256 128 #define NUM_MAP_REGS 32 129 130 /* Bit offset into the GUID which carries HFI id information */ 131 #define GUID_HFI_INDEX_SHIFT 39 132 133 /* extract the emulation revision */ 134 #define emulator_rev(dd) ((dd)->irev >> 8) 135 /* parallel and serial emulation versions are 3 and 4 respectively */ 136 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3) 137 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4) 138 139 /* RSM fields */ 140 141 /* packet type */ 142 #define IB_PACKET_TYPE 2ull 143 #define QW_SHIFT 6ull 144 /* QPN[7..1] */ 145 #define QPN_WIDTH 7ull 146 147 /* LRH.BTH: QW 0, OFFSET 48 - for match */ 148 #define LRH_BTH_QW 0ull 149 #define LRH_BTH_BIT_OFFSET 48ull 150 #define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off)) 151 #define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET) 152 #define LRH_BTH_SELECT 153 #define LRH_BTH_MASK 3ull 154 #define LRH_BTH_VALUE 2ull 155 156 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */ 157 #define LRH_SC_QW 0ull 158 #define LRH_SC_BIT_OFFSET 56ull 159 #define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off)) 160 #define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET) 161 #define LRH_SC_MASK 128ull 162 #define LRH_SC_VALUE 0ull 163 164 /* SC[n..0] QW 0, OFFSET 60 - for select */ 165 #define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull)) 166 167 /* QPN[m+n:1] QW 1, OFFSET 1 */ 168 #define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull)) 169 170 /* defines to build power on SC2VL table */ 171 #define SC2VL_VAL( \ 172 num, \ 173 sc0, sc0val, \ 174 sc1, sc1val, \ 175 sc2, sc2val, \ 176 sc3, sc3val, \ 177 sc4, sc4val, \ 178 sc5, sc5val, \ 179 sc6, sc6val, \ 180 sc7, sc7val) \ 181 ( \ 182 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \ 183 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \ 184 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \ 185 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \ 186 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \ 187 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \ 188 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \ 189 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \ 190 ) 191 192 #define DC_SC_VL_VAL( \ 193 range, \ 194 e0, e0val, \ 195 e1, e1val, \ 196 e2, e2val, \ 197 e3, e3val, \ 198 e4, e4val, \ 199 e5, e5val, \ 200 e6, e6val, \ 201 e7, e7val, \ 202 e8, e8val, \ 203 e9, e9val, \ 204 e10, e10val, \ 205 e11, e11val, \ 206 e12, e12val, \ 207 e13, e13val, \ 208 e14, e14val, \ 209 e15, e15val) \ 210 ( \ 211 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \ 212 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \ 213 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \ 214 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \ 215 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \ 216 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \ 217 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \ 218 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \ 219 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \ 220 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \ 221 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \ 222 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \ 223 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \ 224 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \ 225 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \ 226 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \ 227 ) 228 229 /* all CceStatus sub-block freeze bits */ 230 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \ 231 | CCE_STATUS_RXE_FROZE_SMASK \ 232 | CCE_STATUS_TXE_FROZE_SMASK \ 233 | CCE_STATUS_TXE_PIO_FROZE_SMASK) 234 /* all CceStatus sub-block TXE pause bits */ 235 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \ 236 | CCE_STATUS_TXE_PAUSED_SMASK \ 237 | CCE_STATUS_SDMA_PAUSED_SMASK) 238 /* all CceStatus sub-block RXE pause bits */ 239 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK 240 241 /* 242 * CCE Error flags. 243 */ 244 static struct flag_table cce_err_status_flags[] = { 245 /* 0*/ FLAG_ENTRY0("CceCsrParityErr", 246 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK), 247 /* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr", 248 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK), 249 /* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr", 250 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK), 251 /* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr", 252 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK), 253 /* 4*/ FLAG_ENTRY0("CceTrgtAccessErr", 254 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK), 255 /* 5*/ FLAG_ENTRY0("CceRspdDataParityErr", 256 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK), 257 /* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr", 258 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK), 259 /* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr", 260 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK), 261 /* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr", 262 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK), 263 /* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 264 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK), 265 /*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 266 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK), 267 /*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError", 268 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK), 269 /*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError", 270 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK), 271 /*13*/ FLAG_ENTRY0("PcicRetryMemCorErr", 272 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK), 273 /*14*/ FLAG_ENTRY0("PcicRetryMemCorErr", 274 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK), 275 /*15*/ FLAG_ENTRY0("PcicPostHdQCorErr", 276 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK), 277 /*16*/ FLAG_ENTRY0("PcicPostHdQCorErr", 278 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK), 279 /*17*/ FLAG_ENTRY0("PcicPostHdQCorErr", 280 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK), 281 /*18*/ FLAG_ENTRY0("PcicCplDatQCorErr", 282 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK), 283 /*19*/ FLAG_ENTRY0("PcicNPostHQParityErr", 284 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK), 285 /*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr", 286 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK), 287 /*21*/ FLAG_ENTRY0("PcicRetryMemUncErr", 288 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK), 289 /*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr", 290 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK), 291 /*23*/ FLAG_ENTRY0("PcicPostHdQUncErr", 292 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK), 293 /*24*/ FLAG_ENTRY0("PcicPostDatQUncErr", 294 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK), 295 /*25*/ FLAG_ENTRY0("PcicCplHdQUncErr", 296 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK), 297 /*26*/ FLAG_ENTRY0("PcicCplDatQUncErr", 298 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK), 299 /*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr", 300 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK), 301 /*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr", 302 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK), 303 /*29*/ FLAG_ENTRY0("PcicReceiveParityErr", 304 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK), 305 /*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr", 306 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK), 307 /*31*/ FLAG_ENTRY0("LATriggered", 308 CCE_ERR_STATUS_LA_TRIGGERED_SMASK), 309 /*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr", 310 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK), 311 /*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr", 312 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK), 313 /*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr", 314 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK), 315 /*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr", 316 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK), 317 /*36*/ FLAG_ENTRY0("CceMsixTableCorErr", 318 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK), 319 /*37*/ FLAG_ENTRY0("CceMsixTableUncErr", 320 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK), 321 /*38*/ FLAG_ENTRY0("CceIntMapCorErr", 322 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK), 323 /*39*/ FLAG_ENTRY0("CceIntMapUncErr", 324 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK), 325 /*40*/ FLAG_ENTRY0("CceMsixCsrParityErr", 326 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK), 327 /*41-63 reserved*/ 328 }; 329 330 /* 331 * Misc Error flags 332 */ 333 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK 334 static struct flag_table misc_err_status_flags[] = { 335 /* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)), 336 /* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)), 337 /* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)), 338 /* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)), 339 /* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)), 340 /* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)), 341 /* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)), 342 /* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)), 343 /* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)), 344 /* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)), 345 /*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)), 346 /*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)), 347 /*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL)) 348 }; 349 350 /* 351 * TXE PIO Error flags and consequences 352 */ 353 static struct flag_table pio_err_status_flags[] = { 354 /* 0*/ FLAG_ENTRY("PioWriteBadCtxt", 355 SEC_WRITE_DROPPED, 356 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK), 357 /* 1*/ FLAG_ENTRY("PioWriteAddrParity", 358 SEC_SPC_FREEZE, 359 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK), 360 /* 2*/ FLAG_ENTRY("PioCsrParity", 361 SEC_SPC_FREEZE, 362 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK), 363 /* 3*/ FLAG_ENTRY("PioSbMemFifo0", 364 SEC_SPC_FREEZE, 365 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK), 366 /* 4*/ FLAG_ENTRY("PioSbMemFifo1", 367 SEC_SPC_FREEZE, 368 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK), 369 /* 5*/ FLAG_ENTRY("PioPccFifoParity", 370 SEC_SPC_FREEZE, 371 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK), 372 /* 6*/ FLAG_ENTRY("PioPecFifoParity", 373 SEC_SPC_FREEZE, 374 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK), 375 /* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity", 376 SEC_SPC_FREEZE, 377 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK), 378 /* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity", 379 SEC_SPC_FREEZE, 380 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK), 381 /* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr", 382 SEC_SPC_FREEZE, 383 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK), 384 /*10*/ FLAG_ENTRY("PioSmPktResetParity", 385 SEC_SPC_FREEZE, 386 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK), 387 /*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc", 388 SEC_SPC_FREEZE, 389 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK), 390 /*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc", 391 SEC_SPC_FREEZE, 392 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK), 393 /*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor", 394 0, 395 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK), 396 /*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor", 397 0, 398 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK), 399 /*15*/ FLAG_ENTRY("PioCreditRetFifoParity", 400 SEC_SPC_FREEZE, 401 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK), 402 /*16*/ FLAG_ENTRY("PioPpmcPblFifo", 403 SEC_SPC_FREEZE, 404 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK), 405 /*17*/ FLAG_ENTRY("PioInitSmIn", 406 0, 407 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK), 408 /*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm", 409 SEC_SPC_FREEZE, 410 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK), 411 /*19*/ FLAG_ENTRY("PioHostAddrMemUnc", 412 SEC_SPC_FREEZE, 413 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK), 414 /*20*/ FLAG_ENTRY("PioHostAddrMemCor", 415 0, 416 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK), 417 /*21*/ FLAG_ENTRY("PioWriteDataParity", 418 SEC_SPC_FREEZE, 419 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK), 420 /*22*/ FLAG_ENTRY("PioStateMachine", 421 SEC_SPC_FREEZE, 422 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK), 423 /*23*/ FLAG_ENTRY("PioWriteQwValidParity", 424 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 425 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK), 426 /*24*/ FLAG_ENTRY("PioBlockQwCountParity", 427 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 428 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK), 429 /*25*/ FLAG_ENTRY("PioVlfVlLenParity", 430 SEC_SPC_FREEZE, 431 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK), 432 /*26*/ FLAG_ENTRY("PioVlfSopParity", 433 SEC_SPC_FREEZE, 434 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK), 435 /*27*/ FLAG_ENTRY("PioVlFifoParity", 436 SEC_SPC_FREEZE, 437 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK), 438 /*28*/ FLAG_ENTRY("PioPpmcBqcMemParity", 439 SEC_SPC_FREEZE, 440 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK), 441 /*29*/ FLAG_ENTRY("PioPpmcSopLen", 442 SEC_SPC_FREEZE, 443 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK), 444 /*30-31 reserved*/ 445 /*32*/ FLAG_ENTRY("PioCurrentFreeCntParity", 446 SEC_SPC_FREEZE, 447 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK), 448 /*33*/ FLAG_ENTRY("PioLastReturnedCntParity", 449 SEC_SPC_FREEZE, 450 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK), 451 /*34*/ FLAG_ENTRY("PioPccSopHeadParity", 452 SEC_SPC_FREEZE, 453 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK), 454 /*35*/ FLAG_ENTRY("PioPecSopHeadParityErr", 455 SEC_SPC_FREEZE, 456 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK), 457 /*36-63 reserved*/ 458 }; 459 460 /* TXE PIO errors that cause an SPC freeze */ 461 #define ALL_PIO_FREEZE_ERR \ 462 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \ 463 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \ 464 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \ 465 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \ 466 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \ 467 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \ 468 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \ 469 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \ 470 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \ 471 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \ 472 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \ 473 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \ 474 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \ 475 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \ 476 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \ 477 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \ 478 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \ 479 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \ 480 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \ 481 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \ 482 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \ 483 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \ 484 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \ 485 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \ 486 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \ 487 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \ 488 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \ 489 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \ 490 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK) 491 492 /* 493 * TXE SDMA Error flags 494 */ 495 static struct flag_table sdma_err_status_flags[] = { 496 /* 0*/ FLAG_ENTRY0("SDmaRpyTagErr", 497 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK), 498 /* 1*/ FLAG_ENTRY0("SDmaCsrParityErr", 499 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK), 500 /* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr", 501 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK), 502 /* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr", 503 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK), 504 /*04-63 reserved*/ 505 }; 506 507 /* TXE SDMA errors that cause an SPC freeze */ 508 #define ALL_SDMA_FREEZE_ERR \ 509 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \ 510 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \ 511 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK) 512 513 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */ 514 #define PORT_DISCARD_EGRESS_ERRS \ 515 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \ 516 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \ 517 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK) 518 519 /* 520 * TXE Egress Error flags 521 */ 522 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK 523 static struct flag_table egress_err_status_flags[] = { 524 /* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)), 525 /* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)), 526 /* 2 reserved */ 527 /* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr", 528 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)), 529 /* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)), 530 /* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)), 531 /* 6 reserved */ 532 /* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr", 533 SEES(TX_PIO_LAUNCH_INTF_PARITY)), 534 /* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr", 535 SEES(TX_SDMA_LAUNCH_INTF_PARITY)), 536 /* 9-10 reserved */ 537 /*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr", 538 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)), 539 /*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)), 540 /*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)), 541 /*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)), 542 /*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)), 543 /*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr", 544 SEES(TX_SDMA0_DISALLOWED_PACKET)), 545 /*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr", 546 SEES(TX_SDMA1_DISALLOWED_PACKET)), 547 /*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr", 548 SEES(TX_SDMA2_DISALLOWED_PACKET)), 549 /*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr", 550 SEES(TX_SDMA3_DISALLOWED_PACKET)), 551 /*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr", 552 SEES(TX_SDMA4_DISALLOWED_PACKET)), 553 /*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr", 554 SEES(TX_SDMA5_DISALLOWED_PACKET)), 555 /*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr", 556 SEES(TX_SDMA6_DISALLOWED_PACKET)), 557 /*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr", 558 SEES(TX_SDMA7_DISALLOWED_PACKET)), 559 /*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr", 560 SEES(TX_SDMA8_DISALLOWED_PACKET)), 561 /*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr", 562 SEES(TX_SDMA9_DISALLOWED_PACKET)), 563 /*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr", 564 SEES(TX_SDMA10_DISALLOWED_PACKET)), 565 /*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr", 566 SEES(TX_SDMA11_DISALLOWED_PACKET)), 567 /*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr", 568 SEES(TX_SDMA12_DISALLOWED_PACKET)), 569 /*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr", 570 SEES(TX_SDMA13_DISALLOWED_PACKET)), 571 /*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr", 572 SEES(TX_SDMA14_DISALLOWED_PACKET)), 573 /*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr", 574 SEES(TX_SDMA15_DISALLOWED_PACKET)), 575 /*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr", 576 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)), 577 /*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr", 578 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)), 579 /*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr", 580 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)), 581 /*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr", 582 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)), 583 /*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr", 584 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)), 585 /*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr", 586 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)), 587 /*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr", 588 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)), 589 /*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr", 590 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)), 591 /*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr", 592 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)), 593 /*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)), 594 /*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)), 595 /*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)), 596 /*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)), 597 /*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)), 598 /*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)), 599 /*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)), 600 /*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)), 601 /*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)), 602 /*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)), 603 /*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)), 604 /*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)), 605 /*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)), 606 /*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)), 607 /*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)), 608 /*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)), 609 /*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)), 610 /*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)), 611 /*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)), 612 /*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)), 613 /*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)), 614 /*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr", 615 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)), 616 /*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr", 617 SEES(TX_READ_PIO_MEMORY_CSR_UNC)), 618 }; 619 620 /* 621 * TXE Egress Error Info flags 622 */ 623 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK 624 static struct flag_table egress_err_info_flags[] = { 625 /* 0*/ FLAG_ENTRY0("Reserved", 0ull), 626 /* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)), 627 /* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 628 /* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 629 /* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)), 630 /* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)), 631 /* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)), 632 /* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)), 633 /* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)), 634 /* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)), 635 /*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)), 636 /*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)), 637 /*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)), 638 /*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)), 639 /*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)), 640 /*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)), 641 /*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)), 642 /*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)), 643 /*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)), 644 /*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)), 645 /*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)), 646 /*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)), 647 }; 648 649 /* TXE Egress errors that cause an SPC freeze */ 650 #define ALL_TXE_EGRESS_FREEZE_ERR \ 651 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \ 652 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \ 653 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \ 654 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \ 655 | SEES(TX_LAUNCH_CSR_PARITY) \ 656 | SEES(TX_SBRD_CTL_CSR_PARITY) \ 657 | SEES(TX_CONFIG_PARITY) \ 658 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \ 659 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \ 660 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \ 661 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \ 662 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \ 663 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \ 664 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \ 665 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \ 666 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \ 667 | SEES(TX_CREDIT_RETURN_PARITY)) 668 669 /* 670 * TXE Send error flags 671 */ 672 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK 673 static struct flag_table send_err_status_flags[] = { 674 /* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)), 675 /* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)), 676 /* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR)) 677 }; 678 679 /* 680 * TXE Send Context Error flags and consequences 681 */ 682 static struct flag_table sc_err_status_flags[] = { 683 /* 0*/ FLAG_ENTRY("InconsistentSop", 684 SEC_PACKET_DROPPED | SEC_SC_HALTED, 685 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK), 686 /* 1*/ FLAG_ENTRY("DisallowedPacket", 687 SEC_PACKET_DROPPED | SEC_SC_HALTED, 688 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK), 689 /* 2*/ FLAG_ENTRY("WriteCrossesBoundary", 690 SEC_WRITE_DROPPED | SEC_SC_HALTED, 691 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK), 692 /* 3*/ FLAG_ENTRY("WriteOverflow", 693 SEC_WRITE_DROPPED | SEC_SC_HALTED, 694 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK), 695 /* 4*/ FLAG_ENTRY("WriteOutOfBounds", 696 SEC_WRITE_DROPPED | SEC_SC_HALTED, 697 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK), 698 /* 5-63 reserved*/ 699 }; 700 701 /* 702 * RXE Receive Error flags 703 */ 704 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK 705 static struct flag_table rxe_err_status_flags[] = { 706 /* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)), 707 /* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)), 708 /* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)), 709 /* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)), 710 /* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)), 711 /* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)), 712 /* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)), 713 /* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)), 714 /* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)), 715 /* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)), 716 /*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)), 717 /*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)), 718 /*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)), 719 /*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)), 720 /*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)), 721 /*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)), 722 /*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr", 723 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)), 724 /*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)), 725 /*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)), 726 /*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr", 727 RXES(RBUF_BLOCK_LIST_READ_UNC)), 728 /*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr", 729 RXES(RBUF_BLOCK_LIST_READ_COR)), 730 /*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr", 731 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)), 732 /*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr", 733 RXES(RBUF_CSR_QENT_CNT_PARITY)), 734 /*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr", 735 RXES(RBUF_CSR_QNEXT_BUF_PARITY)), 736 /*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr", 737 RXES(RBUF_CSR_QVLD_BIT_PARITY)), 738 /*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)), 739 /*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)), 740 /*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr", 741 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)), 742 /*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)), 743 /*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)), 744 /*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)), 745 /*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)), 746 /*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)), 747 /*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)), 748 /*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)), 749 /*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr", 750 RXES(RBUF_FL_INITDONE_PARITY)), 751 /*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr", 752 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)), 753 /*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)), 754 /*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)), 755 /*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)), 756 /*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr", 757 RXES(LOOKUP_DES_PART1_UNC_COR)), 758 /*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr", 759 RXES(LOOKUP_DES_PART2_PARITY)), 760 /*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)), 761 /*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)), 762 /*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)), 763 /*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)), 764 /*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)), 765 /*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)), 766 /*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)), 767 /*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)), 768 /*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)), 769 /*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)), 770 /*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)), 771 /*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)), 772 /*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)), 773 /*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)), 774 /*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)), 775 /*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)), 776 /*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)), 777 /*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)), 778 /*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)), 779 /*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)), 780 /*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)), 781 /*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY)) 782 }; 783 784 /* RXE errors that will trigger an SPC freeze */ 785 #define ALL_RXE_FREEZE_ERR \ 786 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \ 787 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \ 788 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \ 789 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \ 790 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \ 791 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \ 792 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \ 793 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \ 794 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \ 795 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \ 796 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \ 797 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \ 798 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \ 799 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \ 800 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \ 801 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \ 802 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \ 803 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \ 804 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \ 805 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \ 806 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \ 807 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \ 808 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \ 809 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \ 810 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \ 811 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \ 812 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \ 813 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \ 814 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \ 815 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \ 816 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \ 817 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \ 818 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \ 819 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \ 820 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \ 821 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \ 822 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \ 823 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \ 824 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \ 825 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \ 826 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \ 827 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \ 828 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \ 829 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK) 830 831 #define RXE_FREEZE_ABORT_MASK \ 832 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \ 833 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \ 834 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK) 835 836 /* 837 * DCC Error Flags 838 */ 839 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK 840 static struct flag_table dcc_err_flags[] = { 841 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)), 842 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)), 843 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)), 844 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)), 845 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)), 846 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)), 847 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)), 848 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)), 849 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)), 850 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)), 851 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)), 852 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)), 853 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)), 854 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)), 855 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)), 856 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)), 857 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)), 858 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)), 859 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)), 860 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)), 861 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)), 862 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)), 863 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)), 864 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)), 865 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)), 866 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)), 867 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)), 868 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)), 869 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)), 870 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)), 871 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)), 872 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)), 873 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)), 874 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)), 875 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)), 876 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)), 877 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)), 878 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)), 879 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)), 880 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)), 881 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)), 882 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)), 883 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)), 884 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)), 885 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)), 886 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)), 887 }; 888 889 /* 890 * LCB error flags 891 */ 892 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK 893 static struct flag_table lcb_err_flags[] = { 894 /* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)), 895 /* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)), 896 /* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)), 897 /* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST", 898 LCBE(ALL_LNS_FAILED_REINIT_TEST)), 899 /* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)), 900 /* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)), 901 /* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)), 902 /* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)), 903 /* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)), 904 /* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)), 905 /*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)), 906 /*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)), 907 /*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)), 908 /*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER", 909 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)), 910 /*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)), 911 /*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)), 912 /*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)), 913 /*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)), 914 /*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)), 915 /*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE", 916 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)), 917 /*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)), 918 /*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)), 919 /*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)), 920 /*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)), 921 /*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)), 922 /*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)), 923 /*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP", 924 LCBE(RST_FOR_INCOMPLT_RND_TRIP)), 925 /*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)), 926 /*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE", 927 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)), 928 /*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR", 929 LCBE(REDUNDANT_FLIT_PARITY_ERR)) 930 }; 931 932 /* 933 * DC8051 Error Flags 934 */ 935 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK 936 static struct flag_table dc8051_err_flags[] = { 937 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)), 938 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)), 939 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)), 940 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)), 941 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)), 942 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)), 943 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)), 944 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)), 945 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES", 946 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)), 947 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)), 948 }; 949 950 /* 951 * DC8051 Information Error flags 952 * 953 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field. 954 */ 955 static struct flag_table dc8051_info_err_flags[] = { 956 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED), 957 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME), 958 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET), 959 FLAG_ENTRY0("Serdes internal loopback failure", 960 FAILED_SERDES_INTERNAL_LOOPBACK), 961 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT), 962 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING), 963 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE), 964 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM), 965 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ), 966 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1), 967 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2), 968 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT), 969 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT) 970 }; 971 972 /* 973 * DC8051 Information Host Information flags 974 * 975 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field. 976 */ 977 static struct flag_table dc8051_info_host_msg_flags[] = { 978 FLAG_ENTRY0("Host request done", 0x0001), 979 FLAG_ENTRY0("BC SMA message", 0x0002), 980 FLAG_ENTRY0("BC PWR_MGM message", 0x0004), 981 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008), 982 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010), 983 FLAG_ENTRY0("External device config request", 0x0020), 984 FLAG_ENTRY0("VerifyCap all frames received", 0x0040), 985 FLAG_ENTRY0("LinkUp achieved", 0x0080), 986 FLAG_ENTRY0("Link going down", 0x0100), 987 }; 988 989 static u32 encoded_size(u32 size); 990 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate); 991 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state); 992 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 993 u8 *continuous); 994 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 995 u8 *vcu, u16 *vl15buf, u8 *crc_sizes); 996 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 997 u8 *remote_tx_rate, u16 *link_widths); 998 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits, 999 u8 *flag_bits, u16 *link_widths); 1000 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 1001 u8 *device_rev); 1002 static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed); 1003 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx); 1004 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx, 1005 u8 *tx_polarity_inversion, 1006 u8 *rx_polarity_inversion, u8 *max_rate); 1007 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 1008 unsigned int context, u64 err_status); 1009 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg); 1010 static void handle_dcc_err(struct hfi1_devdata *dd, 1011 unsigned int context, u64 err_status); 1012 static void handle_lcb_err(struct hfi1_devdata *dd, 1013 unsigned int context, u64 err_status); 1014 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg); 1015 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1016 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1017 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1018 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1019 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1020 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1021 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1022 static void set_partition_keys(struct hfi1_pportdata *); 1023 static const char *link_state_name(u32 state); 1024 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, 1025 u32 state); 1026 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 1027 u64 *out_data); 1028 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data); 1029 static int thermal_init(struct hfi1_devdata *dd); 1030 1031 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1032 int msecs); 1033 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc); 1034 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr); 1035 static void handle_temp_err(struct hfi1_devdata *); 1036 static void dc_shutdown(struct hfi1_devdata *); 1037 static void dc_start(struct hfi1_devdata *); 1038 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 1039 unsigned int *np); 1040 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); 1041 1042 /* 1043 * Error interrupt table entry. This is used as input to the interrupt 1044 * "clear down" routine used for all second tier error interrupt register. 1045 * Second tier interrupt registers have a single bit representing them 1046 * in the top-level CceIntStatus. 1047 */ 1048 struct err_reg_info { 1049 u32 status; /* status CSR offset */ 1050 u32 clear; /* clear CSR offset */ 1051 u32 mask; /* mask CSR offset */ 1052 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg); 1053 const char *desc; 1054 }; 1055 1056 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START) 1057 #define NUM_DC_ERRS (IS_DC_END - IS_DC_START) 1058 #define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START) 1059 1060 /* 1061 * Helpers for building HFI and DC error interrupt table entries. Different 1062 * helpers are needed because of inconsistent register names. 1063 */ 1064 #define EE(reg, handler, desc) \ 1065 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \ 1066 handler, desc } 1067 #define DC_EE1(reg, handler, desc) \ 1068 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc } 1069 #define DC_EE2(reg, handler, desc) \ 1070 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc } 1071 1072 /* 1073 * Table of the "misc" grouping of error interrupts. Each entry refers to 1074 * another register containing more information. 1075 */ 1076 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = { 1077 /* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"), 1078 /* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"), 1079 /* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"), 1080 /* 3*/ { 0, 0, 0, NULL }, /* reserved */ 1081 /* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"), 1082 /* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"), 1083 /* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"), 1084 /* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr") 1085 /* the rest are reserved */ 1086 }; 1087 1088 /* 1089 * Index into the Various section of the interrupt sources 1090 * corresponding to the Critical Temperature interrupt. 1091 */ 1092 #define TCRIT_INT_SOURCE 4 1093 1094 /* 1095 * SDMA error interrupt entry - refers to another register containing more 1096 * information. 1097 */ 1098 static const struct err_reg_info sdma_eng_err = 1099 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr"); 1100 1101 static const struct err_reg_info various_err[NUM_VARIOUS] = { 1102 /* 0*/ { 0, 0, 0, NULL }, /* PbcInt */ 1103 /* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */ 1104 /* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"), 1105 /* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"), 1106 /* 4*/ { 0, 0, 0, NULL }, /* TCritInt */ 1107 /* rest are reserved */ 1108 }; 1109 1110 /* 1111 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG 1112 * register can not be derived from the MTU value because 10K is not 1113 * a power of 2. Therefore, we need a constant. Everything else can 1114 * be calculated. 1115 */ 1116 #define DCC_CFG_PORT_MTU_CAP_10240 7 1117 1118 /* 1119 * Table of the DC grouping of error interrupts. Each entry refers to 1120 * another register containing more information. 1121 */ 1122 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = { 1123 /* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"), 1124 /* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"), 1125 /* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"), 1126 /* 3*/ /* dc_lbm_int - special, see is_dc_int() */ 1127 /* the rest are reserved */ 1128 }; 1129 1130 struct cntr_entry { 1131 /* 1132 * counter name 1133 */ 1134 char *name; 1135 1136 /* 1137 * csr to read for name (if applicable) 1138 */ 1139 u64 csr; 1140 1141 /* 1142 * offset into dd or ppd to store the counter's value 1143 */ 1144 int offset; 1145 1146 /* 1147 * flags 1148 */ 1149 u8 flags; 1150 1151 /* 1152 * accessor for stat element, context either dd or ppd 1153 */ 1154 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl, 1155 int mode, u64 data); 1156 }; 1157 1158 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0 1159 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159 1160 1161 #define CNTR_ELEM(name, csr, offset, flags, accessor) \ 1162 { \ 1163 name, \ 1164 csr, \ 1165 offset, \ 1166 flags, \ 1167 accessor \ 1168 } 1169 1170 /* 32bit RXE */ 1171 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1172 CNTR_ELEM(#name, \ 1173 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1174 0, flags | CNTR_32BIT, \ 1175 port_access_u32_csr) 1176 1177 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \ 1178 CNTR_ELEM(#name, \ 1179 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1180 0, flags | CNTR_32BIT, \ 1181 dev_access_u32_csr) 1182 1183 /* 64bit RXE */ 1184 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1185 CNTR_ELEM(#name, \ 1186 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1187 0, flags, \ 1188 port_access_u64_csr) 1189 1190 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \ 1191 CNTR_ELEM(#name, \ 1192 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1193 0, flags, \ 1194 dev_access_u64_csr) 1195 1196 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx 1197 #define OVR_ELM(ctx) \ 1198 CNTR_ELEM("RcvHdrOvr" #ctx, \ 1199 (RCV_HDR_OVFL_CNT + ctx * 0x100), \ 1200 0, CNTR_NORMAL, port_access_u64_csr) 1201 1202 /* 32bit TXE */ 1203 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1204 CNTR_ELEM(#name, \ 1205 (counter * 8 + SEND_COUNTER_ARRAY32), \ 1206 0, flags | CNTR_32BIT, \ 1207 port_access_u32_csr) 1208 1209 /* 64bit TXE */ 1210 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1211 CNTR_ELEM(#name, \ 1212 (counter * 8 + SEND_COUNTER_ARRAY64), \ 1213 0, flags, \ 1214 port_access_u64_csr) 1215 1216 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \ 1217 CNTR_ELEM(#name,\ 1218 counter * 8 + SEND_COUNTER_ARRAY64, \ 1219 0, \ 1220 flags, \ 1221 dev_access_u64_csr) 1222 1223 /* CCE */ 1224 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \ 1225 CNTR_ELEM(#name, \ 1226 (counter * 8 + CCE_COUNTER_ARRAY32), \ 1227 0, flags | CNTR_32BIT, \ 1228 dev_access_u32_csr) 1229 1230 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \ 1231 CNTR_ELEM(#name, \ 1232 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \ 1233 0, flags | CNTR_32BIT, \ 1234 dev_access_u32_csr) 1235 1236 /* DC */ 1237 #define DC_PERF_CNTR(name, counter, flags) \ 1238 CNTR_ELEM(#name, \ 1239 counter, \ 1240 0, \ 1241 flags, \ 1242 dev_access_u64_csr) 1243 1244 #define DC_PERF_CNTR_LCB(name, counter, flags) \ 1245 CNTR_ELEM(#name, \ 1246 counter, \ 1247 0, \ 1248 flags, \ 1249 dc_access_lcb_cntr) 1250 1251 /* ibp counters */ 1252 #define SW_IBP_CNTR(name, cntr) \ 1253 CNTR_ELEM(#name, \ 1254 0, \ 1255 0, \ 1256 CNTR_SYNTH, \ 1257 access_ibp_##cntr) 1258 1259 u64 read_csr(const struct hfi1_devdata *dd, u32 offset) 1260 { 1261 if (dd->flags & HFI1_PRESENT) { 1262 return readq((void __iomem *)dd->kregbase + offset); 1263 } 1264 return -1; 1265 } 1266 1267 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value) 1268 { 1269 if (dd->flags & HFI1_PRESENT) 1270 writeq(value, (void __iomem *)dd->kregbase + offset); 1271 } 1272 1273 void __iomem *get_csr_addr( 1274 struct hfi1_devdata *dd, 1275 u32 offset) 1276 { 1277 return (void __iomem *)dd->kregbase + offset; 1278 } 1279 1280 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr, 1281 int mode, u64 value) 1282 { 1283 u64 ret; 1284 1285 if (mode == CNTR_MODE_R) { 1286 ret = read_csr(dd, csr); 1287 } else if (mode == CNTR_MODE_W) { 1288 write_csr(dd, csr, value); 1289 ret = value; 1290 } else { 1291 dd_dev_err(dd, "Invalid cntr register access mode"); 1292 return 0; 1293 } 1294 1295 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode); 1296 return ret; 1297 } 1298 1299 /* Dev Access */ 1300 static u64 dev_access_u32_csr(const struct cntr_entry *entry, 1301 void *context, int vl, int mode, u64 data) 1302 { 1303 struct hfi1_devdata *dd = context; 1304 u64 csr = entry->csr; 1305 1306 if (entry->flags & CNTR_SDMA) { 1307 if (vl == CNTR_INVALID_VL) 1308 return 0; 1309 csr += 0x100 * vl; 1310 } else { 1311 if (vl != CNTR_INVALID_VL) 1312 return 0; 1313 } 1314 return read_write_csr(dd, csr, mode, data); 1315 } 1316 1317 static u64 access_sde_err_cnt(const struct cntr_entry *entry, 1318 void *context, int idx, int mode, u64 data) 1319 { 1320 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1321 1322 if (dd->per_sdma && idx < dd->num_sdma) 1323 return dd->per_sdma[idx].err_cnt; 1324 return 0; 1325 } 1326 1327 static u64 access_sde_int_cnt(const struct cntr_entry *entry, 1328 void *context, int idx, int mode, u64 data) 1329 { 1330 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1331 1332 if (dd->per_sdma && idx < dd->num_sdma) 1333 return dd->per_sdma[idx].sdma_int_cnt; 1334 return 0; 1335 } 1336 1337 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry, 1338 void *context, int idx, int mode, u64 data) 1339 { 1340 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1341 1342 if (dd->per_sdma && idx < dd->num_sdma) 1343 return dd->per_sdma[idx].idle_int_cnt; 1344 return 0; 1345 } 1346 1347 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry, 1348 void *context, int idx, int mode, 1349 u64 data) 1350 { 1351 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1352 1353 if (dd->per_sdma && idx < dd->num_sdma) 1354 return dd->per_sdma[idx].progress_int_cnt; 1355 return 0; 1356 } 1357 1358 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context, 1359 int vl, int mode, u64 data) 1360 { 1361 struct hfi1_devdata *dd = context; 1362 1363 u64 val = 0; 1364 u64 csr = entry->csr; 1365 1366 if (entry->flags & CNTR_VL) { 1367 if (vl == CNTR_INVALID_VL) 1368 return 0; 1369 csr += 8 * vl; 1370 } else { 1371 if (vl != CNTR_INVALID_VL) 1372 return 0; 1373 } 1374 1375 val = read_write_csr(dd, csr, mode, data); 1376 return val; 1377 } 1378 1379 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context, 1380 int vl, int mode, u64 data) 1381 { 1382 struct hfi1_devdata *dd = context; 1383 u32 csr = entry->csr; 1384 int ret = 0; 1385 1386 if (vl != CNTR_INVALID_VL) 1387 return 0; 1388 if (mode == CNTR_MODE_R) 1389 ret = read_lcb_csr(dd, csr, &data); 1390 else if (mode == CNTR_MODE_W) 1391 ret = write_lcb_csr(dd, csr, data); 1392 1393 if (ret) { 1394 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr); 1395 return 0; 1396 } 1397 1398 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode); 1399 return data; 1400 } 1401 1402 /* Port Access */ 1403 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context, 1404 int vl, int mode, u64 data) 1405 { 1406 struct hfi1_pportdata *ppd = context; 1407 1408 if (vl != CNTR_INVALID_VL) 1409 return 0; 1410 return read_write_csr(ppd->dd, entry->csr, mode, data); 1411 } 1412 1413 static u64 port_access_u64_csr(const struct cntr_entry *entry, 1414 void *context, int vl, int mode, u64 data) 1415 { 1416 struct hfi1_pportdata *ppd = context; 1417 u64 val; 1418 u64 csr = entry->csr; 1419 1420 if (entry->flags & CNTR_VL) { 1421 if (vl == CNTR_INVALID_VL) 1422 return 0; 1423 csr += 8 * vl; 1424 } else { 1425 if (vl != CNTR_INVALID_VL) 1426 return 0; 1427 } 1428 val = read_write_csr(ppd->dd, csr, mode, data); 1429 return val; 1430 } 1431 1432 /* Software defined */ 1433 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode, 1434 u64 data) 1435 { 1436 u64 ret; 1437 1438 if (mode == CNTR_MODE_R) { 1439 ret = *cntr; 1440 } else if (mode == CNTR_MODE_W) { 1441 *cntr = data; 1442 ret = data; 1443 } else { 1444 dd_dev_err(dd, "Invalid cntr sw access mode"); 1445 return 0; 1446 } 1447 1448 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode); 1449 1450 return ret; 1451 } 1452 1453 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context, 1454 int vl, int mode, u64 data) 1455 { 1456 struct hfi1_pportdata *ppd = context; 1457 1458 if (vl != CNTR_INVALID_VL) 1459 return 0; 1460 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data); 1461 } 1462 1463 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context, 1464 int vl, int mode, u64 data) 1465 { 1466 struct hfi1_pportdata *ppd = context; 1467 1468 if (vl != CNTR_INVALID_VL) 1469 return 0; 1470 return read_write_sw(ppd->dd, &ppd->link_up, mode, data); 1471 } 1472 1473 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry, 1474 void *context, int vl, int mode, 1475 u64 data) 1476 { 1477 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1478 1479 if (vl != CNTR_INVALID_VL) 1480 return 0; 1481 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data); 1482 } 1483 1484 static u64 access_sw_xmit_discards(const struct cntr_entry *entry, 1485 void *context, int vl, int mode, u64 data) 1486 { 1487 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1488 u64 zero = 0; 1489 u64 *counter; 1490 1491 if (vl == CNTR_INVALID_VL) 1492 counter = &ppd->port_xmit_discards; 1493 else if (vl >= 0 && vl < C_VL_COUNT) 1494 counter = &ppd->port_xmit_discards_vl[vl]; 1495 else 1496 counter = &zero; 1497 1498 return read_write_sw(ppd->dd, counter, mode, data); 1499 } 1500 1501 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry, 1502 void *context, int vl, int mode, 1503 u64 data) 1504 { 1505 struct hfi1_pportdata *ppd = context; 1506 1507 if (vl != CNTR_INVALID_VL) 1508 return 0; 1509 1510 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors, 1511 mode, data); 1512 } 1513 1514 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry, 1515 void *context, int vl, int mode, u64 data) 1516 { 1517 struct hfi1_pportdata *ppd = context; 1518 1519 if (vl != CNTR_INVALID_VL) 1520 return 0; 1521 1522 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors, 1523 mode, data); 1524 } 1525 1526 u64 get_all_cpu_total(u64 __percpu *cntr) 1527 { 1528 int cpu; 1529 u64 counter = 0; 1530 1531 for_each_possible_cpu(cpu) 1532 counter += *per_cpu_ptr(cntr, cpu); 1533 return counter; 1534 } 1535 1536 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val, 1537 u64 __percpu *cntr, 1538 int vl, int mode, u64 data) 1539 { 1540 u64 ret = 0; 1541 1542 if (vl != CNTR_INVALID_VL) 1543 return 0; 1544 1545 if (mode == CNTR_MODE_R) { 1546 ret = get_all_cpu_total(cntr) - *z_val; 1547 } else if (mode == CNTR_MODE_W) { 1548 /* A write can only zero the counter */ 1549 if (data == 0) 1550 *z_val = get_all_cpu_total(cntr); 1551 else 1552 dd_dev_err(dd, "Per CPU cntrs can only be zeroed"); 1553 } else { 1554 dd_dev_err(dd, "Invalid cntr sw cpu access mode"); 1555 return 0; 1556 } 1557 1558 return ret; 1559 } 1560 1561 static u64 access_sw_cpu_intr(const struct cntr_entry *entry, 1562 void *context, int vl, int mode, u64 data) 1563 { 1564 struct hfi1_devdata *dd = context; 1565 1566 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl, 1567 mode, data); 1568 } 1569 1570 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry, 1571 void *context, int vl, int mode, u64 data) 1572 { 1573 struct hfi1_devdata *dd = context; 1574 1575 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl, 1576 mode, data); 1577 } 1578 1579 static u64 access_sw_pio_wait(const struct cntr_entry *entry, 1580 void *context, int vl, int mode, u64 data) 1581 { 1582 struct hfi1_devdata *dd = context; 1583 1584 return dd->verbs_dev.n_piowait; 1585 } 1586 1587 static u64 access_sw_pio_drain(const struct cntr_entry *entry, 1588 void *context, int vl, int mode, u64 data) 1589 { 1590 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1591 1592 return dd->verbs_dev.n_piodrain; 1593 } 1594 1595 static u64 access_sw_vtx_wait(const struct cntr_entry *entry, 1596 void *context, int vl, int mode, u64 data) 1597 { 1598 struct hfi1_devdata *dd = context; 1599 1600 return dd->verbs_dev.n_txwait; 1601 } 1602 1603 static u64 access_sw_kmem_wait(const struct cntr_entry *entry, 1604 void *context, int vl, int mode, u64 data) 1605 { 1606 struct hfi1_devdata *dd = context; 1607 1608 return dd->verbs_dev.n_kmem_wait; 1609 } 1610 1611 static u64 access_sw_send_schedule(const struct cntr_entry *entry, 1612 void *context, int vl, int mode, u64 data) 1613 { 1614 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1615 1616 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl, 1617 mode, data); 1618 } 1619 1620 /* Software counters for the error status bits within MISC_ERR_STATUS */ 1621 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry, 1622 void *context, int vl, int mode, 1623 u64 data) 1624 { 1625 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1626 1627 return dd->misc_err_status_cnt[12]; 1628 } 1629 1630 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry, 1631 void *context, int vl, int mode, 1632 u64 data) 1633 { 1634 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1635 1636 return dd->misc_err_status_cnt[11]; 1637 } 1638 1639 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry, 1640 void *context, int vl, int mode, 1641 u64 data) 1642 { 1643 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1644 1645 return dd->misc_err_status_cnt[10]; 1646 } 1647 1648 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry, 1649 void *context, int vl, 1650 int mode, u64 data) 1651 { 1652 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1653 1654 return dd->misc_err_status_cnt[9]; 1655 } 1656 1657 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry, 1658 void *context, int vl, int mode, 1659 u64 data) 1660 { 1661 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1662 1663 return dd->misc_err_status_cnt[8]; 1664 } 1665 1666 static u64 access_misc_efuse_read_bad_addr_err_cnt( 1667 const struct cntr_entry *entry, 1668 void *context, int vl, int mode, u64 data) 1669 { 1670 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1671 1672 return dd->misc_err_status_cnt[7]; 1673 } 1674 1675 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry, 1676 void *context, int vl, 1677 int mode, u64 data) 1678 { 1679 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1680 1681 return dd->misc_err_status_cnt[6]; 1682 } 1683 1684 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry, 1685 void *context, int vl, int mode, 1686 u64 data) 1687 { 1688 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1689 1690 return dd->misc_err_status_cnt[5]; 1691 } 1692 1693 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry, 1694 void *context, int vl, int mode, 1695 u64 data) 1696 { 1697 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1698 1699 return dd->misc_err_status_cnt[4]; 1700 } 1701 1702 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry, 1703 void *context, int vl, 1704 int mode, u64 data) 1705 { 1706 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1707 1708 return dd->misc_err_status_cnt[3]; 1709 } 1710 1711 static u64 access_misc_csr_write_bad_addr_err_cnt( 1712 const struct cntr_entry *entry, 1713 void *context, int vl, int mode, u64 data) 1714 { 1715 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1716 1717 return dd->misc_err_status_cnt[2]; 1718 } 1719 1720 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1721 void *context, int vl, 1722 int mode, u64 data) 1723 { 1724 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1725 1726 return dd->misc_err_status_cnt[1]; 1727 } 1728 1729 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry, 1730 void *context, int vl, int mode, 1731 u64 data) 1732 { 1733 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1734 1735 return dd->misc_err_status_cnt[0]; 1736 } 1737 1738 /* 1739 * Software counter for the aggregate of 1740 * individual CceErrStatus counters 1741 */ 1742 static u64 access_sw_cce_err_status_aggregated_cnt( 1743 const struct cntr_entry *entry, 1744 void *context, int vl, int mode, u64 data) 1745 { 1746 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1747 1748 return dd->sw_cce_err_status_aggregate; 1749 } 1750 1751 /* 1752 * Software counters corresponding to each of the 1753 * error status bits within CceErrStatus 1754 */ 1755 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry, 1756 void *context, int vl, int mode, 1757 u64 data) 1758 { 1759 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1760 1761 return dd->cce_err_status_cnt[40]; 1762 } 1763 1764 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry, 1765 void *context, int vl, int mode, 1766 u64 data) 1767 { 1768 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1769 1770 return dd->cce_err_status_cnt[39]; 1771 } 1772 1773 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry, 1774 void *context, int vl, int mode, 1775 u64 data) 1776 { 1777 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1778 1779 return dd->cce_err_status_cnt[38]; 1780 } 1781 1782 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry, 1783 void *context, int vl, int mode, 1784 u64 data) 1785 { 1786 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1787 1788 return dd->cce_err_status_cnt[37]; 1789 } 1790 1791 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry, 1792 void *context, int vl, int mode, 1793 u64 data) 1794 { 1795 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1796 1797 return dd->cce_err_status_cnt[36]; 1798 } 1799 1800 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt( 1801 const struct cntr_entry *entry, 1802 void *context, int vl, int mode, u64 data) 1803 { 1804 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1805 1806 return dd->cce_err_status_cnt[35]; 1807 } 1808 1809 static u64 access_cce_rcpl_async_fifo_parity_err_cnt( 1810 const struct cntr_entry *entry, 1811 void *context, int vl, int mode, u64 data) 1812 { 1813 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1814 1815 return dd->cce_err_status_cnt[34]; 1816 } 1817 1818 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry, 1819 void *context, int vl, 1820 int mode, u64 data) 1821 { 1822 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1823 1824 return dd->cce_err_status_cnt[33]; 1825 } 1826 1827 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1828 void *context, int vl, int mode, 1829 u64 data) 1830 { 1831 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1832 1833 return dd->cce_err_status_cnt[32]; 1834 } 1835 1836 static u64 access_la_triggered_cnt(const struct cntr_entry *entry, 1837 void *context, int vl, int mode, u64 data) 1838 { 1839 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1840 1841 return dd->cce_err_status_cnt[31]; 1842 } 1843 1844 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry, 1845 void *context, int vl, int mode, 1846 u64 data) 1847 { 1848 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1849 1850 return dd->cce_err_status_cnt[30]; 1851 } 1852 1853 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry, 1854 void *context, int vl, int mode, 1855 u64 data) 1856 { 1857 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1858 1859 return dd->cce_err_status_cnt[29]; 1860 } 1861 1862 static u64 access_pcic_transmit_back_parity_err_cnt( 1863 const struct cntr_entry *entry, 1864 void *context, int vl, int mode, u64 data) 1865 { 1866 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1867 1868 return dd->cce_err_status_cnt[28]; 1869 } 1870 1871 static u64 access_pcic_transmit_front_parity_err_cnt( 1872 const struct cntr_entry *entry, 1873 void *context, int vl, int mode, u64 data) 1874 { 1875 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1876 1877 return dd->cce_err_status_cnt[27]; 1878 } 1879 1880 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1881 void *context, int vl, int mode, 1882 u64 data) 1883 { 1884 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1885 1886 return dd->cce_err_status_cnt[26]; 1887 } 1888 1889 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1890 void *context, int vl, int mode, 1891 u64 data) 1892 { 1893 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1894 1895 return dd->cce_err_status_cnt[25]; 1896 } 1897 1898 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1899 void *context, int vl, int mode, 1900 u64 data) 1901 { 1902 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1903 1904 return dd->cce_err_status_cnt[24]; 1905 } 1906 1907 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1908 void *context, int vl, int mode, 1909 u64 data) 1910 { 1911 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1912 1913 return dd->cce_err_status_cnt[23]; 1914 } 1915 1916 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry, 1917 void *context, int vl, 1918 int mode, u64 data) 1919 { 1920 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1921 1922 return dd->cce_err_status_cnt[22]; 1923 } 1924 1925 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry, 1926 void *context, int vl, int mode, 1927 u64 data) 1928 { 1929 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1930 1931 return dd->cce_err_status_cnt[21]; 1932 } 1933 1934 static u64 access_pcic_n_post_dat_q_parity_err_cnt( 1935 const struct cntr_entry *entry, 1936 void *context, int vl, int mode, u64 data) 1937 { 1938 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1939 1940 return dd->cce_err_status_cnt[20]; 1941 } 1942 1943 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry, 1944 void *context, int vl, 1945 int mode, u64 data) 1946 { 1947 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1948 1949 return dd->cce_err_status_cnt[19]; 1950 } 1951 1952 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry, 1953 void *context, int vl, int mode, 1954 u64 data) 1955 { 1956 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1957 1958 return dd->cce_err_status_cnt[18]; 1959 } 1960 1961 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry, 1962 void *context, int vl, int mode, 1963 u64 data) 1964 { 1965 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1966 1967 return dd->cce_err_status_cnt[17]; 1968 } 1969 1970 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry, 1971 void *context, int vl, int mode, 1972 u64 data) 1973 { 1974 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1975 1976 return dd->cce_err_status_cnt[16]; 1977 } 1978 1979 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry, 1980 void *context, int vl, int mode, 1981 u64 data) 1982 { 1983 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1984 1985 return dd->cce_err_status_cnt[15]; 1986 } 1987 1988 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry, 1989 void *context, int vl, 1990 int mode, u64 data) 1991 { 1992 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1993 1994 return dd->cce_err_status_cnt[14]; 1995 } 1996 1997 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry, 1998 void *context, int vl, int mode, 1999 u64 data) 2000 { 2001 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2002 2003 return dd->cce_err_status_cnt[13]; 2004 } 2005 2006 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt( 2007 const struct cntr_entry *entry, 2008 void *context, int vl, int mode, u64 data) 2009 { 2010 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2011 2012 return dd->cce_err_status_cnt[12]; 2013 } 2014 2015 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt( 2016 const struct cntr_entry *entry, 2017 void *context, int vl, int mode, u64 data) 2018 { 2019 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2020 2021 return dd->cce_err_status_cnt[11]; 2022 } 2023 2024 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt( 2025 const struct cntr_entry *entry, 2026 void *context, int vl, int mode, u64 data) 2027 { 2028 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2029 2030 return dd->cce_err_status_cnt[10]; 2031 } 2032 2033 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt( 2034 const struct cntr_entry *entry, 2035 void *context, int vl, int mode, u64 data) 2036 { 2037 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2038 2039 return dd->cce_err_status_cnt[9]; 2040 } 2041 2042 static u64 access_cce_cli2_async_fifo_parity_err_cnt( 2043 const struct cntr_entry *entry, 2044 void *context, int vl, int mode, u64 data) 2045 { 2046 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2047 2048 return dd->cce_err_status_cnt[8]; 2049 } 2050 2051 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry, 2052 void *context, int vl, 2053 int mode, u64 data) 2054 { 2055 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2056 2057 return dd->cce_err_status_cnt[7]; 2058 } 2059 2060 static u64 access_cce_cli0_async_fifo_parity_err_cnt( 2061 const struct cntr_entry *entry, 2062 void *context, int vl, int mode, u64 data) 2063 { 2064 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2065 2066 return dd->cce_err_status_cnt[6]; 2067 } 2068 2069 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry, 2070 void *context, int vl, int mode, 2071 u64 data) 2072 { 2073 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2074 2075 return dd->cce_err_status_cnt[5]; 2076 } 2077 2078 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry, 2079 void *context, int vl, int mode, 2080 u64 data) 2081 { 2082 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2083 2084 return dd->cce_err_status_cnt[4]; 2085 } 2086 2087 static u64 access_cce_trgt_async_fifo_parity_err_cnt( 2088 const struct cntr_entry *entry, 2089 void *context, int vl, int mode, u64 data) 2090 { 2091 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2092 2093 return dd->cce_err_status_cnt[3]; 2094 } 2095 2096 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2097 void *context, int vl, 2098 int mode, u64 data) 2099 { 2100 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2101 2102 return dd->cce_err_status_cnt[2]; 2103 } 2104 2105 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2106 void *context, int vl, 2107 int mode, u64 data) 2108 { 2109 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2110 2111 return dd->cce_err_status_cnt[1]; 2112 } 2113 2114 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry, 2115 void *context, int vl, int mode, 2116 u64 data) 2117 { 2118 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2119 2120 return dd->cce_err_status_cnt[0]; 2121 } 2122 2123 /* 2124 * Software counters corresponding to each of the 2125 * error status bits within RcvErrStatus 2126 */ 2127 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry, 2128 void *context, int vl, int mode, 2129 u64 data) 2130 { 2131 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2132 2133 return dd->rcv_err_status_cnt[63]; 2134 } 2135 2136 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2137 void *context, int vl, 2138 int mode, u64 data) 2139 { 2140 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2141 2142 return dd->rcv_err_status_cnt[62]; 2143 } 2144 2145 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2146 void *context, int vl, int mode, 2147 u64 data) 2148 { 2149 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2150 2151 return dd->rcv_err_status_cnt[61]; 2152 } 2153 2154 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry, 2155 void *context, int vl, int mode, 2156 u64 data) 2157 { 2158 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2159 2160 return dd->rcv_err_status_cnt[60]; 2161 } 2162 2163 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2164 void *context, int vl, 2165 int mode, u64 data) 2166 { 2167 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2168 2169 return dd->rcv_err_status_cnt[59]; 2170 } 2171 2172 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2173 void *context, int vl, 2174 int mode, u64 data) 2175 { 2176 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2177 2178 return dd->rcv_err_status_cnt[58]; 2179 } 2180 2181 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry, 2182 void *context, int vl, int mode, 2183 u64 data) 2184 { 2185 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2186 2187 return dd->rcv_err_status_cnt[57]; 2188 } 2189 2190 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry, 2191 void *context, int vl, int mode, 2192 u64 data) 2193 { 2194 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2195 2196 return dd->rcv_err_status_cnt[56]; 2197 } 2198 2199 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry, 2200 void *context, int vl, int mode, 2201 u64 data) 2202 { 2203 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2204 2205 return dd->rcv_err_status_cnt[55]; 2206 } 2207 2208 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt( 2209 const struct cntr_entry *entry, 2210 void *context, int vl, int mode, u64 data) 2211 { 2212 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2213 2214 return dd->rcv_err_status_cnt[54]; 2215 } 2216 2217 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt( 2218 const struct cntr_entry *entry, 2219 void *context, int vl, int mode, u64 data) 2220 { 2221 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2222 2223 return dd->rcv_err_status_cnt[53]; 2224 } 2225 2226 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry, 2227 void *context, int vl, 2228 int mode, u64 data) 2229 { 2230 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2231 2232 return dd->rcv_err_status_cnt[52]; 2233 } 2234 2235 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry, 2236 void *context, int vl, 2237 int mode, u64 data) 2238 { 2239 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2240 2241 return dd->rcv_err_status_cnt[51]; 2242 } 2243 2244 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry, 2245 void *context, int vl, 2246 int mode, u64 data) 2247 { 2248 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2249 2250 return dd->rcv_err_status_cnt[50]; 2251 } 2252 2253 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry, 2254 void *context, int vl, 2255 int mode, u64 data) 2256 { 2257 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2258 2259 return dd->rcv_err_status_cnt[49]; 2260 } 2261 2262 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry, 2263 void *context, int vl, 2264 int mode, u64 data) 2265 { 2266 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2267 2268 return dd->rcv_err_status_cnt[48]; 2269 } 2270 2271 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry, 2272 void *context, int vl, 2273 int mode, u64 data) 2274 { 2275 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2276 2277 return dd->rcv_err_status_cnt[47]; 2278 } 2279 2280 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry, 2281 void *context, int vl, int mode, 2282 u64 data) 2283 { 2284 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2285 2286 return dd->rcv_err_status_cnt[46]; 2287 } 2288 2289 static u64 access_rx_hq_intr_csr_parity_err_cnt( 2290 const struct cntr_entry *entry, 2291 void *context, int vl, int mode, u64 data) 2292 { 2293 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2294 2295 return dd->rcv_err_status_cnt[45]; 2296 } 2297 2298 static u64 access_rx_lookup_csr_parity_err_cnt( 2299 const struct cntr_entry *entry, 2300 void *context, int vl, int mode, u64 data) 2301 { 2302 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2303 2304 return dd->rcv_err_status_cnt[44]; 2305 } 2306 2307 static u64 access_rx_lookup_rcv_array_cor_err_cnt( 2308 const struct cntr_entry *entry, 2309 void *context, int vl, int mode, u64 data) 2310 { 2311 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2312 2313 return dd->rcv_err_status_cnt[43]; 2314 } 2315 2316 static u64 access_rx_lookup_rcv_array_unc_err_cnt( 2317 const struct cntr_entry *entry, 2318 void *context, int vl, int mode, u64 data) 2319 { 2320 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2321 2322 return dd->rcv_err_status_cnt[42]; 2323 } 2324 2325 static u64 access_rx_lookup_des_part2_parity_err_cnt( 2326 const struct cntr_entry *entry, 2327 void *context, int vl, int mode, u64 data) 2328 { 2329 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2330 2331 return dd->rcv_err_status_cnt[41]; 2332 } 2333 2334 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt( 2335 const struct cntr_entry *entry, 2336 void *context, int vl, int mode, u64 data) 2337 { 2338 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2339 2340 return dd->rcv_err_status_cnt[40]; 2341 } 2342 2343 static u64 access_rx_lookup_des_part1_unc_err_cnt( 2344 const struct cntr_entry *entry, 2345 void *context, int vl, int mode, u64 data) 2346 { 2347 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2348 2349 return dd->rcv_err_status_cnt[39]; 2350 } 2351 2352 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt( 2353 const struct cntr_entry *entry, 2354 void *context, int vl, int mode, u64 data) 2355 { 2356 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2357 2358 return dd->rcv_err_status_cnt[38]; 2359 } 2360 2361 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt( 2362 const struct cntr_entry *entry, 2363 void *context, int vl, int mode, u64 data) 2364 { 2365 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2366 2367 return dd->rcv_err_status_cnt[37]; 2368 } 2369 2370 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt( 2371 const struct cntr_entry *entry, 2372 void *context, int vl, int mode, u64 data) 2373 { 2374 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2375 2376 return dd->rcv_err_status_cnt[36]; 2377 } 2378 2379 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt( 2380 const struct cntr_entry *entry, 2381 void *context, int vl, int mode, u64 data) 2382 { 2383 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2384 2385 return dd->rcv_err_status_cnt[35]; 2386 } 2387 2388 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt( 2389 const struct cntr_entry *entry, 2390 void *context, int vl, int mode, u64 data) 2391 { 2392 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2393 2394 return dd->rcv_err_status_cnt[34]; 2395 } 2396 2397 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt( 2398 const struct cntr_entry *entry, 2399 void *context, int vl, int mode, u64 data) 2400 { 2401 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2402 2403 return dd->rcv_err_status_cnt[33]; 2404 } 2405 2406 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry, 2407 void *context, int vl, int mode, 2408 u64 data) 2409 { 2410 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2411 2412 return dd->rcv_err_status_cnt[32]; 2413 } 2414 2415 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry, 2416 void *context, int vl, int mode, 2417 u64 data) 2418 { 2419 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2420 2421 return dd->rcv_err_status_cnt[31]; 2422 } 2423 2424 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry, 2425 void *context, int vl, int mode, 2426 u64 data) 2427 { 2428 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2429 2430 return dd->rcv_err_status_cnt[30]; 2431 } 2432 2433 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry, 2434 void *context, int vl, int mode, 2435 u64 data) 2436 { 2437 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2438 2439 return dd->rcv_err_status_cnt[29]; 2440 } 2441 2442 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry, 2443 void *context, int vl, 2444 int mode, u64 data) 2445 { 2446 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2447 2448 return dd->rcv_err_status_cnt[28]; 2449 } 2450 2451 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt( 2452 const struct cntr_entry *entry, 2453 void *context, int vl, int mode, u64 data) 2454 { 2455 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2456 2457 return dd->rcv_err_status_cnt[27]; 2458 } 2459 2460 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt( 2461 const struct cntr_entry *entry, 2462 void *context, int vl, int mode, u64 data) 2463 { 2464 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2465 2466 return dd->rcv_err_status_cnt[26]; 2467 } 2468 2469 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt( 2470 const struct cntr_entry *entry, 2471 void *context, int vl, int mode, u64 data) 2472 { 2473 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2474 2475 return dd->rcv_err_status_cnt[25]; 2476 } 2477 2478 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt( 2479 const struct cntr_entry *entry, 2480 void *context, int vl, int mode, u64 data) 2481 { 2482 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2483 2484 return dd->rcv_err_status_cnt[24]; 2485 } 2486 2487 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt( 2488 const struct cntr_entry *entry, 2489 void *context, int vl, int mode, u64 data) 2490 { 2491 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2492 2493 return dd->rcv_err_status_cnt[23]; 2494 } 2495 2496 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt( 2497 const struct cntr_entry *entry, 2498 void *context, int vl, int mode, u64 data) 2499 { 2500 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2501 2502 return dd->rcv_err_status_cnt[22]; 2503 } 2504 2505 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt( 2506 const struct cntr_entry *entry, 2507 void *context, int vl, int mode, u64 data) 2508 { 2509 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2510 2511 return dd->rcv_err_status_cnt[21]; 2512 } 2513 2514 static u64 access_rx_rbuf_block_list_read_cor_err_cnt( 2515 const struct cntr_entry *entry, 2516 void *context, int vl, int mode, u64 data) 2517 { 2518 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2519 2520 return dd->rcv_err_status_cnt[20]; 2521 } 2522 2523 static u64 access_rx_rbuf_block_list_read_unc_err_cnt( 2524 const struct cntr_entry *entry, 2525 void *context, int vl, int mode, u64 data) 2526 { 2527 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2528 2529 return dd->rcv_err_status_cnt[19]; 2530 } 2531 2532 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry, 2533 void *context, int vl, 2534 int mode, u64 data) 2535 { 2536 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2537 2538 return dd->rcv_err_status_cnt[18]; 2539 } 2540 2541 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry, 2542 void *context, int vl, 2543 int mode, u64 data) 2544 { 2545 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2546 2547 return dd->rcv_err_status_cnt[17]; 2548 } 2549 2550 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt( 2551 const struct cntr_entry *entry, 2552 void *context, int vl, int mode, u64 data) 2553 { 2554 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2555 2556 return dd->rcv_err_status_cnt[16]; 2557 } 2558 2559 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt( 2560 const struct cntr_entry *entry, 2561 void *context, int vl, int mode, u64 data) 2562 { 2563 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2564 2565 return dd->rcv_err_status_cnt[15]; 2566 } 2567 2568 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry, 2569 void *context, int vl, 2570 int mode, u64 data) 2571 { 2572 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2573 2574 return dd->rcv_err_status_cnt[14]; 2575 } 2576 2577 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry, 2578 void *context, int vl, 2579 int mode, u64 data) 2580 { 2581 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2582 2583 return dd->rcv_err_status_cnt[13]; 2584 } 2585 2586 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2587 void *context, int vl, int mode, 2588 u64 data) 2589 { 2590 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2591 2592 return dd->rcv_err_status_cnt[12]; 2593 } 2594 2595 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry, 2596 void *context, int vl, int mode, 2597 u64 data) 2598 { 2599 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2600 2601 return dd->rcv_err_status_cnt[11]; 2602 } 2603 2604 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry, 2605 void *context, int vl, int mode, 2606 u64 data) 2607 { 2608 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2609 2610 return dd->rcv_err_status_cnt[10]; 2611 } 2612 2613 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry, 2614 void *context, int vl, int mode, 2615 u64 data) 2616 { 2617 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2618 2619 return dd->rcv_err_status_cnt[9]; 2620 } 2621 2622 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry, 2623 void *context, int vl, int mode, 2624 u64 data) 2625 { 2626 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2627 2628 return dd->rcv_err_status_cnt[8]; 2629 } 2630 2631 static u64 access_rx_rcv_qp_map_table_cor_err_cnt( 2632 const struct cntr_entry *entry, 2633 void *context, int vl, int mode, u64 data) 2634 { 2635 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2636 2637 return dd->rcv_err_status_cnt[7]; 2638 } 2639 2640 static u64 access_rx_rcv_qp_map_table_unc_err_cnt( 2641 const struct cntr_entry *entry, 2642 void *context, int vl, int mode, u64 data) 2643 { 2644 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2645 2646 return dd->rcv_err_status_cnt[6]; 2647 } 2648 2649 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry, 2650 void *context, int vl, int mode, 2651 u64 data) 2652 { 2653 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2654 2655 return dd->rcv_err_status_cnt[5]; 2656 } 2657 2658 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry, 2659 void *context, int vl, int mode, 2660 u64 data) 2661 { 2662 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2663 2664 return dd->rcv_err_status_cnt[4]; 2665 } 2666 2667 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry, 2668 void *context, int vl, int mode, 2669 u64 data) 2670 { 2671 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2672 2673 return dd->rcv_err_status_cnt[3]; 2674 } 2675 2676 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry, 2677 void *context, int vl, int mode, 2678 u64 data) 2679 { 2680 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2681 2682 return dd->rcv_err_status_cnt[2]; 2683 } 2684 2685 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry, 2686 void *context, int vl, int mode, 2687 u64 data) 2688 { 2689 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2690 2691 return dd->rcv_err_status_cnt[1]; 2692 } 2693 2694 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry, 2695 void *context, int vl, int mode, 2696 u64 data) 2697 { 2698 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2699 2700 return dd->rcv_err_status_cnt[0]; 2701 } 2702 2703 /* 2704 * Software counters corresponding to each of the 2705 * error status bits within SendPioErrStatus 2706 */ 2707 static u64 access_pio_pec_sop_head_parity_err_cnt( 2708 const struct cntr_entry *entry, 2709 void *context, int vl, int mode, u64 data) 2710 { 2711 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2712 2713 return dd->send_pio_err_status_cnt[35]; 2714 } 2715 2716 static u64 access_pio_pcc_sop_head_parity_err_cnt( 2717 const struct cntr_entry *entry, 2718 void *context, int vl, int mode, u64 data) 2719 { 2720 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2721 2722 return dd->send_pio_err_status_cnt[34]; 2723 } 2724 2725 static u64 access_pio_last_returned_cnt_parity_err_cnt( 2726 const struct cntr_entry *entry, 2727 void *context, int vl, int mode, u64 data) 2728 { 2729 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2730 2731 return dd->send_pio_err_status_cnt[33]; 2732 } 2733 2734 static u64 access_pio_current_free_cnt_parity_err_cnt( 2735 const struct cntr_entry *entry, 2736 void *context, int vl, int mode, u64 data) 2737 { 2738 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2739 2740 return dd->send_pio_err_status_cnt[32]; 2741 } 2742 2743 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry, 2744 void *context, int vl, int mode, 2745 u64 data) 2746 { 2747 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2748 2749 return dd->send_pio_err_status_cnt[31]; 2750 } 2751 2752 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry, 2753 void *context, int vl, int mode, 2754 u64 data) 2755 { 2756 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2757 2758 return dd->send_pio_err_status_cnt[30]; 2759 } 2760 2761 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry, 2762 void *context, int vl, int mode, 2763 u64 data) 2764 { 2765 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2766 2767 return dd->send_pio_err_status_cnt[29]; 2768 } 2769 2770 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt( 2771 const struct cntr_entry *entry, 2772 void *context, int vl, int mode, u64 data) 2773 { 2774 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2775 2776 return dd->send_pio_err_status_cnt[28]; 2777 } 2778 2779 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry, 2780 void *context, int vl, int mode, 2781 u64 data) 2782 { 2783 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2784 2785 return dd->send_pio_err_status_cnt[27]; 2786 } 2787 2788 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry, 2789 void *context, int vl, int mode, 2790 u64 data) 2791 { 2792 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2793 2794 return dd->send_pio_err_status_cnt[26]; 2795 } 2796 2797 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry, 2798 void *context, int vl, 2799 int mode, u64 data) 2800 { 2801 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2802 2803 return dd->send_pio_err_status_cnt[25]; 2804 } 2805 2806 static u64 access_pio_block_qw_count_parity_err_cnt( 2807 const struct cntr_entry *entry, 2808 void *context, int vl, int mode, u64 data) 2809 { 2810 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2811 2812 return dd->send_pio_err_status_cnt[24]; 2813 } 2814 2815 static u64 access_pio_write_qw_valid_parity_err_cnt( 2816 const struct cntr_entry *entry, 2817 void *context, int vl, int mode, u64 data) 2818 { 2819 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2820 2821 return dd->send_pio_err_status_cnt[23]; 2822 } 2823 2824 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry, 2825 void *context, int vl, int mode, 2826 u64 data) 2827 { 2828 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2829 2830 return dd->send_pio_err_status_cnt[22]; 2831 } 2832 2833 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry, 2834 void *context, int vl, 2835 int mode, u64 data) 2836 { 2837 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2838 2839 return dd->send_pio_err_status_cnt[21]; 2840 } 2841 2842 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry, 2843 void *context, int vl, 2844 int mode, u64 data) 2845 { 2846 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2847 2848 return dd->send_pio_err_status_cnt[20]; 2849 } 2850 2851 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry, 2852 void *context, int vl, 2853 int mode, u64 data) 2854 { 2855 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2856 2857 return dd->send_pio_err_status_cnt[19]; 2858 } 2859 2860 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt( 2861 const struct cntr_entry *entry, 2862 void *context, int vl, int mode, u64 data) 2863 { 2864 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2865 2866 return dd->send_pio_err_status_cnt[18]; 2867 } 2868 2869 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry, 2870 void *context, int vl, int mode, 2871 u64 data) 2872 { 2873 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2874 2875 return dd->send_pio_err_status_cnt[17]; 2876 } 2877 2878 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry, 2879 void *context, int vl, int mode, 2880 u64 data) 2881 { 2882 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2883 2884 return dd->send_pio_err_status_cnt[16]; 2885 } 2886 2887 static u64 access_pio_credit_ret_fifo_parity_err_cnt( 2888 const struct cntr_entry *entry, 2889 void *context, int vl, int mode, u64 data) 2890 { 2891 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2892 2893 return dd->send_pio_err_status_cnt[15]; 2894 } 2895 2896 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt( 2897 const struct cntr_entry *entry, 2898 void *context, int vl, int mode, u64 data) 2899 { 2900 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2901 2902 return dd->send_pio_err_status_cnt[14]; 2903 } 2904 2905 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt( 2906 const struct cntr_entry *entry, 2907 void *context, int vl, int mode, u64 data) 2908 { 2909 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2910 2911 return dd->send_pio_err_status_cnt[13]; 2912 } 2913 2914 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt( 2915 const struct cntr_entry *entry, 2916 void *context, int vl, int mode, u64 data) 2917 { 2918 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2919 2920 return dd->send_pio_err_status_cnt[12]; 2921 } 2922 2923 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt( 2924 const struct cntr_entry *entry, 2925 void *context, int vl, int mode, u64 data) 2926 { 2927 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2928 2929 return dd->send_pio_err_status_cnt[11]; 2930 } 2931 2932 static u64 access_pio_sm_pkt_reset_parity_err_cnt( 2933 const struct cntr_entry *entry, 2934 void *context, int vl, int mode, u64 data) 2935 { 2936 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2937 2938 return dd->send_pio_err_status_cnt[10]; 2939 } 2940 2941 static u64 access_pio_pkt_evict_fifo_parity_err_cnt( 2942 const struct cntr_entry *entry, 2943 void *context, int vl, int mode, u64 data) 2944 { 2945 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2946 2947 return dd->send_pio_err_status_cnt[9]; 2948 } 2949 2950 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt( 2951 const struct cntr_entry *entry, 2952 void *context, int vl, int mode, u64 data) 2953 { 2954 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2955 2956 return dd->send_pio_err_status_cnt[8]; 2957 } 2958 2959 static u64 access_pio_sbrdctl_crrel_parity_err_cnt( 2960 const struct cntr_entry *entry, 2961 void *context, int vl, int mode, u64 data) 2962 { 2963 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2964 2965 return dd->send_pio_err_status_cnt[7]; 2966 } 2967 2968 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry, 2969 void *context, int vl, int mode, 2970 u64 data) 2971 { 2972 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2973 2974 return dd->send_pio_err_status_cnt[6]; 2975 } 2976 2977 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry, 2978 void *context, int vl, int mode, 2979 u64 data) 2980 { 2981 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2982 2983 return dd->send_pio_err_status_cnt[5]; 2984 } 2985 2986 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry, 2987 void *context, int vl, int mode, 2988 u64 data) 2989 { 2990 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2991 2992 return dd->send_pio_err_status_cnt[4]; 2993 } 2994 2995 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry, 2996 void *context, int vl, int mode, 2997 u64 data) 2998 { 2999 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3000 3001 return dd->send_pio_err_status_cnt[3]; 3002 } 3003 3004 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry, 3005 void *context, int vl, int mode, 3006 u64 data) 3007 { 3008 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3009 3010 return dd->send_pio_err_status_cnt[2]; 3011 } 3012 3013 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry, 3014 void *context, int vl, 3015 int mode, u64 data) 3016 { 3017 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3018 3019 return dd->send_pio_err_status_cnt[1]; 3020 } 3021 3022 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry, 3023 void *context, int vl, int mode, 3024 u64 data) 3025 { 3026 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3027 3028 return dd->send_pio_err_status_cnt[0]; 3029 } 3030 3031 /* 3032 * Software counters corresponding to each of the 3033 * error status bits within SendDmaErrStatus 3034 */ 3035 static u64 access_sdma_pcie_req_tracking_cor_err_cnt( 3036 const struct cntr_entry *entry, 3037 void *context, int vl, int mode, u64 data) 3038 { 3039 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3040 3041 return dd->send_dma_err_status_cnt[3]; 3042 } 3043 3044 static u64 access_sdma_pcie_req_tracking_unc_err_cnt( 3045 const struct cntr_entry *entry, 3046 void *context, int vl, int mode, u64 data) 3047 { 3048 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3049 3050 return dd->send_dma_err_status_cnt[2]; 3051 } 3052 3053 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry, 3054 void *context, int vl, int mode, 3055 u64 data) 3056 { 3057 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3058 3059 return dd->send_dma_err_status_cnt[1]; 3060 } 3061 3062 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry, 3063 void *context, int vl, int mode, 3064 u64 data) 3065 { 3066 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3067 3068 return dd->send_dma_err_status_cnt[0]; 3069 } 3070 3071 /* 3072 * Software counters corresponding to each of the 3073 * error status bits within SendEgressErrStatus 3074 */ 3075 static u64 access_tx_read_pio_memory_csr_unc_err_cnt( 3076 const struct cntr_entry *entry, 3077 void *context, int vl, int mode, u64 data) 3078 { 3079 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3080 3081 return dd->send_egress_err_status_cnt[63]; 3082 } 3083 3084 static u64 access_tx_read_sdma_memory_csr_err_cnt( 3085 const struct cntr_entry *entry, 3086 void *context, int vl, int mode, u64 data) 3087 { 3088 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3089 3090 return dd->send_egress_err_status_cnt[62]; 3091 } 3092 3093 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry, 3094 void *context, int vl, int mode, 3095 u64 data) 3096 { 3097 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3098 3099 return dd->send_egress_err_status_cnt[61]; 3100 } 3101 3102 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry, 3103 void *context, int vl, 3104 int mode, u64 data) 3105 { 3106 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3107 3108 return dd->send_egress_err_status_cnt[60]; 3109 } 3110 3111 static u64 access_tx_read_sdma_memory_cor_err_cnt( 3112 const struct cntr_entry *entry, 3113 void *context, int vl, int mode, u64 data) 3114 { 3115 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3116 3117 return dd->send_egress_err_status_cnt[59]; 3118 } 3119 3120 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry, 3121 void *context, int vl, int mode, 3122 u64 data) 3123 { 3124 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3125 3126 return dd->send_egress_err_status_cnt[58]; 3127 } 3128 3129 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry, 3130 void *context, int vl, int mode, 3131 u64 data) 3132 { 3133 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3134 3135 return dd->send_egress_err_status_cnt[57]; 3136 } 3137 3138 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry, 3139 void *context, int vl, int mode, 3140 u64 data) 3141 { 3142 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3143 3144 return dd->send_egress_err_status_cnt[56]; 3145 } 3146 3147 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry, 3148 void *context, int vl, int mode, 3149 u64 data) 3150 { 3151 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3152 3153 return dd->send_egress_err_status_cnt[55]; 3154 } 3155 3156 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry, 3157 void *context, int vl, int mode, 3158 u64 data) 3159 { 3160 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3161 3162 return dd->send_egress_err_status_cnt[54]; 3163 } 3164 3165 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry, 3166 void *context, int vl, int mode, 3167 u64 data) 3168 { 3169 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3170 3171 return dd->send_egress_err_status_cnt[53]; 3172 } 3173 3174 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry, 3175 void *context, int vl, int mode, 3176 u64 data) 3177 { 3178 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3179 3180 return dd->send_egress_err_status_cnt[52]; 3181 } 3182 3183 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry, 3184 void *context, int vl, int mode, 3185 u64 data) 3186 { 3187 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3188 3189 return dd->send_egress_err_status_cnt[51]; 3190 } 3191 3192 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry, 3193 void *context, int vl, int mode, 3194 u64 data) 3195 { 3196 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3197 3198 return dd->send_egress_err_status_cnt[50]; 3199 } 3200 3201 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry, 3202 void *context, int vl, int mode, 3203 u64 data) 3204 { 3205 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3206 3207 return dd->send_egress_err_status_cnt[49]; 3208 } 3209 3210 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry, 3211 void *context, int vl, int mode, 3212 u64 data) 3213 { 3214 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3215 3216 return dd->send_egress_err_status_cnt[48]; 3217 } 3218 3219 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry, 3220 void *context, int vl, int mode, 3221 u64 data) 3222 { 3223 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3224 3225 return dd->send_egress_err_status_cnt[47]; 3226 } 3227 3228 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry, 3229 void *context, int vl, int mode, 3230 u64 data) 3231 { 3232 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3233 3234 return dd->send_egress_err_status_cnt[46]; 3235 } 3236 3237 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry, 3238 void *context, int vl, int mode, 3239 u64 data) 3240 { 3241 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3242 3243 return dd->send_egress_err_status_cnt[45]; 3244 } 3245 3246 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry, 3247 void *context, int vl, 3248 int mode, u64 data) 3249 { 3250 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3251 3252 return dd->send_egress_err_status_cnt[44]; 3253 } 3254 3255 static u64 access_tx_read_sdma_memory_unc_err_cnt( 3256 const struct cntr_entry *entry, 3257 void *context, int vl, int mode, u64 data) 3258 { 3259 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3260 3261 return dd->send_egress_err_status_cnt[43]; 3262 } 3263 3264 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry, 3265 void *context, int vl, int mode, 3266 u64 data) 3267 { 3268 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3269 3270 return dd->send_egress_err_status_cnt[42]; 3271 } 3272 3273 static u64 access_tx_credit_return_partiy_err_cnt( 3274 const struct cntr_entry *entry, 3275 void *context, int vl, int mode, u64 data) 3276 { 3277 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3278 3279 return dd->send_egress_err_status_cnt[41]; 3280 } 3281 3282 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt( 3283 const struct cntr_entry *entry, 3284 void *context, int vl, int mode, u64 data) 3285 { 3286 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3287 3288 return dd->send_egress_err_status_cnt[40]; 3289 } 3290 3291 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt( 3292 const struct cntr_entry *entry, 3293 void *context, int vl, int mode, u64 data) 3294 { 3295 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3296 3297 return dd->send_egress_err_status_cnt[39]; 3298 } 3299 3300 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt( 3301 const struct cntr_entry *entry, 3302 void *context, int vl, int mode, u64 data) 3303 { 3304 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3305 3306 return dd->send_egress_err_status_cnt[38]; 3307 } 3308 3309 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt( 3310 const struct cntr_entry *entry, 3311 void *context, int vl, int mode, u64 data) 3312 { 3313 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3314 3315 return dd->send_egress_err_status_cnt[37]; 3316 } 3317 3318 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt( 3319 const struct cntr_entry *entry, 3320 void *context, int vl, int mode, u64 data) 3321 { 3322 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3323 3324 return dd->send_egress_err_status_cnt[36]; 3325 } 3326 3327 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt( 3328 const struct cntr_entry *entry, 3329 void *context, int vl, int mode, u64 data) 3330 { 3331 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3332 3333 return dd->send_egress_err_status_cnt[35]; 3334 } 3335 3336 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt( 3337 const struct cntr_entry *entry, 3338 void *context, int vl, int mode, u64 data) 3339 { 3340 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3341 3342 return dd->send_egress_err_status_cnt[34]; 3343 } 3344 3345 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt( 3346 const struct cntr_entry *entry, 3347 void *context, int vl, int mode, u64 data) 3348 { 3349 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3350 3351 return dd->send_egress_err_status_cnt[33]; 3352 } 3353 3354 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt( 3355 const struct cntr_entry *entry, 3356 void *context, int vl, int mode, u64 data) 3357 { 3358 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3359 3360 return dd->send_egress_err_status_cnt[32]; 3361 } 3362 3363 static u64 access_tx_sdma15_disallowed_packet_err_cnt( 3364 const struct cntr_entry *entry, 3365 void *context, int vl, int mode, u64 data) 3366 { 3367 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3368 3369 return dd->send_egress_err_status_cnt[31]; 3370 } 3371 3372 static u64 access_tx_sdma14_disallowed_packet_err_cnt( 3373 const struct cntr_entry *entry, 3374 void *context, int vl, int mode, u64 data) 3375 { 3376 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3377 3378 return dd->send_egress_err_status_cnt[30]; 3379 } 3380 3381 static u64 access_tx_sdma13_disallowed_packet_err_cnt( 3382 const struct cntr_entry *entry, 3383 void *context, int vl, int mode, u64 data) 3384 { 3385 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3386 3387 return dd->send_egress_err_status_cnt[29]; 3388 } 3389 3390 static u64 access_tx_sdma12_disallowed_packet_err_cnt( 3391 const struct cntr_entry *entry, 3392 void *context, int vl, int mode, u64 data) 3393 { 3394 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3395 3396 return dd->send_egress_err_status_cnt[28]; 3397 } 3398 3399 static u64 access_tx_sdma11_disallowed_packet_err_cnt( 3400 const struct cntr_entry *entry, 3401 void *context, int vl, int mode, u64 data) 3402 { 3403 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3404 3405 return dd->send_egress_err_status_cnt[27]; 3406 } 3407 3408 static u64 access_tx_sdma10_disallowed_packet_err_cnt( 3409 const struct cntr_entry *entry, 3410 void *context, int vl, int mode, u64 data) 3411 { 3412 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3413 3414 return dd->send_egress_err_status_cnt[26]; 3415 } 3416 3417 static u64 access_tx_sdma9_disallowed_packet_err_cnt( 3418 const struct cntr_entry *entry, 3419 void *context, int vl, int mode, u64 data) 3420 { 3421 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3422 3423 return dd->send_egress_err_status_cnt[25]; 3424 } 3425 3426 static u64 access_tx_sdma8_disallowed_packet_err_cnt( 3427 const struct cntr_entry *entry, 3428 void *context, int vl, int mode, u64 data) 3429 { 3430 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3431 3432 return dd->send_egress_err_status_cnt[24]; 3433 } 3434 3435 static u64 access_tx_sdma7_disallowed_packet_err_cnt( 3436 const struct cntr_entry *entry, 3437 void *context, int vl, int mode, u64 data) 3438 { 3439 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3440 3441 return dd->send_egress_err_status_cnt[23]; 3442 } 3443 3444 static u64 access_tx_sdma6_disallowed_packet_err_cnt( 3445 const struct cntr_entry *entry, 3446 void *context, int vl, int mode, u64 data) 3447 { 3448 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3449 3450 return dd->send_egress_err_status_cnt[22]; 3451 } 3452 3453 static u64 access_tx_sdma5_disallowed_packet_err_cnt( 3454 const struct cntr_entry *entry, 3455 void *context, int vl, int mode, u64 data) 3456 { 3457 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3458 3459 return dd->send_egress_err_status_cnt[21]; 3460 } 3461 3462 static u64 access_tx_sdma4_disallowed_packet_err_cnt( 3463 const struct cntr_entry *entry, 3464 void *context, int vl, int mode, u64 data) 3465 { 3466 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3467 3468 return dd->send_egress_err_status_cnt[20]; 3469 } 3470 3471 static u64 access_tx_sdma3_disallowed_packet_err_cnt( 3472 const struct cntr_entry *entry, 3473 void *context, int vl, int mode, u64 data) 3474 { 3475 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3476 3477 return dd->send_egress_err_status_cnt[19]; 3478 } 3479 3480 static u64 access_tx_sdma2_disallowed_packet_err_cnt( 3481 const struct cntr_entry *entry, 3482 void *context, int vl, int mode, u64 data) 3483 { 3484 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3485 3486 return dd->send_egress_err_status_cnt[18]; 3487 } 3488 3489 static u64 access_tx_sdma1_disallowed_packet_err_cnt( 3490 const struct cntr_entry *entry, 3491 void *context, int vl, int mode, u64 data) 3492 { 3493 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3494 3495 return dd->send_egress_err_status_cnt[17]; 3496 } 3497 3498 static u64 access_tx_sdma0_disallowed_packet_err_cnt( 3499 const struct cntr_entry *entry, 3500 void *context, int vl, int mode, u64 data) 3501 { 3502 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3503 3504 return dd->send_egress_err_status_cnt[16]; 3505 } 3506 3507 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry, 3508 void *context, int vl, int mode, 3509 u64 data) 3510 { 3511 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3512 3513 return dd->send_egress_err_status_cnt[15]; 3514 } 3515 3516 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry, 3517 void *context, int vl, 3518 int mode, u64 data) 3519 { 3520 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3521 3522 return dd->send_egress_err_status_cnt[14]; 3523 } 3524 3525 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry, 3526 void *context, int vl, int mode, 3527 u64 data) 3528 { 3529 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3530 3531 return dd->send_egress_err_status_cnt[13]; 3532 } 3533 3534 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry, 3535 void *context, int vl, int mode, 3536 u64 data) 3537 { 3538 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3539 3540 return dd->send_egress_err_status_cnt[12]; 3541 } 3542 3543 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt( 3544 const struct cntr_entry *entry, 3545 void *context, int vl, int mode, u64 data) 3546 { 3547 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3548 3549 return dd->send_egress_err_status_cnt[11]; 3550 } 3551 3552 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry, 3553 void *context, int vl, int mode, 3554 u64 data) 3555 { 3556 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3557 3558 return dd->send_egress_err_status_cnt[10]; 3559 } 3560 3561 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry, 3562 void *context, int vl, int mode, 3563 u64 data) 3564 { 3565 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3566 3567 return dd->send_egress_err_status_cnt[9]; 3568 } 3569 3570 static u64 access_tx_sdma_launch_intf_parity_err_cnt( 3571 const struct cntr_entry *entry, 3572 void *context, int vl, int mode, u64 data) 3573 { 3574 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3575 3576 return dd->send_egress_err_status_cnt[8]; 3577 } 3578 3579 static u64 access_tx_pio_launch_intf_parity_err_cnt( 3580 const struct cntr_entry *entry, 3581 void *context, int vl, int mode, u64 data) 3582 { 3583 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3584 3585 return dd->send_egress_err_status_cnt[7]; 3586 } 3587 3588 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry, 3589 void *context, int vl, int mode, 3590 u64 data) 3591 { 3592 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3593 3594 return dd->send_egress_err_status_cnt[6]; 3595 } 3596 3597 static u64 access_tx_incorrect_link_state_err_cnt( 3598 const struct cntr_entry *entry, 3599 void *context, int vl, int mode, u64 data) 3600 { 3601 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3602 3603 return dd->send_egress_err_status_cnt[5]; 3604 } 3605 3606 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry, 3607 void *context, int vl, int mode, 3608 u64 data) 3609 { 3610 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3611 3612 return dd->send_egress_err_status_cnt[4]; 3613 } 3614 3615 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt( 3616 const struct cntr_entry *entry, 3617 void *context, int vl, int mode, u64 data) 3618 { 3619 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3620 3621 return dd->send_egress_err_status_cnt[3]; 3622 } 3623 3624 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry, 3625 void *context, int vl, int mode, 3626 u64 data) 3627 { 3628 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3629 3630 return dd->send_egress_err_status_cnt[2]; 3631 } 3632 3633 static u64 access_tx_pkt_integrity_mem_unc_err_cnt( 3634 const struct cntr_entry *entry, 3635 void *context, int vl, int mode, u64 data) 3636 { 3637 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3638 3639 return dd->send_egress_err_status_cnt[1]; 3640 } 3641 3642 static u64 access_tx_pkt_integrity_mem_cor_err_cnt( 3643 const struct cntr_entry *entry, 3644 void *context, int vl, int mode, u64 data) 3645 { 3646 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3647 3648 return dd->send_egress_err_status_cnt[0]; 3649 } 3650 3651 /* 3652 * Software counters corresponding to each of the 3653 * error status bits within SendErrStatus 3654 */ 3655 static u64 access_send_csr_write_bad_addr_err_cnt( 3656 const struct cntr_entry *entry, 3657 void *context, int vl, int mode, u64 data) 3658 { 3659 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3660 3661 return dd->send_err_status_cnt[2]; 3662 } 3663 3664 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 3665 void *context, int vl, 3666 int mode, u64 data) 3667 { 3668 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3669 3670 return dd->send_err_status_cnt[1]; 3671 } 3672 3673 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry, 3674 void *context, int vl, int mode, 3675 u64 data) 3676 { 3677 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3678 3679 return dd->send_err_status_cnt[0]; 3680 } 3681 3682 /* 3683 * Software counters corresponding to each of the 3684 * error status bits within SendCtxtErrStatus 3685 */ 3686 static u64 access_pio_write_out_of_bounds_err_cnt( 3687 const struct cntr_entry *entry, 3688 void *context, int vl, int mode, u64 data) 3689 { 3690 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3691 3692 return dd->sw_ctxt_err_status_cnt[4]; 3693 } 3694 3695 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry, 3696 void *context, int vl, int mode, 3697 u64 data) 3698 { 3699 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3700 3701 return dd->sw_ctxt_err_status_cnt[3]; 3702 } 3703 3704 static u64 access_pio_write_crosses_boundary_err_cnt( 3705 const struct cntr_entry *entry, 3706 void *context, int vl, int mode, u64 data) 3707 { 3708 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3709 3710 return dd->sw_ctxt_err_status_cnt[2]; 3711 } 3712 3713 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry, 3714 void *context, int vl, 3715 int mode, u64 data) 3716 { 3717 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3718 3719 return dd->sw_ctxt_err_status_cnt[1]; 3720 } 3721 3722 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry, 3723 void *context, int vl, int mode, 3724 u64 data) 3725 { 3726 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3727 3728 return dd->sw_ctxt_err_status_cnt[0]; 3729 } 3730 3731 /* 3732 * Software counters corresponding to each of the 3733 * error status bits within SendDmaEngErrStatus 3734 */ 3735 static u64 access_sdma_header_request_fifo_cor_err_cnt( 3736 const struct cntr_entry *entry, 3737 void *context, int vl, int mode, u64 data) 3738 { 3739 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3740 3741 return dd->sw_send_dma_eng_err_status_cnt[23]; 3742 } 3743 3744 static u64 access_sdma_header_storage_cor_err_cnt( 3745 const struct cntr_entry *entry, 3746 void *context, int vl, int mode, u64 data) 3747 { 3748 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3749 3750 return dd->sw_send_dma_eng_err_status_cnt[22]; 3751 } 3752 3753 static u64 access_sdma_packet_tracking_cor_err_cnt( 3754 const struct cntr_entry *entry, 3755 void *context, int vl, int mode, u64 data) 3756 { 3757 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3758 3759 return dd->sw_send_dma_eng_err_status_cnt[21]; 3760 } 3761 3762 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry, 3763 void *context, int vl, int mode, 3764 u64 data) 3765 { 3766 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3767 3768 return dd->sw_send_dma_eng_err_status_cnt[20]; 3769 } 3770 3771 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry, 3772 void *context, int vl, int mode, 3773 u64 data) 3774 { 3775 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3776 3777 return dd->sw_send_dma_eng_err_status_cnt[19]; 3778 } 3779 3780 static u64 access_sdma_header_request_fifo_unc_err_cnt( 3781 const struct cntr_entry *entry, 3782 void *context, int vl, int mode, u64 data) 3783 { 3784 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3785 3786 return dd->sw_send_dma_eng_err_status_cnt[18]; 3787 } 3788 3789 static u64 access_sdma_header_storage_unc_err_cnt( 3790 const struct cntr_entry *entry, 3791 void *context, int vl, int mode, u64 data) 3792 { 3793 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3794 3795 return dd->sw_send_dma_eng_err_status_cnt[17]; 3796 } 3797 3798 static u64 access_sdma_packet_tracking_unc_err_cnt( 3799 const struct cntr_entry *entry, 3800 void *context, int vl, int mode, u64 data) 3801 { 3802 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3803 3804 return dd->sw_send_dma_eng_err_status_cnt[16]; 3805 } 3806 3807 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry, 3808 void *context, int vl, int mode, 3809 u64 data) 3810 { 3811 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3812 3813 return dd->sw_send_dma_eng_err_status_cnt[15]; 3814 } 3815 3816 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry, 3817 void *context, int vl, int mode, 3818 u64 data) 3819 { 3820 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3821 3822 return dd->sw_send_dma_eng_err_status_cnt[14]; 3823 } 3824 3825 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry, 3826 void *context, int vl, int mode, 3827 u64 data) 3828 { 3829 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3830 3831 return dd->sw_send_dma_eng_err_status_cnt[13]; 3832 } 3833 3834 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry, 3835 void *context, int vl, int mode, 3836 u64 data) 3837 { 3838 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3839 3840 return dd->sw_send_dma_eng_err_status_cnt[12]; 3841 } 3842 3843 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry, 3844 void *context, int vl, int mode, 3845 u64 data) 3846 { 3847 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3848 3849 return dd->sw_send_dma_eng_err_status_cnt[11]; 3850 } 3851 3852 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry, 3853 void *context, int vl, int mode, 3854 u64 data) 3855 { 3856 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3857 3858 return dd->sw_send_dma_eng_err_status_cnt[10]; 3859 } 3860 3861 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry, 3862 void *context, int vl, int mode, 3863 u64 data) 3864 { 3865 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3866 3867 return dd->sw_send_dma_eng_err_status_cnt[9]; 3868 } 3869 3870 static u64 access_sdma_packet_desc_overflow_err_cnt( 3871 const struct cntr_entry *entry, 3872 void *context, int vl, int mode, u64 data) 3873 { 3874 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3875 3876 return dd->sw_send_dma_eng_err_status_cnt[8]; 3877 } 3878 3879 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry, 3880 void *context, int vl, 3881 int mode, u64 data) 3882 { 3883 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3884 3885 return dd->sw_send_dma_eng_err_status_cnt[7]; 3886 } 3887 3888 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry, 3889 void *context, int vl, int mode, u64 data) 3890 { 3891 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3892 3893 return dd->sw_send_dma_eng_err_status_cnt[6]; 3894 } 3895 3896 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry, 3897 void *context, int vl, int mode, 3898 u64 data) 3899 { 3900 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3901 3902 return dd->sw_send_dma_eng_err_status_cnt[5]; 3903 } 3904 3905 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry, 3906 void *context, int vl, int mode, 3907 u64 data) 3908 { 3909 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3910 3911 return dd->sw_send_dma_eng_err_status_cnt[4]; 3912 } 3913 3914 static u64 access_sdma_tail_out_of_bounds_err_cnt( 3915 const struct cntr_entry *entry, 3916 void *context, int vl, int mode, u64 data) 3917 { 3918 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3919 3920 return dd->sw_send_dma_eng_err_status_cnt[3]; 3921 } 3922 3923 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry, 3924 void *context, int vl, int mode, 3925 u64 data) 3926 { 3927 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3928 3929 return dd->sw_send_dma_eng_err_status_cnt[2]; 3930 } 3931 3932 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry, 3933 void *context, int vl, int mode, 3934 u64 data) 3935 { 3936 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3937 3938 return dd->sw_send_dma_eng_err_status_cnt[1]; 3939 } 3940 3941 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry, 3942 void *context, int vl, int mode, 3943 u64 data) 3944 { 3945 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3946 3947 return dd->sw_send_dma_eng_err_status_cnt[0]; 3948 } 3949 3950 #define def_access_sw_cpu(cntr) \ 3951 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \ 3952 void *context, int vl, int mode, u64 data) \ 3953 { \ 3954 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 3955 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \ 3956 ppd->ibport_data.rvp.cntr, vl, \ 3957 mode, data); \ 3958 } 3959 3960 def_access_sw_cpu(rc_acks); 3961 def_access_sw_cpu(rc_qacks); 3962 def_access_sw_cpu(rc_delayed_comp); 3963 3964 #define def_access_ibp_counter(cntr) \ 3965 static u64 access_ibp_##cntr(const struct cntr_entry *entry, \ 3966 void *context, int vl, int mode, u64 data) \ 3967 { \ 3968 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 3969 \ 3970 if (vl != CNTR_INVALID_VL) \ 3971 return 0; \ 3972 \ 3973 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \ 3974 mode, data); \ 3975 } 3976 3977 def_access_ibp_counter(loop_pkts); 3978 def_access_ibp_counter(rc_resends); 3979 def_access_ibp_counter(rnr_naks); 3980 def_access_ibp_counter(other_naks); 3981 def_access_ibp_counter(rc_timeouts); 3982 def_access_ibp_counter(pkt_drops); 3983 def_access_ibp_counter(dmawait); 3984 def_access_ibp_counter(rc_seqnak); 3985 def_access_ibp_counter(rc_dupreq); 3986 def_access_ibp_counter(rdma_seq); 3987 def_access_ibp_counter(unaligned); 3988 def_access_ibp_counter(seq_naks); 3989 3990 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { 3991 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH), 3992 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT, 3993 CNTR_NORMAL), 3994 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT, 3995 CNTR_NORMAL), 3996 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs, 3997 RCV_TID_FLOW_GEN_MISMATCH_CNT, 3998 CNTR_NORMAL), 3999 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL, 4000 CNTR_NORMAL), 4001 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs, 4002 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL), 4003 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt, 4004 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL), 4005 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT, 4006 CNTR_NORMAL), 4007 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT, 4008 CNTR_NORMAL), 4009 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT, 4010 CNTR_NORMAL), 4011 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT, 4012 CNTR_NORMAL), 4013 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT, 4014 CNTR_NORMAL), 4015 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT, 4016 CNTR_NORMAL), 4017 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt, 4018 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL), 4019 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt, 4020 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL), 4021 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT, 4022 CNTR_SYNTH), 4023 [C_DC_RCV_ERR] = DC_PERF_CNTR(DcRecvErr, DCC_ERR_PORTRCV_ERR_CNT, CNTR_SYNTH), 4024 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT, 4025 CNTR_SYNTH), 4026 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT, 4027 CNTR_SYNTH), 4028 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT, 4029 CNTR_SYNTH), 4030 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts, 4031 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH), 4032 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts, 4033 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT, 4034 CNTR_SYNTH), 4035 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr, 4036 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH), 4037 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT, 4038 CNTR_SYNTH), 4039 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT, 4040 CNTR_SYNTH), 4041 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT, 4042 CNTR_SYNTH), 4043 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT, 4044 CNTR_SYNTH), 4045 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT, 4046 CNTR_SYNTH), 4047 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT, 4048 CNTR_SYNTH), 4049 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT, 4050 CNTR_SYNTH), 4051 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT, 4052 CNTR_SYNTH | CNTR_VL), 4053 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT, 4054 CNTR_SYNTH | CNTR_VL), 4055 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH), 4056 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT, 4057 CNTR_SYNTH | CNTR_VL), 4058 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH), 4059 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT, 4060 CNTR_SYNTH | CNTR_VL), 4061 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT, 4062 CNTR_SYNTH), 4063 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT, 4064 CNTR_SYNTH | CNTR_VL), 4065 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT, 4066 CNTR_SYNTH), 4067 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT, 4068 CNTR_SYNTH | CNTR_VL), 4069 [C_DC_TOTAL_CRC] = 4070 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR, 4071 CNTR_SYNTH), 4072 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0, 4073 CNTR_SYNTH), 4074 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1, 4075 CNTR_SYNTH), 4076 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2, 4077 CNTR_SYNTH), 4078 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3, 4079 CNTR_SYNTH), 4080 [C_DC_CRC_MULT_LN] = 4081 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN, 4082 CNTR_SYNTH), 4083 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT, 4084 CNTR_SYNTH), 4085 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT, 4086 CNTR_SYNTH), 4087 [C_DC_SEQ_CRC_CNT] = 4088 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT, 4089 CNTR_SYNTH), 4090 [C_DC_ESC0_ONLY_CNT] = 4091 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT, 4092 CNTR_SYNTH), 4093 [C_DC_ESC0_PLUS1_CNT] = 4094 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT, 4095 CNTR_SYNTH), 4096 [C_DC_ESC0_PLUS2_CNT] = 4097 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT, 4098 CNTR_SYNTH), 4099 [C_DC_REINIT_FROM_PEER_CNT] = 4100 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 4101 CNTR_SYNTH), 4102 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT, 4103 CNTR_SYNTH), 4104 [C_DC_MISC_FLG_CNT] = 4105 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT, 4106 CNTR_SYNTH), 4107 [C_DC_PRF_GOOD_LTP_CNT] = 4108 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH), 4109 [C_DC_PRF_ACCEPTED_LTP_CNT] = 4110 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT, 4111 CNTR_SYNTH), 4112 [C_DC_PRF_RX_FLIT_CNT] = 4113 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH), 4114 [C_DC_PRF_TX_FLIT_CNT] = 4115 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH), 4116 [C_DC_PRF_CLK_CNTR] = 4117 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH), 4118 [C_DC_PG_DBG_FLIT_CRDTS_CNT] = 4119 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH), 4120 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] = 4121 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT, 4122 CNTR_SYNTH), 4123 [C_DC_PG_STS_TX_SBE_CNT] = 4124 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH), 4125 [C_DC_PG_STS_TX_MBE_CNT] = 4126 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT, 4127 CNTR_SYNTH), 4128 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL, 4129 access_sw_cpu_intr), 4130 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL, 4131 access_sw_cpu_rcv_limit), 4132 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL, 4133 access_sw_vtx_wait), 4134 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL, 4135 access_sw_pio_wait), 4136 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL, 4137 access_sw_pio_drain), 4138 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL, 4139 access_sw_kmem_wait), 4140 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL, 4141 access_sw_send_schedule), 4142 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn", 4143 SEND_DMA_DESC_FETCHED_CNT, 0, 4144 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4145 dev_access_u32_csr), 4146 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0, 4147 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4148 access_sde_int_cnt), 4149 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0, 4150 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4151 access_sde_err_cnt), 4152 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0, 4153 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4154 access_sde_idle_int_cnt), 4155 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0, 4156 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4157 access_sde_progress_int_cnt), 4158 /* MISC_ERR_STATUS */ 4159 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0, 4160 CNTR_NORMAL, 4161 access_misc_pll_lock_fail_err_cnt), 4162 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0, 4163 CNTR_NORMAL, 4164 access_misc_mbist_fail_err_cnt), 4165 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0, 4166 CNTR_NORMAL, 4167 access_misc_invalid_eep_cmd_err_cnt), 4168 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0, 4169 CNTR_NORMAL, 4170 access_misc_efuse_done_parity_err_cnt), 4171 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0, 4172 CNTR_NORMAL, 4173 access_misc_efuse_write_err_cnt), 4174 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0, 4175 0, CNTR_NORMAL, 4176 access_misc_efuse_read_bad_addr_err_cnt), 4177 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0, 4178 CNTR_NORMAL, 4179 access_misc_efuse_csr_parity_err_cnt), 4180 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0, 4181 CNTR_NORMAL, 4182 access_misc_fw_auth_failed_err_cnt), 4183 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0, 4184 CNTR_NORMAL, 4185 access_misc_key_mismatch_err_cnt), 4186 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0, 4187 CNTR_NORMAL, 4188 access_misc_sbus_write_failed_err_cnt), 4189 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0, 4190 CNTR_NORMAL, 4191 access_misc_csr_write_bad_addr_err_cnt), 4192 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0, 4193 CNTR_NORMAL, 4194 access_misc_csr_read_bad_addr_err_cnt), 4195 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0, 4196 CNTR_NORMAL, 4197 access_misc_csr_parity_err_cnt), 4198 /* CceErrStatus */ 4199 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0, 4200 CNTR_NORMAL, 4201 access_sw_cce_err_status_aggregated_cnt), 4202 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0, 4203 CNTR_NORMAL, 4204 access_cce_msix_csr_parity_err_cnt), 4205 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0, 4206 CNTR_NORMAL, 4207 access_cce_int_map_unc_err_cnt), 4208 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0, 4209 CNTR_NORMAL, 4210 access_cce_int_map_cor_err_cnt), 4211 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0, 4212 CNTR_NORMAL, 4213 access_cce_msix_table_unc_err_cnt), 4214 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0, 4215 CNTR_NORMAL, 4216 access_cce_msix_table_cor_err_cnt), 4217 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0, 4218 0, CNTR_NORMAL, 4219 access_cce_rxdma_conv_fifo_parity_err_cnt), 4220 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0, 4221 0, CNTR_NORMAL, 4222 access_cce_rcpl_async_fifo_parity_err_cnt), 4223 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0, 4224 CNTR_NORMAL, 4225 access_cce_seg_write_bad_addr_err_cnt), 4226 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0, 4227 CNTR_NORMAL, 4228 access_cce_seg_read_bad_addr_err_cnt), 4229 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0, 4230 CNTR_NORMAL, 4231 access_la_triggered_cnt), 4232 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0, 4233 CNTR_NORMAL, 4234 access_cce_trgt_cpl_timeout_err_cnt), 4235 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0, 4236 CNTR_NORMAL, 4237 access_pcic_receive_parity_err_cnt), 4238 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0, 4239 CNTR_NORMAL, 4240 access_pcic_transmit_back_parity_err_cnt), 4241 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0, 4242 0, CNTR_NORMAL, 4243 access_pcic_transmit_front_parity_err_cnt), 4244 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0, 4245 CNTR_NORMAL, 4246 access_pcic_cpl_dat_q_unc_err_cnt), 4247 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0, 4248 CNTR_NORMAL, 4249 access_pcic_cpl_hd_q_unc_err_cnt), 4250 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0, 4251 CNTR_NORMAL, 4252 access_pcic_post_dat_q_unc_err_cnt), 4253 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0, 4254 CNTR_NORMAL, 4255 access_pcic_post_hd_q_unc_err_cnt), 4256 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0, 4257 CNTR_NORMAL, 4258 access_pcic_retry_sot_mem_unc_err_cnt), 4259 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0, 4260 CNTR_NORMAL, 4261 access_pcic_retry_mem_unc_err), 4262 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0, 4263 CNTR_NORMAL, 4264 access_pcic_n_post_dat_q_parity_err_cnt), 4265 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0, 4266 CNTR_NORMAL, 4267 access_pcic_n_post_h_q_parity_err_cnt), 4268 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0, 4269 CNTR_NORMAL, 4270 access_pcic_cpl_dat_q_cor_err_cnt), 4271 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0, 4272 CNTR_NORMAL, 4273 access_pcic_cpl_hd_q_cor_err_cnt), 4274 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0, 4275 CNTR_NORMAL, 4276 access_pcic_post_dat_q_cor_err_cnt), 4277 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0, 4278 CNTR_NORMAL, 4279 access_pcic_post_hd_q_cor_err_cnt), 4280 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0, 4281 CNTR_NORMAL, 4282 access_pcic_retry_sot_mem_cor_err_cnt), 4283 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0, 4284 CNTR_NORMAL, 4285 access_pcic_retry_mem_cor_err_cnt), 4286 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM( 4287 "CceCli1AsyncFifoDbgParityError", 0, 0, 4288 CNTR_NORMAL, 4289 access_cce_cli1_async_fifo_dbg_parity_err_cnt), 4290 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM( 4291 "CceCli1AsyncFifoRxdmaParityError", 0, 0, 4292 CNTR_NORMAL, 4293 access_cce_cli1_async_fifo_rxdma_parity_err_cnt 4294 ), 4295 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM( 4296 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0, 4297 CNTR_NORMAL, 4298 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt), 4299 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM( 4300 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0, 4301 CNTR_NORMAL, 4302 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt), 4303 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0, 4304 0, CNTR_NORMAL, 4305 access_cce_cli2_async_fifo_parity_err_cnt), 4306 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0, 4307 CNTR_NORMAL, 4308 access_cce_csr_cfg_bus_parity_err_cnt), 4309 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0, 4310 0, CNTR_NORMAL, 4311 access_cce_cli0_async_fifo_parity_err_cnt), 4312 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0, 4313 CNTR_NORMAL, 4314 access_cce_rspd_data_parity_err_cnt), 4315 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0, 4316 CNTR_NORMAL, 4317 access_cce_trgt_access_err_cnt), 4318 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0, 4319 0, CNTR_NORMAL, 4320 access_cce_trgt_async_fifo_parity_err_cnt), 4321 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0, 4322 CNTR_NORMAL, 4323 access_cce_csr_write_bad_addr_err_cnt), 4324 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0, 4325 CNTR_NORMAL, 4326 access_cce_csr_read_bad_addr_err_cnt), 4327 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0, 4328 CNTR_NORMAL, 4329 access_ccs_csr_parity_err_cnt), 4330 4331 /* RcvErrStatus */ 4332 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0, 4333 CNTR_NORMAL, 4334 access_rx_csr_parity_err_cnt), 4335 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0, 4336 CNTR_NORMAL, 4337 access_rx_csr_write_bad_addr_err_cnt), 4338 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0, 4339 CNTR_NORMAL, 4340 access_rx_csr_read_bad_addr_err_cnt), 4341 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0, 4342 CNTR_NORMAL, 4343 access_rx_dma_csr_unc_err_cnt), 4344 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0, 4345 CNTR_NORMAL, 4346 access_rx_dma_dq_fsm_encoding_err_cnt), 4347 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0, 4348 CNTR_NORMAL, 4349 access_rx_dma_eq_fsm_encoding_err_cnt), 4350 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0, 4351 CNTR_NORMAL, 4352 access_rx_dma_csr_parity_err_cnt), 4353 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0, 4354 CNTR_NORMAL, 4355 access_rx_rbuf_data_cor_err_cnt), 4356 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0, 4357 CNTR_NORMAL, 4358 access_rx_rbuf_data_unc_err_cnt), 4359 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0, 4360 CNTR_NORMAL, 4361 access_rx_dma_data_fifo_rd_cor_err_cnt), 4362 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0, 4363 CNTR_NORMAL, 4364 access_rx_dma_data_fifo_rd_unc_err_cnt), 4365 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0, 4366 CNTR_NORMAL, 4367 access_rx_dma_hdr_fifo_rd_cor_err_cnt), 4368 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0, 4369 CNTR_NORMAL, 4370 access_rx_dma_hdr_fifo_rd_unc_err_cnt), 4371 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0, 4372 CNTR_NORMAL, 4373 access_rx_rbuf_desc_part2_cor_err_cnt), 4374 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0, 4375 CNTR_NORMAL, 4376 access_rx_rbuf_desc_part2_unc_err_cnt), 4377 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0, 4378 CNTR_NORMAL, 4379 access_rx_rbuf_desc_part1_cor_err_cnt), 4380 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0, 4381 CNTR_NORMAL, 4382 access_rx_rbuf_desc_part1_unc_err_cnt), 4383 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0, 4384 CNTR_NORMAL, 4385 access_rx_hq_intr_fsm_err_cnt), 4386 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0, 4387 CNTR_NORMAL, 4388 access_rx_hq_intr_csr_parity_err_cnt), 4389 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0, 4390 CNTR_NORMAL, 4391 access_rx_lookup_csr_parity_err_cnt), 4392 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0, 4393 CNTR_NORMAL, 4394 access_rx_lookup_rcv_array_cor_err_cnt), 4395 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0, 4396 CNTR_NORMAL, 4397 access_rx_lookup_rcv_array_unc_err_cnt), 4398 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0, 4399 0, CNTR_NORMAL, 4400 access_rx_lookup_des_part2_parity_err_cnt), 4401 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0, 4402 0, CNTR_NORMAL, 4403 access_rx_lookup_des_part1_unc_cor_err_cnt), 4404 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0, 4405 CNTR_NORMAL, 4406 access_rx_lookup_des_part1_unc_err_cnt), 4407 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0, 4408 CNTR_NORMAL, 4409 access_rx_rbuf_next_free_buf_cor_err_cnt), 4410 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0, 4411 CNTR_NORMAL, 4412 access_rx_rbuf_next_free_buf_unc_err_cnt), 4413 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM( 4414 "RxRbufFlInitWrAddrParityErr", 0, 0, 4415 CNTR_NORMAL, 4416 access_rbuf_fl_init_wr_addr_parity_err_cnt), 4417 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0, 4418 0, CNTR_NORMAL, 4419 access_rx_rbuf_fl_initdone_parity_err_cnt), 4420 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0, 4421 0, CNTR_NORMAL, 4422 access_rx_rbuf_fl_write_addr_parity_err_cnt), 4423 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0, 4424 CNTR_NORMAL, 4425 access_rx_rbuf_fl_rd_addr_parity_err_cnt), 4426 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0, 4427 CNTR_NORMAL, 4428 access_rx_rbuf_empty_err_cnt), 4429 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0, 4430 CNTR_NORMAL, 4431 access_rx_rbuf_full_err_cnt), 4432 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0, 4433 CNTR_NORMAL, 4434 access_rbuf_bad_lookup_err_cnt), 4435 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0, 4436 CNTR_NORMAL, 4437 access_rbuf_ctx_id_parity_err_cnt), 4438 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0, 4439 CNTR_NORMAL, 4440 access_rbuf_csr_qeopdw_parity_err_cnt), 4441 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM( 4442 "RxRbufCsrQNumOfPktParityErr", 0, 0, 4443 CNTR_NORMAL, 4444 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt), 4445 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM( 4446 "RxRbufCsrQTlPtrParityErr", 0, 0, 4447 CNTR_NORMAL, 4448 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt), 4449 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0, 4450 0, CNTR_NORMAL, 4451 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt), 4452 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0, 4453 0, CNTR_NORMAL, 4454 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt), 4455 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr", 4456 0, 0, CNTR_NORMAL, 4457 access_rx_rbuf_csr_q_next_buf_parity_err_cnt), 4458 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0, 4459 0, CNTR_NORMAL, 4460 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt), 4461 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM( 4462 "RxRbufCsrQHeadBufNumParityErr", 0, 0, 4463 CNTR_NORMAL, 4464 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt), 4465 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0, 4466 0, CNTR_NORMAL, 4467 access_rx_rbuf_block_list_read_cor_err_cnt), 4468 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0, 4469 0, CNTR_NORMAL, 4470 access_rx_rbuf_block_list_read_unc_err_cnt), 4471 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0, 4472 CNTR_NORMAL, 4473 access_rx_rbuf_lookup_des_cor_err_cnt), 4474 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0, 4475 CNTR_NORMAL, 4476 access_rx_rbuf_lookup_des_unc_err_cnt), 4477 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM( 4478 "RxRbufLookupDesRegUncCorErr", 0, 0, 4479 CNTR_NORMAL, 4480 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt), 4481 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0, 4482 CNTR_NORMAL, 4483 access_rx_rbuf_lookup_des_reg_unc_err_cnt), 4484 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0, 4485 CNTR_NORMAL, 4486 access_rx_rbuf_free_list_cor_err_cnt), 4487 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0, 4488 CNTR_NORMAL, 4489 access_rx_rbuf_free_list_unc_err_cnt), 4490 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0, 4491 CNTR_NORMAL, 4492 access_rx_rcv_fsm_encoding_err_cnt), 4493 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0, 4494 CNTR_NORMAL, 4495 access_rx_dma_flag_cor_err_cnt), 4496 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0, 4497 CNTR_NORMAL, 4498 access_rx_dma_flag_unc_err_cnt), 4499 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0, 4500 CNTR_NORMAL, 4501 access_rx_dc_sop_eop_parity_err_cnt), 4502 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0, 4503 CNTR_NORMAL, 4504 access_rx_rcv_csr_parity_err_cnt), 4505 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0, 4506 CNTR_NORMAL, 4507 access_rx_rcv_qp_map_table_cor_err_cnt), 4508 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0, 4509 CNTR_NORMAL, 4510 access_rx_rcv_qp_map_table_unc_err_cnt), 4511 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0, 4512 CNTR_NORMAL, 4513 access_rx_rcv_data_cor_err_cnt), 4514 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0, 4515 CNTR_NORMAL, 4516 access_rx_rcv_data_unc_err_cnt), 4517 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0, 4518 CNTR_NORMAL, 4519 access_rx_rcv_hdr_cor_err_cnt), 4520 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0, 4521 CNTR_NORMAL, 4522 access_rx_rcv_hdr_unc_err_cnt), 4523 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0, 4524 CNTR_NORMAL, 4525 access_rx_dc_intf_parity_err_cnt), 4526 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0, 4527 CNTR_NORMAL, 4528 access_rx_dma_csr_cor_err_cnt), 4529 /* SendPioErrStatus */ 4530 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0, 4531 CNTR_NORMAL, 4532 access_pio_pec_sop_head_parity_err_cnt), 4533 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0, 4534 CNTR_NORMAL, 4535 access_pio_pcc_sop_head_parity_err_cnt), 4536 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr", 4537 0, 0, CNTR_NORMAL, 4538 access_pio_last_returned_cnt_parity_err_cnt), 4539 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0, 4540 0, CNTR_NORMAL, 4541 access_pio_current_free_cnt_parity_err_cnt), 4542 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0, 4543 CNTR_NORMAL, 4544 access_pio_reserved_31_err_cnt), 4545 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0, 4546 CNTR_NORMAL, 4547 access_pio_reserved_30_err_cnt), 4548 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0, 4549 CNTR_NORMAL, 4550 access_pio_ppmc_sop_len_err_cnt), 4551 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0, 4552 CNTR_NORMAL, 4553 access_pio_ppmc_bqc_mem_parity_err_cnt), 4554 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0, 4555 CNTR_NORMAL, 4556 access_pio_vl_fifo_parity_err_cnt), 4557 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0, 4558 CNTR_NORMAL, 4559 access_pio_vlf_sop_parity_err_cnt), 4560 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0, 4561 CNTR_NORMAL, 4562 access_pio_vlf_v1_len_parity_err_cnt), 4563 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0, 4564 CNTR_NORMAL, 4565 access_pio_block_qw_count_parity_err_cnt), 4566 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0, 4567 CNTR_NORMAL, 4568 access_pio_write_qw_valid_parity_err_cnt), 4569 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0, 4570 CNTR_NORMAL, 4571 access_pio_state_machine_err_cnt), 4572 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0, 4573 CNTR_NORMAL, 4574 access_pio_write_data_parity_err_cnt), 4575 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0, 4576 CNTR_NORMAL, 4577 access_pio_host_addr_mem_cor_err_cnt), 4578 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0, 4579 CNTR_NORMAL, 4580 access_pio_host_addr_mem_unc_err_cnt), 4581 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0, 4582 CNTR_NORMAL, 4583 access_pio_pkt_evict_sm_or_arb_sm_err_cnt), 4584 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0, 4585 CNTR_NORMAL, 4586 access_pio_init_sm_in_err_cnt), 4587 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0, 4588 CNTR_NORMAL, 4589 access_pio_ppmc_pbl_fifo_err_cnt), 4590 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0, 4591 0, CNTR_NORMAL, 4592 access_pio_credit_ret_fifo_parity_err_cnt), 4593 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0, 4594 CNTR_NORMAL, 4595 access_pio_v1_len_mem_bank1_cor_err_cnt), 4596 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0, 4597 CNTR_NORMAL, 4598 access_pio_v1_len_mem_bank0_cor_err_cnt), 4599 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0, 4600 CNTR_NORMAL, 4601 access_pio_v1_len_mem_bank1_unc_err_cnt), 4602 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0, 4603 CNTR_NORMAL, 4604 access_pio_v1_len_mem_bank0_unc_err_cnt), 4605 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0, 4606 CNTR_NORMAL, 4607 access_pio_sm_pkt_reset_parity_err_cnt), 4608 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0, 4609 CNTR_NORMAL, 4610 access_pio_pkt_evict_fifo_parity_err_cnt), 4611 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM( 4612 "PioSbrdctrlCrrelFifoParityErr", 0, 0, 4613 CNTR_NORMAL, 4614 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt), 4615 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0, 4616 CNTR_NORMAL, 4617 access_pio_sbrdctl_crrel_parity_err_cnt), 4618 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0, 4619 CNTR_NORMAL, 4620 access_pio_pec_fifo_parity_err_cnt), 4621 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0, 4622 CNTR_NORMAL, 4623 access_pio_pcc_fifo_parity_err_cnt), 4624 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0, 4625 CNTR_NORMAL, 4626 access_pio_sb_mem_fifo1_err_cnt), 4627 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0, 4628 CNTR_NORMAL, 4629 access_pio_sb_mem_fifo0_err_cnt), 4630 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0, 4631 CNTR_NORMAL, 4632 access_pio_csr_parity_err_cnt), 4633 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0, 4634 CNTR_NORMAL, 4635 access_pio_write_addr_parity_err_cnt), 4636 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0, 4637 CNTR_NORMAL, 4638 access_pio_write_bad_ctxt_err_cnt), 4639 /* SendDmaErrStatus */ 4640 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0, 4641 0, CNTR_NORMAL, 4642 access_sdma_pcie_req_tracking_cor_err_cnt), 4643 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0, 4644 0, CNTR_NORMAL, 4645 access_sdma_pcie_req_tracking_unc_err_cnt), 4646 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0, 4647 CNTR_NORMAL, 4648 access_sdma_csr_parity_err_cnt), 4649 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0, 4650 CNTR_NORMAL, 4651 access_sdma_rpy_tag_err_cnt), 4652 /* SendEgressErrStatus */ 4653 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0, 4654 CNTR_NORMAL, 4655 access_tx_read_pio_memory_csr_unc_err_cnt), 4656 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0, 4657 0, CNTR_NORMAL, 4658 access_tx_read_sdma_memory_csr_err_cnt), 4659 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0, 4660 CNTR_NORMAL, 4661 access_tx_egress_fifo_cor_err_cnt), 4662 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0, 4663 CNTR_NORMAL, 4664 access_tx_read_pio_memory_cor_err_cnt), 4665 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0, 4666 CNTR_NORMAL, 4667 access_tx_read_sdma_memory_cor_err_cnt), 4668 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0, 4669 CNTR_NORMAL, 4670 access_tx_sb_hdr_cor_err_cnt), 4671 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0, 4672 CNTR_NORMAL, 4673 access_tx_credit_overrun_err_cnt), 4674 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0, 4675 CNTR_NORMAL, 4676 access_tx_launch_fifo8_cor_err_cnt), 4677 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0, 4678 CNTR_NORMAL, 4679 access_tx_launch_fifo7_cor_err_cnt), 4680 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0, 4681 CNTR_NORMAL, 4682 access_tx_launch_fifo6_cor_err_cnt), 4683 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0, 4684 CNTR_NORMAL, 4685 access_tx_launch_fifo5_cor_err_cnt), 4686 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0, 4687 CNTR_NORMAL, 4688 access_tx_launch_fifo4_cor_err_cnt), 4689 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0, 4690 CNTR_NORMAL, 4691 access_tx_launch_fifo3_cor_err_cnt), 4692 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0, 4693 CNTR_NORMAL, 4694 access_tx_launch_fifo2_cor_err_cnt), 4695 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0, 4696 CNTR_NORMAL, 4697 access_tx_launch_fifo1_cor_err_cnt), 4698 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0, 4699 CNTR_NORMAL, 4700 access_tx_launch_fifo0_cor_err_cnt), 4701 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0, 4702 CNTR_NORMAL, 4703 access_tx_credit_return_vl_err_cnt), 4704 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0, 4705 CNTR_NORMAL, 4706 access_tx_hcrc_insertion_err_cnt), 4707 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0, 4708 CNTR_NORMAL, 4709 access_tx_egress_fifo_unc_err_cnt), 4710 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0, 4711 CNTR_NORMAL, 4712 access_tx_read_pio_memory_unc_err_cnt), 4713 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0, 4714 CNTR_NORMAL, 4715 access_tx_read_sdma_memory_unc_err_cnt), 4716 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0, 4717 CNTR_NORMAL, 4718 access_tx_sb_hdr_unc_err_cnt), 4719 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0, 4720 CNTR_NORMAL, 4721 access_tx_credit_return_partiy_err_cnt), 4722 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr", 4723 0, 0, CNTR_NORMAL, 4724 access_tx_launch_fifo8_unc_or_parity_err_cnt), 4725 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr", 4726 0, 0, CNTR_NORMAL, 4727 access_tx_launch_fifo7_unc_or_parity_err_cnt), 4728 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr", 4729 0, 0, CNTR_NORMAL, 4730 access_tx_launch_fifo6_unc_or_parity_err_cnt), 4731 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr", 4732 0, 0, CNTR_NORMAL, 4733 access_tx_launch_fifo5_unc_or_parity_err_cnt), 4734 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr", 4735 0, 0, CNTR_NORMAL, 4736 access_tx_launch_fifo4_unc_or_parity_err_cnt), 4737 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr", 4738 0, 0, CNTR_NORMAL, 4739 access_tx_launch_fifo3_unc_or_parity_err_cnt), 4740 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr", 4741 0, 0, CNTR_NORMAL, 4742 access_tx_launch_fifo2_unc_or_parity_err_cnt), 4743 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr", 4744 0, 0, CNTR_NORMAL, 4745 access_tx_launch_fifo1_unc_or_parity_err_cnt), 4746 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr", 4747 0, 0, CNTR_NORMAL, 4748 access_tx_launch_fifo0_unc_or_parity_err_cnt), 4749 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr", 4750 0, 0, CNTR_NORMAL, 4751 access_tx_sdma15_disallowed_packet_err_cnt), 4752 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr", 4753 0, 0, CNTR_NORMAL, 4754 access_tx_sdma14_disallowed_packet_err_cnt), 4755 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr", 4756 0, 0, CNTR_NORMAL, 4757 access_tx_sdma13_disallowed_packet_err_cnt), 4758 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr", 4759 0, 0, CNTR_NORMAL, 4760 access_tx_sdma12_disallowed_packet_err_cnt), 4761 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr", 4762 0, 0, CNTR_NORMAL, 4763 access_tx_sdma11_disallowed_packet_err_cnt), 4764 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr", 4765 0, 0, CNTR_NORMAL, 4766 access_tx_sdma10_disallowed_packet_err_cnt), 4767 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr", 4768 0, 0, CNTR_NORMAL, 4769 access_tx_sdma9_disallowed_packet_err_cnt), 4770 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr", 4771 0, 0, CNTR_NORMAL, 4772 access_tx_sdma8_disallowed_packet_err_cnt), 4773 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr", 4774 0, 0, CNTR_NORMAL, 4775 access_tx_sdma7_disallowed_packet_err_cnt), 4776 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr", 4777 0, 0, CNTR_NORMAL, 4778 access_tx_sdma6_disallowed_packet_err_cnt), 4779 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr", 4780 0, 0, CNTR_NORMAL, 4781 access_tx_sdma5_disallowed_packet_err_cnt), 4782 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr", 4783 0, 0, CNTR_NORMAL, 4784 access_tx_sdma4_disallowed_packet_err_cnt), 4785 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr", 4786 0, 0, CNTR_NORMAL, 4787 access_tx_sdma3_disallowed_packet_err_cnt), 4788 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr", 4789 0, 0, CNTR_NORMAL, 4790 access_tx_sdma2_disallowed_packet_err_cnt), 4791 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr", 4792 0, 0, CNTR_NORMAL, 4793 access_tx_sdma1_disallowed_packet_err_cnt), 4794 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr", 4795 0, 0, CNTR_NORMAL, 4796 access_tx_sdma0_disallowed_packet_err_cnt), 4797 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0, 4798 CNTR_NORMAL, 4799 access_tx_config_parity_err_cnt), 4800 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0, 4801 CNTR_NORMAL, 4802 access_tx_sbrd_ctl_csr_parity_err_cnt), 4803 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0, 4804 CNTR_NORMAL, 4805 access_tx_launch_csr_parity_err_cnt), 4806 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0, 4807 CNTR_NORMAL, 4808 access_tx_illegal_vl_err_cnt), 4809 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM( 4810 "TxSbrdCtlStateMachineParityErr", 0, 0, 4811 CNTR_NORMAL, 4812 access_tx_sbrd_ctl_state_machine_parity_err_cnt), 4813 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0, 4814 CNTR_NORMAL, 4815 access_egress_reserved_10_err_cnt), 4816 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0, 4817 CNTR_NORMAL, 4818 access_egress_reserved_9_err_cnt), 4819 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr", 4820 0, 0, CNTR_NORMAL, 4821 access_tx_sdma_launch_intf_parity_err_cnt), 4822 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0, 4823 CNTR_NORMAL, 4824 access_tx_pio_launch_intf_parity_err_cnt), 4825 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0, 4826 CNTR_NORMAL, 4827 access_egress_reserved_6_err_cnt), 4828 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0, 4829 CNTR_NORMAL, 4830 access_tx_incorrect_link_state_err_cnt), 4831 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0, 4832 CNTR_NORMAL, 4833 access_tx_linkdown_err_cnt), 4834 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM( 4835 "EgressFifoUnderrunOrParityErr", 0, 0, 4836 CNTR_NORMAL, 4837 access_tx_egress_fifi_underrun_or_parity_err_cnt), 4838 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0, 4839 CNTR_NORMAL, 4840 access_egress_reserved_2_err_cnt), 4841 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0, 4842 CNTR_NORMAL, 4843 access_tx_pkt_integrity_mem_unc_err_cnt), 4844 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0, 4845 CNTR_NORMAL, 4846 access_tx_pkt_integrity_mem_cor_err_cnt), 4847 /* SendErrStatus */ 4848 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0, 4849 CNTR_NORMAL, 4850 access_send_csr_write_bad_addr_err_cnt), 4851 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0, 4852 CNTR_NORMAL, 4853 access_send_csr_read_bad_addr_err_cnt), 4854 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0, 4855 CNTR_NORMAL, 4856 access_send_csr_parity_cnt), 4857 /* SendCtxtErrStatus */ 4858 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0, 4859 CNTR_NORMAL, 4860 access_pio_write_out_of_bounds_err_cnt), 4861 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0, 4862 CNTR_NORMAL, 4863 access_pio_write_overflow_err_cnt), 4864 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr", 4865 0, 0, CNTR_NORMAL, 4866 access_pio_write_crosses_boundary_err_cnt), 4867 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0, 4868 CNTR_NORMAL, 4869 access_pio_disallowed_packet_err_cnt), 4870 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0, 4871 CNTR_NORMAL, 4872 access_pio_inconsistent_sop_err_cnt), 4873 /* SendDmaEngErrStatus */ 4874 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr", 4875 0, 0, CNTR_NORMAL, 4876 access_sdma_header_request_fifo_cor_err_cnt), 4877 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0, 4878 CNTR_NORMAL, 4879 access_sdma_header_storage_cor_err_cnt), 4880 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0, 4881 CNTR_NORMAL, 4882 access_sdma_packet_tracking_cor_err_cnt), 4883 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0, 4884 CNTR_NORMAL, 4885 access_sdma_assembly_cor_err_cnt), 4886 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0, 4887 CNTR_NORMAL, 4888 access_sdma_desc_table_cor_err_cnt), 4889 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr", 4890 0, 0, CNTR_NORMAL, 4891 access_sdma_header_request_fifo_unc_err_cnt), 4892 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0, 4893 CNTR_NORMAL, 4894 access_sdma_header_storage_unc_err_cnt), 4895 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0, 4896 CNTR_NORMAL, 4897 access_sdma_packet_tracking_unc_err_cnt), 4898 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0, 4899 CNTR_NORMAL, 4900 access_sdma_assembly_unc_err_cnt), 4901 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0, 4902 CNTR_NORMAL, 4903 access_sdma_desc_table_unc_err_cnt), 4904 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0, 4905 CNTR_NORMAL, 4906 access_sdma_timeout_err_cnt), 4907 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0, 4908 CNTR_NORMAL, 4909 access_sdma_header_length_err_cnt), 4910 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0, 4911 CNTR_NORMAL, 4912 access_sdma_header_address_err_cnt), 4913 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0, 4914 CNTR_NORMAL, 4915 access_sdma_header_select_err_cnt), 4916 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0, 4917 CNTR_NORMAL, 4918 access_sdma_reserved_9_err_cnt), 4919 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0, 4920 CNTR_NORMAL, 4921 access_sdma_packet_desc_overflow_err_cnt), 4922 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0, 4923 CNTR_NORMAL, 4924 access_sdma_length_mismatch_err_cnt), 4925 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0, 4926 CNTR_NORMAL, 4927 access_sdma_halt_err_cnt), 4928 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0, 4929 CNTR_NORMAL, 4930 access_sdma_mem_read_err_cnt), 4931 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0, 4932 CNTR_NORMAL, 4933 access_sdma_first_desc_err_cnt), 4934 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0, 4935 CNTR_NORMAL, 4936 access_sdma_tail_out_of_bounds_err_cnt), 4937 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0, 4938 CNTR_NORMAL, 4939 access_sdma_too_long_err_cnt), 4940 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0, 4941 CNTR_NORMAL, 4942 access_sdma_gen_mismatch_err_cnt), 4943 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0, 4944 CNTR_NORMAL, 4945 access_sdma_wrong_dw_err_cnt), 4946 }; 4947 4948 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = { 4949 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT, 4950 CNTR_NORMAL), 4951 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT, 4952 CNTR_NORMAL), 4953 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT, 4954 CNTR_NORMAL), 4955 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT, 4956 CNTR_NORMAL), 4957 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT, 4958 CNTR_NORMAL), 4959 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT, 4960 CNTR_NORMAL), 4961 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT, 4962 CNTR_NORMAL), 4963 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL), 4964 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL), 4965 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH), 4966 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT, 4967 CNTR_SYNTH | CNTR_VL), 4968 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT, 4969 CNTR_SYNTH | CNTR_VL), 4970 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT, 4971 CNTR_SYNTH | CNTR_VL), 4972 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL), 4973 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL), 4974 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT, 4975 access_sw_link_dn_cnt), 4976 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT, 4977 access_sw_link_up_cnt), 4978 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL, 4979 access_sw_unknown_frame_cnt), 4980 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT, 4981 access_sw_xmit_discards), 4982 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0, 4983 CNTR_SYNTH | CNTR_32BIT | CNTR_VL, 4984 access_sw_xmit_discards), 4985 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH, 4986 access_xmit_constraint_errs), 4987 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH, 4988 access_rcv_constraint_errs), 4989 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts), 4990 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends), 4991 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks), 4992 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks), 4993 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts), 4994 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops), 4995 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait), 4996 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak), 4997 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq), 4998 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq), 4999 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned), 5000 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks), 5001 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL, 5002 access_sw_cpu_rc_acks), 5003 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL, 5004 access_sw_cpu_rc_qacks), 5005 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL, 5006 access_sw_cpu_rc_delayed_comp), 5007 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1), 5008 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3), 5009 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5), 5010 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7), 5011 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9), 5012 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11), 5013 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13), 5014 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15), 5015 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17), 5016 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19), 5017 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21), 5018 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23), 5019 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25), 5020 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27), 5021 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29), 5022 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31), 5023 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33), 5024 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35), 5025 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37), 5026 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39), 5027 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41), 5028 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43), 5029 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45), 5030 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47), 5031 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49), 5032 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51), 5033 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53), 5034 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55), 5035 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57), 5036 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59), 5037 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61), 5038 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63), 5039 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65), 5040 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67), 5041 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69), 5042 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71), 5043 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73), 5044 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75), 5045 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77), 5046 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79), 5047 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81), 5048 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83), 5049 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85), 5050 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87), 5051 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89), 5052 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91), 5053 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93), 5054 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95), 5055 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97), 5056 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99), 5057 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101), 5058 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103), 5059 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105), 5060 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107), 5061 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109), 5062 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111), 5063 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113), 5064 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115), 5065 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117), 5066 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119), 5067 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121), 5068 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123), 5069 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125), 5070 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127), 5071 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129), 5072 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131), 5073 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133), 5074 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135), 5075 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137), 5076 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139), 5077 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141), 5078 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143), 5079 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145), 5080 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147), 5081 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149), 5082 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151), 5083 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153), 5084 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155), 5085 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157), 5086 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159), 5087 }; 5088 5089 /* ======================================================================== */ 5090 5091 /* return true if this is chip revision revision a */ 5092 int is_ax(struct hfi1_devdata *dd) 5093 { 5094 u8 chip_rev_minor = 5095 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5096 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5097 return (chip_rev_minor & 0xf0) == 0; 5098 } 5099 5100 /* return true if this is chip revision revision b */ 5101 int is_bx(struct hfi1_devdata *dd) 5102 { 5103 u8 chip_rev_minor = 5104 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5105 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5106 return (chip_rev_minor & 0xF0) == 0x10; 5107 } 5108 5109 /* 5110 * Append string s to buffer buf. Arguments curp and len are the current 5111 * position and remaining length, respectively. 5112 * 5113 * return 0 on success, 1 on out of room 5114 */ 5115 static int append_str(char *buf, char **curp, int *lenp, const char *s) 5116 { 5117 char *p = *curp; 5118 int len = *lenp; 5119 int result = 0; /* success */ 5120 char c; 5121 5122 /* add a comma, if first in the buffer */ 5123 if (p != buf) { 5124 if (len == 0) { 5125 result = 1; /* out of room */ 5126 goto done; 5127 } 5128 *p++ = ','; 5129 len--; 5130 } 5131 5132 /* copy the string */ 5133 while ((c = *s++) != 0) { 5134 if (len == 0) { 5135 result = 1; /* out of room */ 5136 goto done; 5137 } 5138 *p++ = c; 5139 len--; 5140 } 5141 5142 done: 5143 /* write return values */ 5144 *curp = p; 5145 *lenp = len; 5146 5147 return result; 5148 } 5149 5150 /* 5151 * Using the given flag table, print a comma separated string into 5152 * the buffer. End in '*' if the buffer is too short. 5153 */ 5154 static char *flag_string(char *buf, int buf_len, u64 flags, 5155 struct flag_table *table, int table_size) 5156 { 5157 char extra[32]; 5158 char *p = buf; 5159 int len = buf_len; 5160 int no_room = 0; 5161 int i; 5162 5163 /* make sure there is at least 2 so we can form "*" */ 5164 if (len < 2) 5165 return ""; 5166 5167 len--; /* leave room for a nul */ 5168 for (i = 0; i < table_size; i++) { 5169 if (flags & table[i].flag) { 5170 no_room = append_str(buf, &p, &len, table[i].str); 5171 if (no_room) 5172 break; 5173 flags &= ~table[i].flag; 5174 } 5175 } 5176 5177 /* any undocumented bits left? */ 5178 if (!no_room && flags) { 5179 snprintf(extra, sizeof(extra), "bits 0x%llx", flags); 5180 no_room = append_str(buf, &p, &len, extra); 5181 } 5182 5183 /* add * if ran out of room */ 5184 if (no_room) { 5185 /* may need to back up to add space for a '*' */ 5186 if (len == 0) 5187 --p; 5188 *p++ = '*'; 5189 } 5190 5191 /* add final nul - space already allocated above */ 5192 *p = 0; 5193 return buf; 5194 } 5195 5196 /* first 8 CCE error interrupt source names */ 5197 static const char * const cce_misc_names[] = { 5198 "CceErrInt", /* 0 */ 5199 "RxeErrInt", /* 1 */ 5200 "MiscErrInt", /* 2 */ 5201 "Reserved3", /* 3 */ 5202 "PioErrInt", /* 4 */ 5203 "SDmaErrInt", /* 5 */ 5204 "EgressErrInt", /* 6 */ 5205 "TxeErrInt" /* 7 */ 5206 }; 5207 5208 /* 5209 * Return the miscellaneous error interrupt name. 5210 */ 5211 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source) 5212 { 5213 if (source < ARRAY_SIZE(cce_misc_names)) 5214 strncpy(buf, cce_misc_names[source], bsize); 5215 else 5216 snprintf(buf, bsize, "Reserved%u", 5217 source + IS_GENERAL_ERR_START); 5218 5219 return buf; 5220 } 5221 5222 /* 5223 * Return the SDMA engine error interrupt name. 5224 */ 5225 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source) 5226 { 5227 snprintf(buf, bsize, "SDmaEngErrInt%u", source); 5228 return buf; 5229 } 5230 5231 /* 5232 * Return the send context error interrupt name. 5233 */ 5234 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source) 5235 { 5236 snprintf(buf, bsize, "SendCtxtErrInt%u", source); 5237 return buf; 5238 } 5239 5240 static const char * const various_names[] = { 5241 "PbcInt", 5242 "GpioAssertInt", 5243 "Qsfp1Int", 5244 "Qsfp2Int", 5245 "TCritInt" 5246 }; 5247 5248 /* 5249 * Return the various interrupt name. 5250 */ 5251 static char *is_various_name(char *buf, size_t bsize, unsigned int source) 5252 { 5253 if (source < ARRAY_SIZE(various_names)) 5254 strncpy(buf, various_names[source], bsize); 5255 else 5256 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START); 5257 return buf; 5258 } 5259 5260 /* 5261 * Return the DC interrupt name. 5262 */ 5263 static char *is_dc_name(char *buf, size_t bsize, unsigned int source) 5264 { 5265 static const char * const dc_int_names[] = { 5266 "common", 5267 "lcb", 5268 "8051", 5269 "lbm" /* local block merge */ 5270 }; 5271 5272 if (source < ARRAY_SIZE(dc_int_names)) 5273 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]); 5274 else 5275 snprintf(buf, bsize, "DCInt%u", source); 5276 return buf; 5277 } 5278 5279 static const char * const sdma_int_names[] = { 5280 "SDmaInt", 5281 "SdmaIdleInt", 5282 "SdmaProgressInt", 5283 }; 5284 5285 /* 5286 * Return the SDMA engine interrupt name. 5287 */ 5288 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source) 5289 { 5290 /* what interrupt */ 5291 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 5292 /* which engine */ 5293 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 5294 5295 if (likely(what < 3)) 5296 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which); 5297 else 5298 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source); 5299 return buf; 5300 } 5301 5302 /* 5303 * Return the receive available interrupt name. 5304 */ 5305 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source) 5306 { 5307 snprintf(buf, bsize, "RcvAvailInt%u", source); 5308 return buf; 5309 } 5310 5311 /* 5312 * Return the receive urgent interrupt name. 5313 */ 5314 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source) 5315 { 5316 snprintf(buf, bsize, "RcvUrgentInt%u", source); 5317 return buf; 5318 } 5319 5320 /* 5321 * Return the send credit interrupt name. 5322 */ 5323 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source) 5324 { 5325 snprintf(buf, bsize, "SendCreditInt%u", source); 5326 return buf; 5327 } 5328 5329 /* 5330 * Return the reserved interrupt name. 5331 */ 5332 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source) 5333 { 5334 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START); 5335 return buf; 5336 } 5337 5338 static char *cce_err_status_string(char *buf, int buf_len, u64 flags) 5339 { 5340 return flag_string(buf, buf_len, flags, 5341 cce_err_status_flags, 5342 ARRAY_SIZE(cce_err_status_flags)); 5343 } 5344 5345 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags) 5346 { 5347 return flag_string(buf, buf_len, flags, 5348 rxe_err_status_flags, 5349 ARRAY_SIZE(rxe_err_status_flags)); 5350 } 5351 5352 static char *misc_err_status_string(char *buf, int buf_len, u64 flags) 5353 { 5354 return flag_string(buf, buf_len, flags, misc_err_status_flags, 5355 ARRAY_SIZE(misc_err_status_flags)); 5356 } 5357 5358 static char *pio_err_status_string(char *buf, int buf_len, u64 flags) 5359 { 5360 return flag_string(buf, buf_len, flags, 5361 pio_err_status_flags, 5362 ARRAY_SIZE(pio_err_status_flags)); 5363 } 5364 5365 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags) 5366 { 5367 return flag_string(buf, buf_len, flags, 5368 sdma_err_status_flags, 5369 ARRAY_SIZE(sdma_err_status_flags)); 5370 } 5371 5372 static char *egress_err_status_string(char *buf, int buf_len, u64 flags) 5373 { 5374 return flag_string(buf, buf_len, flags, 5375 egress_err_status_flags, 5376 ARRAY_SIZE(egress_err_status_flags)); 5377 } 5378 5379 static char *egress_err_info_string(char *buf, int buf_len, u64 flags) 5380 { 5381 return flag_string(buf, buf_len, flags, 5382 egress_err_info_flags, 5383 ARRAY_SIZE(egress_err_info_flags)); 5384 } 5385 5386 static char *send_err_status_string(char *buf, int buf_len, u64 flags) 5387 { 5388 return flag_string(buf, buf_len, flags, 5389 send_err_status_flags, 5390 ARRAY_SIZE(send_err_status_flags)); 5391 } 5392 5393 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5394 { 5395 char buf[96]; 5396 int i = 0; 5397 5398 /* 5399 * For most these errors, there is nothing that can be done except 5400 * report or record it. 5401 */ 5402 dd_dev_info(dd, "CCE Error: %s\n", 5403 cce_err_status_string(buf, sizeof(buf), reg)); 5404 5405 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) && 5406 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) { 5407 /* this error requires a manual drop into SPC freeze mode */ 5408 /* then a fix up */ 5409 start_freeze_handling(dd->pport, FREEZE_SELF); 5410 } 5411 5412 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) { 5413 if (reg & (1ull << i)) { 5414 incr_cntr64(&dd->cce_err_status_cnt[i]); 5415 /* maintain a counter over all cce_err_status errors */ 5416 incr_cntr64(&dd->sw_cce_err_status_aggregate); 5417 } 5418 } 5419 } 5420 5421 /* 5422 * Check counters for receive errors that do not have an interrupt 5423 * associated with them. 5424 */ 5425 #define RCVERR_CHECK_TIME 10 5426 static void update_rcverr_timer(unsigned long opaque) 5427 { 5428 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque; 5429 struct hfi1_pportdata *ppd = dd->pport; 5430 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL); 5431 5432 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt && 5433 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) { 5434 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__); 5435 set_link_down_reason( 5436 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0, 5437 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN); 5438 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work); 5439 } 5440 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt; 5441 5442 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5443 } 5444 5445 static int init_rcverr(struct hfi1_devdata *dd) 5446 { 5447 setup_timer(&dd->rcverr_timer, update_rcverr_timer, (unsigned long)dd); 5448 /* Assume the hardware counter has been reset */ 5449 dd->rcv_ovfl_cnt = 0; 5450 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5451 } 5452 5453 static void free_rcverr(struct hfi1_devdata *dd) 5454 { 5455 if (dd->rcverr_timer.data) 5456 del_timer_sync(&dd->rcverr_timer); 5457 dd->rcverr_timer.data = 0; 5458 } 5459 5460 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5461 { 5462 char buf[96]; 5463 int i = 0; 5464 5465 dd_dev_info(dd, "Receive Error: %s\n", 5466 rxe_err_status_string(buf, sizeof(buf), reg)); 5467 5468 if (reg & ALL_RXE_FREEZE_ERR) { 5469 int flags = 0; 5470 5471 /* 5472 * Freeze mode recovery is disabled for the errors 5473 * in RXE_FREEZE_ABORT_MASK 5474 */ 5475 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK)) 5476 flags = FREEZE_ABORT; 5477 5478 start_freeze_handling(dd->pport, flags); 5479 } 5480 5481 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) { 5482 if (reg & (1ull << i)) 5483 incr_cntr64(&dd->rcv_err_status_cnt[i]); 5484 } 5485 } 5486 5487 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5488 { 5489 char buf[96]; 5490 int i = 0; 5491 5492 dd_dev_info(dd, "Misc Error: %s", 5493 misc_err_status_string(buf, sizeof(buf), reg)); 5494 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) { 5495 if (reg & (1ull << i)) 5496 incr_cntr64(&dd->misc_err_status_cnt[i]); 5497 } 5498 } 5499 5500 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5501 { 5502 char buf[96]; 5503 int i = 0; 5504 5505 dd_dev_info(dd, "PIO Error: %s\n", 5506 pio_err_status_string(buf, sizeof(buf), reg)); 5507 5508 if (reg & ALL_PIO_FREEZE_ERR) 5509 start_freeze_handling(dd->pport, 0); 5510 5511 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) { 5512 if (reg & (1ull << i)) 5513 incr_cntr64(&dd->send_pio_err_status_cnt[i]); 5514 } 5515 } 5516 5517 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5518 { 5519 char buf[96]; 5520 int i = 0; 5521 5522 dd_dev_info(dd, "SDMA Error: %s\n", 5523 sdma_err_status_string(buf, sizeof(buf), reg)); 5524 5525 if (reg & ALL_SDMA_FREEZE_ERR) 5526 start_freeze_handling(dd->pport, 0); 5527 5528 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) { 5529 if (reg & (1ull << i)) 5530 incr_cntr64(&dd->send_dma_err_status_cnt[i]); 5531 } 5532 } 5533 5534 static inline void __count_port_discards(struct hfi1_pportdata *ppd) 5535 { 5536 incr_cntr64(&ppd->port_xmit_discards); 5537 } 5538 5539 static void count_port_inactive(struct hfi1_devdata *dd) 5540 { 5541 __count_port_discards(dd->pport); 5542 } 5543 5544 /* 5545 * We have had a "disallowed packet" error during egress. Determine the 5546 * integrity check which failed, and update relevant error counter, etc. 5547 * 5548 * Note that the SEND_EGRESS_ERR_INFO register has only a single 5549 * bit of state per integrity check, and so we can miss the reason for an 5550 * egress error if more than one packet fails the same integrity check 5551 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO. 5552 */ 5553 static void handle_send_egress_err_info(struct hfi1_devdata *dd, 5554 int vl) 5555 { 5556 struct hfi1_pportdata *ppd = dd->pport; 5557 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */ 5558 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO); 5559 char buf[96]; 5560 5561 /* clear down all observed info as quickly as possible after read */ 5562 write_csr(dd, SEND_EGRESS_ERR_INFO, info); 5563 5564 dd_dev_info(dd, 5565 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n", 5566 info, egress_err_info_string(buf, sizeof(buf), info), src); 5567 5568 /* Eventually add other counters for each bit */ 5569 if (info & PORT_DISCARD_EGRESS_ERRS) { 5570 int weight, i; 5571 5572 /* 5573 * Count all applicable bits as individual errors and 5574 * attribute them to the packet that triggered this handler. 5575 * This may not be completely accurate due to limitations 5576 * on the available hardware error information. There is 5577 * a single information register and any number of error 5578 * packets may have occurred and contributed to it before 5579 * this routine is called. This means that: 5580 * a) If multiple packets with the same error occur before 5581 * this routine is called, earlier packets are missed. 5582 * There is only a single bit for each error type. 5583 * b) Errors may not be attributed to the correct VL. 5584 * The driver is attributing all bits in the info register 5585 * to the packet that triggered this call, but bits 5586 * could be an accumulation of different packets with 5587 * different VLs. 5588 * c) A single error packet may have multiple counts attached 5589 * to it. There is no way for the driver to know if 5590 * multiple bits set in the info register are due to a 5591 * single packet or multiple packets. The driver assumes 5592 * multiple packets. 5593 */ 5594 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS); 5595 for (i = 0; i < weight; i++) { 5596 __count_port_discards(ppd); 5597 if (vl >= 0 && vl < TXE_NUM_DATA_VL) 5598 incr_cntr64(&ppd->port_xmit_discards_vl[vl]); 5599 else if (vl == 15) 5600 incr_cntr64(&ppd->port_xmit_discards_vl 5601 [C_VL_15]); 5602 } 5603 } 5604 } 5605 5606 /* 5607 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5608 * register. Does it represent a 'port inactive' error? 5609 */ 5610 static inline int port_inactive_err(u64 posn) 5611 { 5612 return (posn >= SEES(TX_LINKDOWN) && 5613 posn <= SEES(TX_INCORRECT_LINK_STATE)); 5614 } 5615 5616 /* 5617 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5618 * register. Does it represent a 'disallowed packet' error? 5619 */ 5620 static inline int disallowed_pkt_err(int posn) 5621 { 5622 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) && 5623 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET)); 5624 } 5625 5626 /* 5627 * Input value is a bit position of one of the SDMA engine disallowed 5628 * packet errors. Return which engine. Use of this must be guarded by 5629 * disallowed_pkt_err(). 5630 */ 5631 static inline int disallowed_pkt_engine(int posn) 5632 { 5633 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET); 5634 } 5635 5636 /* 5637 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot 5638 * be done. 5639 */ 5640 static int engine_to_vl(struct hfi1_devdata *dd, int engine) 5641 { 5642 struct sdma_vl_map *m; 5643 int vl; 5644 5645 /* range check */ 5646 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES) 5647 return -1; 5648 5649 rcu_read_lock(); 5650 m = rcu_dereference(dd->sdma_map); 5651 vl = m->engine_to_vl[engine]; 5652 rcu_read_unlock(); 5653 5654 return vl; 5655 } 5656 5657 /* 5658 * Translate the send context (sofware index) into a VL. Return -1 if the 5659 * translation cannot be done. 5660 */ 5661 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index) 5662 { 5663 struct send_context_info *sci; 5664 struct send_context *sc; 5665 int i; 5666 5667 sci = &dd->send_contexts[sw_index]; 5668 5669 /* there is no information for user (PSM) and ack contexts */ 5670 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15)) 5671 return -1; 5672 5673 sc = sci->sc; 5674 if (!sc) 5675 return -1; 5676 if (dd->vld[15].sc == sc) 5677 return 15; 5678 for (i = 0; i < num_vls; i++) 5679 if (dd->vld[i].sc == sc) 5680 return i; 5681 5682 return -1; 5683 } 5684 5685 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5686 { 5687 u64 reg_copy = reg, handled = 0; 5688 char buf[96]; 5689 int i = 0; 5690 5691 if (reg & ALL_TXE_EGRESS_FREEZE_ERR) 5692 start_freeze_handling(dd->pport, 0); 5693 else if (is_ax(dd) && 5694 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) && 5695 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) 5696 start_freeze_handling(dd->pport, 0); 5697 5698 while (reg_copy) { 5699 int posn = fls64(reg_copy); 5700 /* fls64() returns a 1-based offset, we want it zero based */ 5701 int shift = posn - 1; 5702 u64 mask = 1ULL << shift; 5703 5704 if (port_inactive_err(shift)) { 5705 count_port_inactive(dd); 5706 handled |= mask; 5707 } else if (disallowed_pkt_err(shift)) { 5708 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift)); 5709 5710 handle_send_egress_err_info(dd, vl); 5711 handled |= mask; 5712 } 5713 reg_copy &= ~mask; 5714 } 5715 5716 reg &= ~handled; 5717 5718 if (reg) 5719 dd_dev_info(dd, "Egress Error: %s\n", 5720 egress_err_status_string(buf, sizeof(buf), reg)); 5721 5722 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) { 5723 if (reg & (1ull << i)) 5724 incr_cntr64(&dd->send_egress_err_status_cnt[i]); 5725 } 5726 } 5727 5728 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5729 { 5730 char buf[96]; 5731 int i = 0; 5732 5733 dd_dev_info(dd, "Send Error: %s\n", 5734 send_err_status_string(buf, sizeof(buf), reg)); 5735 5736 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) { 5737 if (reg & (1ull << i)) 5738 incr_cntr64(&dd->send_err_status_cnt[i]); 5739 } 5740 } 5741 5742 /* 5743 * The maximum number of times the error clear down will loop before 5744 * blocking a repeating error. This value is arbitrary. 5745 */ 5746 #define MAX_CLEAR_COUNT 20 5747 5748 /* 5749 * Clear and handle an error register. All error interrupts are funneled 5750 * through here to have a central location to correctly handle single- 5751 * or multi-shot errors. 5752 * 5753 * For non per-context registers, call this routine with a context value 5754 * of 0 so the per-context offset is zero. 5755 * 5756 * If the handler loops too many times, assume that something is wrong 5757 * and can't be fixed, so mask the error bits. 5758 */ 5759 static void interrupt_clear_down(struct hfi1_devdata *dd, 5760 u32 context, 5761 const struct err_reg_info *eri) 5762 { 5763 u64 reg; 5764 u32 count; 5765 5766 /* read in a loop until no more errors are seen */ 5767 count = 0; 5768 while (1) { 5769 reg = read_kctxt_csr(dd, context, eri->status); 5770 if (reg == 0) 5771 break; 5772 write_kctxt_csr(dd, context, eri->clear, reg); 5773 if (likely(eri->handler)) 5774 eri->handler(dd, context, reg); 5775 count++; 5776 if (count > MAX_CLEAR_COUNT) { 5777 u64 mask; 5778 5779 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n", 5780 eri->desc, reg); 5781 /* 5782 * Read-modify-write so any other masked bits 5783 * remain masked. 5784 */ 5785 mask = read_kctxt_csr(dd, context, eri->mask); 5786 mask &= ~reg; 5787 write_kctxt_csr(dd, context, eri->mask, mask); 5788 break; 5789 } 5790 } 5791 } 5792 5793 /* 5794 * CCE block "misc" interrupt. Source is < 16. 5795 */ 5796 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source) 5797 { 5798 const struct err_reg_info *eri = &misc_errs[source]; 5799 5800 if (eri->handler) { 5801 interrupt_clear_down(dd, 0, eri); 5802 } else { 5803 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n", 5804 source); 5805 } 5806 } 5807 5808 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags) 5809 { 5810 return flag_string(buf, buf_len, flags, 5811 sc_err_status_flags, 5812 ARRAY_SIZE(sc_err_status_flags)); 5813 } 5814 5815 /* 5816 * Send context error interrupt. Source (hw_context) is < 160. 5817 * 5818 * All send context errors cause the send context to halt. The normal 5819 * clear-down mechanism cannot be used because we cannot clear the 5820 * error bits until several other long-running items are done first. 5821 * This is OK because with the context halted, nothing else is going 5822 * to happen on it anyway. 5823 */ 5824 static void is_sendctxt_err_int(struct hfi1_devdata *dd, 5825 unsigned int hw_context) 5826 { 5827 struct send_context_info *sci; 5828 struct send_context *sc; 5829 char flags[96]; 5830 u64 status; 5831 u32 sw_index; 5832 int i = 0; 5833 5834 sw_index = dd->hw_to_sw[hw_context]; 5835 if (sw_index >= dd->num_send_contexts) { 5836 dd_dev_err(dd, 5837 "out of range sw index %u for send context %u\n", 5838 sw_index, hw_context); 5839 return; 5840 } 5841 sci = &dd->send_contexts[sw_index]; 5842 sc = sci->sc; 5843 if (!sc) { 5844 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__, 5845 sw_index, hw_context); 5846 return; 5847 } 5848 5849 /* tell the software that a halt has begun */ 5850 sc_stop(sc, SCF_HALTED); 5851 5852 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS); 5853 5854 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context, 5855 send_context_err_status_string(flags, sizeof(flags), 5856 status)); 5857 5858 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK) 5859 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index)); 5860 5861 /* 5862 * Automatically restart halted kernel contexts out of interrupt 5863 * context. User contexts must ask the driver to restart the context. 5864 */ 5865 if (sc->type != SC_USER) 5866 queue_work(dd->pport->hfi1_wq, &sc->halt_work); 5867 5868 /* 5869 * Update the counters for the corresponding status bits. 5870 * Note that these particular counters are aggregated over all 5871 * 160 contexts. 5872 */ 5873 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) { 5874 if (status & (1ull << i)) 5875 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]); 5876 } 5877 } 5878 5879 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 5880 unsigned int source, u64 status) 5881 { 5882 struct sdma_engine *sde; 5883 int i = 0; 5884 5885 sde = &dd->per_sdma[source]; 5886 #ifdef CONFIG_SDMA_VERBOSITY 5887 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 5888 slashstrip(__FILE__), __LINE__, __func__); 5889 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n", 5890 sde->this_idx, source, (unsigned long long)status); 5891 #endif 5892 sde->err_cnt++; 5893 sdma_engine_error(sde, status); 5894 5895 /* 5896 * Update the counters for the corresponding status bits. 5897 * Note that these particular counters are aggregated over 5898 * all 16 DMA engines. 5899 */ 5900 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) { 5901 if (status & (1ull << i)) 5902 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]); 5903 } 5904 } 5905 5906 /* 5907 * CCE block SDMA error interrupt. Source is < 16. 5908 */ 5909 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source) 5910 { 5911 #ifdef CONFIG_SDMA_VERBOSITY 5912 struct sdma_engine *sde = &dd->per_sdma[source]; 5913 5914 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 5915 slashstrip(__FILE__), __LINE__, __func__); 5916 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx, 5917 source); 5918 sdma_dumpstate(sde); 5919 #endif 5920 interrupt_clear_down(dd, source, &sdma_eng_err); 5921 } 5922 5923 /* 5924 * CCE block "various" interrupt. Source is < 8. 5925 */ 5926 static void is_various_int(struct hfi1_devdata *dd, unsigned int source) 5927 { 5928 const struct err_reg_info *eri = &various_err[source]; 5929 5930 /* 5931 * TCritInt cannot go through interrupt_clear_down() 5932 * because it is not a second tier interrupt. The handler 5933 * should be called directly. 5934 */ 5935 if (source == TCRIT_INT_SOURCE) 5936 handle_temp_err(dd); 5937 else if (eri->handler) 5938 interrupt_clear_down(dd, 0, eri); 5939 else 5940 dd_dev_info(dd, 5941 "%s: Unimplemented/reserved interrupt %d\n", 5942 __func__, source); 5943 } 5944 5945 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg) 5946 { 5947 /* src_ctx is always zero */ 5948 struct hfi1_pportdata *ppd = dd->pport; 5949 unsigned long flags; 5950 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 5951 5952 if (reg & QSFP_HFI0_MODPRST_N) { 5953 if (!qsfp_mod_present(ppd)) { 5954 dd_dev_info(dd, "%s: QSFP module removed\n", 5955 __func__); 5956 5957 ppd->driver_link_ready = 0; 5958 /* 5959 * Cable removed, reset all our information about the 5960 * cache and cable capabilities 5961 */ 5962 5963 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 5964 /* 5965 * We don't set cache_refresh_required here as we expect 5966 * an interrupt when a cable is inserted 5967 */ 5968 ppd->qsfp_info.cache_valid = 0; 5969 ppd->qsfp_info.reset_needed = 0; 5970 ppd->qsfp_info.limiting_active = 0; 5971 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 5972 flags); 5973 /* Invert the ModPresent pin now to detect plug-in */ 5974 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 5975 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 5976 5977 if ((ppd->offline_disabled_reason > 5978 HFI1_ODR_MASK( 5979 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) || 5980 (ppd->offline_disabled_reason == 5981 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))) 5982 ppd->offline_disabled_reason = 5983 HFI1_ODR_MASK( 5984 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED); 5985 5986 if (ppd->host_link_state == HLS_DN_POLL) { 5987 /* 5988 * The link is still in POLL. This means 5989 * that the normal link down processing 5990 * will not happen. We have to do it here 5991 * before turning the DC off. 5992 */ 5993 queue_work(ppd->hfi1_wq, &ppd->link_down_work); 5994 } 5995 } else { 5996 dd_dev_info(dd, "%s: QSFP module inserted\n", 5997 __func__); 5998 5999 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6000 ppd->qsfp_info.cache_valid = 0; 6001 ppd->qsfp_info.cache_refresh_required = 1; 6002 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6003 flags); 6004 6005 /* 6006 * Stop inversion of ModPresent pin to detect 6007 * removal of the cable 6008 */ 6009 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N; 6010 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6011 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6012 6013 ppd->offline_disabled_reason = 6014 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 6015 } 6016 } 6017 6018 if (reg & QSFP_HFI0_INT_N) { 6019 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n", 6020 __func__); 6021 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6022 ppd->qsfp_info.check_interrupt_flags = 1; 6023 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags); 6024 } 6025 6026 /* Schedule the QSFP work only if there is a cable attached. */ 6027 if (qsfp_mod_present(ppd)) 6028 queue_work(ppd->hfi1_wq, &ppd->qsfp_info.qsfp_work); 6029 } 6030 6031 static int request_host_lcb_access(struct hfi1_devdata *dd) 6032 { 6033 int ret; 6034 6035 ret = do_8051_command(dd, HCMD_MISC, 6036 (u64)HCMD_MISC_REQUEST_LCB_ACCESS << 6037 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6038 if (ret != HCMD_SUCCESS) { 6039 dd_dev_err(dd, "%s: command failed with error %d\n", 6040 __func__, ret); 6041 } 6042 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6043 } 6044 6045 static int request_8051_lcb_access(struct hfi1_devdata *dd) 6046 { 6047 int ret; 6048 6049 ret = do_8051_command(dd, HCMD_MISC, 6050 (u64)HCMD_MISC_GRANT_LCB_ACCESS << 6051 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6052 if (ret != HCMD_SUCCESS) { 6053 dd_dev_err(dd, "%s: command failed with error %d\n", 6054 __func__, ret); 6055 } 6056 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6057 } 6058 6059 /* 6060 * Set the LCB selector - allow host access. The DCC selector always 6061 * points to the host. 6062 */ 6063 static inline void set_host_lcb_access(struct hfi1_devdata *dd) 6064 { 6065 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6066 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK | 6067 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK); 6068 } 6069 6070 /* 6071 * Clear the LCB selector - allow 8051 access. The DCC selector always 6072 * points to the host. 6073 */ 6074 static inline void set_8051_lcb_access(struct hfi1_devdata *dd) 6075 { 6076 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6077 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK); 6078 } 6079 6080 /* 6081 * Acquire LCB access from the 8051. If the host already has access, 6082 * just increment a counter. Otherwise, inform the 8051 that the 6083 * host is taking access. 6084 * 6085 * Returns: 6086 * 0 on success 6087 * -EBUSY if the 8051 has control and cannot be disturbed 6088 * -errno if unable to acquire access from the 8051 6089 */ 6090 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6091 { 6092 struct hfi1_pportdata *ppd = dd->pport; 6093 int ret = 0; 6094 6095 /* 6096 * Use the host link state lock so the operation of this routine 6097 * { link state check, selector change, count increment } can occur 6098 * as a unit against a link state change. Otherwise there is a 6099 * race between the state change and the count increment. 6100 */ 6101 if (sleep_ok) { 6102 mutex_lock(&ppd->hls_lock); 6103 } else { 6104 while (!mutex_trylock(&ppd->hls_lock)) 6105 udelay(1); 6106 } 6107 6108 /* this access is valid only when the link is up */ 6109 if (ppd->host_link_state & HLS_DOWN) { 6110 dd_dev_info(dd, "%s: link state %s not up\n", 6111 __func__, link_state_name(ppd->host_link_state)); 6112 ret = -EBUSY; 6113 goto done; 6114 } 6115 6116 if (dd->lcb_access_count == 0) { 6117 ret = request_host_lcb_access(dd); 6118 if (ret) { 6119 dd_dev_err(dd, 6120 "%s: unable to acquire LCB access, err %d\n", 6121 __func__, ret); 6122 goto done; 6123 } 6124 set_host_lcb_access(dd); 6125 } 6126 dd->lcb_access_count++; 6127 done: 6128 mutex_unlock(&ppd->hls_lock); 6129 return ret; 6130 } 6131 6132 /* 6133 * Release LCB access by decrementing the use count. If the count is moving 6134 * from 1 to 0, inform 8051 that it has control back. 6135 * 6136 * Returns: 6137 * 0 on success 6138 * -errno if unable to release access to the 8051 6139 */ 6140 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6141 { 6142 int ret = 0; 6143 6144 /* 6145 * Use the host link state lock because the acquire needed it. 6146 * Here, we only need to keep { selector change, count decrement } 6147 * as a unit. 6148 */ 6149 if (sleep_ok) { 6150 mutex_lock(&dd->pport->hls_lock); 6151 } else { 6152 while (!mutex_trylock(&dd->pport->hls_lock)) 6153 udelay(1); 6154 } 6155 6156 if (dd->lcb_access_count == 0) { 6157 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n", 6158 __func__); 6159 goto done; 6160 } 6161 6162 if (dd->lcb_access_count == 1) { 6163 set_8051_lcb_access(dd); 6164 ret = request_8051_lcb_access(dd); 6165 if (ret) { 6166 dd_dev_err(dd, 6167 "%s: unable to release LCB access, err %d\n", 6168 __func__, ret); 6169 /* restore host access if the grant didn't work */ 6170 set_host_lcb_access(dd); 6171 goto done; 6172 } 6173 } 6174 dd->lcb_access_count--; 6175 done: 6176 mutex_unlock(&dd->pport->hls_lock); 6177 return ret; 6178 } 6179 6180 /* 6181 * Initialize LCB access variables and state. Called during driver load, 6182 * after most of the initialization is finished. 6183 * 6184 * The DC default is LCB access on for the host. The driver defaults to 6185 * leaving access to the 8051. Assign access now - this constrains the call 6186 * to this routine to be after all LCB set-up is done. In particular, after 6187 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts() 6188 */ 6189 static void init_lcb_access(struct hfi1_devdata *dd) 6190 { 6191 dd->lcb_access_count = 0; 6192 } 6193 6194 /* 6195 * Write a response back to a 8051 request. 6196 */ 6197 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data) 6198 { 6199 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 6200 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK | 6201 (u64)return_code << 6202 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT | 6203 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 6204 } 6205 6206 /* 6207 * Handle host requests from the 8051. 6208 */ 6209 static void handle_8051_request(struct hfi1_pportdata *ppd) 6210 { 6211 struct hfi1_devdata *dd = ppd->dd; 6212 u64 reg; 6213 u16 data = 0; 6214 u8 type; 6215 6216 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1); 6217 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0) 6218 return; /* no request */ 6219 6220 /* zero out COMPLETED so the response is seen */ 6221 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0); 6222 6223 /* extract request details */ 6224 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT) 6225 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK; 6226 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT) 6227 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK; 6228 6229 switch (type) { 6230 case HREQ_LOAD_CONFIG: 6231 case HREQ_SAVE_CONFIG: 6232 case HREQ_READ_CONFIG: 6233 case HREQ_SET_TX_EQ_ABS: 6234 case HREQ_SET_TX_EQ_REL: 6235 case HREQ_ENABLE: 6236 dd_dev_info(dd, "8051 request: request 0x%x not supported\n", 6237 type); 6238 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6239 break; 6240 case HREQ_CONFIG_DONE: 6241 hreq_response(dd, HREQ_SUCCESS, 0); 6242 break; 6243 6244 case HREQ_INTERFACE_TEST: 6245 hreq_response(dd, HREQ_SUCCESS, data); 6246 break; 6247 default: 6248 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type); 6249 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6250 break; 6251 } 6252 } 6253 6254 static void write_global_credit(struct hfi1_devdata *dd, 6255 u8 vau, u16 total, u16 shared) 6256 { 6257 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 6258 ((u64)total << 6259 SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) | 6260 ((u64)shared << 6261 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) | 6262 ((u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT)); 6263 } 6264 6265 /* 6266 * Set up initial VL15 credits of the remote. Assumes the rest of 6267 * the CM credit registers are zero from a previous global or credit reset . 6268 */ 6269 void set_up_vl15(struct hfi1_devdata *dd, u8 vau, u16 vl15buf) 6270 { 6271 /* leave shared count at zero for both global and VL15 */ 6272 write_global_credit(dd, vau, vl15buf, 0); 6273 6274 /* We may need some credits for another VL when sending packets 6275 * with the snoop interface. Dividing it down the middle for VL15 6276 * and VL0 should suffice. 6277 */ 6278 if (unlikely(dd->hfi1_snoop.mode_flag == HFI1_PORT_SNOOP_MODE)) { 6279 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)(vl15buf >> 1) 6280 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT); 6281 write_csr(dd, SEND_CM_CREDIT_VL, (u64)(vl15buf >> 1) 6282 << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT); 6283 } else { 6284 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf 6285 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT); 6286 } 6287 } 6288 6289 /* 6290 * Zero all credit details from the previous connection and 6291 * reset the CM manager's internal counters. 6292 */ 6293 void reset_link_credits(struct hfi1_devdata *dd) 6294 { 6295 int i; 6296 6297 /* remove all previous VL credit limits */ 6298 for (i = 0; i < TXE_NUM_DATA_VL; i++) 6299 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 6300 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 6301 write_global_credit(dd, 0, 0, 0); 6302 /* reset the CM block */ 6303 pio_send_control(dd, PSC_CM_RESET); 6304 } 6305 6306 /* convert a vCU to a CU */ 6307 static u32 vcu_to_cu(u8 vcu) 6308 { 6309 return 1 << vcu; 6310 } 6311 6312 /* convert a CU to a vCU */ 6313 static u8 cu_to_vcu(u32 cu) 6314 { 6315 return ilog2(cu); 6316 } 6317 6318 /* convert a vAU to an AU */ 6319 static u32 vau_to_au(u8 vau) 6320 { 6321 return 8 * (1 << vau); 6322 } 6323 6324 static void set_linkup_defaults(struct hfi1_pportdata *ppd) 6325 { 6326 ppd->sm_trap_qp = 0x0; 6327 ppd->sa_qp = 0x1; 6328 } 6329 6330 /* 6331 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset. 6332 */ 6333 static void lcb_shutdown(struct hfi1_devdata *dd, int abort) 6334 { 6335 u64 reg; 6336 6337 /* clear lcb run: LCB_CFG_RUN.EN = 0 */ 6338 write_csr(dd, DC_LCB_CFG_RUN, 0); 6339 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */ 6340 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 6341 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT); 6342 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */ 6343 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN); 6344 reg = read_csr(dd, DCC_CFG_RESET); 6345 write_csr(dd, DCC_CFG_RESET, reg | 6346 (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) | 6347 (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT)); 6348 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */ 6349 if (!abort) { 6350 udelay(1); /* must hold for the longer of 16cclks or 20ns */ 6351 write_csr(dd, DCC_CFG_RESET, reg); 6352 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6353 } 6354 } 6355 6356 /* 6357 * This routine should be called after the link has been transitioned to 6358 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into 6359 * reset). 6360 * 6361 * The expectation is that the caller of this routine would have taken 6362 * care of properly transitioning the link into the correct state. 6363 */ 6364 static void dc_shutdown(struct hfi1_devdata *dd) 6365 { 6366 unsigned long flags; 6367 6368 spin_lock_irqsave(&dd->dc8051_lock, flags); 6369 if (dd->dc_shutdown) { 6370 spin_unlock_irqrestore(&dd->dc8051_lock, flags); 6371 return; 6372 } 6373 dd->dc_shutdown = 1; 6374 spin_unlock_irqrestore(&dd->dc8051_lock, flags); 6375 /* Shutdown the LCB */ 6376 lcb_shutdown(dd, 1); 6377 /* 6378 * Going to OFFLINE would have causes the 8051 to put the 6379 * SerDes into reset already. Just need to shut down the 8051, 6380 * itself. 6381 */ 6382 write_csr(dd, DC_DC8051_CFG_RST, 0x1); 6383 } 6384 6385 /* 6386 * Calling this after the DC has been brought out of reset should not 6387 * do any damage. 6388 */ 6389 static void dc_start(struct hfi1_devdata *dd) 6390 { 6391 unsigned long flags; 6392 int ret; 6393 6394 spin_lock_irqsave(&dd->dc8051_lock, flags); 6395 if (!dd->dc_shutdown) 6396 goto done; 6397 spin_unlock_irqrestore(&dd->dc8051_lock, flags); 6398 /* Take the 8051 out of reset */ 6399 write_csr(dd, DC_DC8051_CFG_RST, 0ull); 6400 /* Wait until 8051 is ready */ 6401 ret = wait_fm_ready(dd, TIMEOUT_8051_START); 6402 if (ret) { 6403 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n", 6404 __func__); 6405 } 6406 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */ 6407 write_csr(dd, DCC_CFG_RESET, 0x10); 6408 /* lcb_shutdown() with abort=1 does not restore these */ 6409 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6410 spin_lock_irqsave(&dd->dc8051_lock, flags); 6411 dd->dc_shutdown = 0; 6412 done: 6413 spin_unlock_irqrestore(&dd->dc8051_lock, flags); 6414 } 6415 6416 /* 6417 * These LCB adjustments are for the Aurora SerDes core in the FPGA. 6418 */ 6419 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd) 6420 { 6421 u64 rx_radr, tx_radr; 6422 u32 version; 6423 6424 if (dd->icode != ICODE_FPGA_EMULATION) 6425 return; 6426 6427 /* 6428 * These LCB defaults on emulator _s are good, nothing to do here: 6429 * LCB_CFG_TX_FIFOS_RADR 6430 * LCB_CFG_RX_FIFOS_RADR 6431 * LCB_CFG_LN_DCLK 6432 * LCB_CFG_IGNORE_LOST_RCLK 6433 */ 6434 if (is_emulator_s(dd)) 6435 return; 6436 /* else this is _p */ 6437 6438 version = emulator_rev(dd); 6439 if (!is_ax(dd)) 6440 version = 0x2d; /* all B0 use 0x2d or higher settings */ 6441 6442 if (version <= 0x12) { 6443 /* release 0x12 and below */ 6444 6445 /* 6446 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9 6447 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9 6448 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa 6449 */ 6450 rx_radr = 6451 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6452 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6453 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6454 /* 6455 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default) 6456 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6 6457 */ 6458 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6459 } else if (version <= 0x18) { 6460 /* release 0x13 up to 0x18 */ 6461 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6462 rx_radr = 6463 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6464 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6465 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6466 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6467 } else if (version == 0x19) { 6468 /* release 0x19 */ 6469 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */ 6470 rx_radr = 6471 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6472 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6473 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6474 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6475 } else if (version == 0x1a) { 6476 /* release 0x1a */ 6477 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6478 rx_radr = 6479 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6480 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6481 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6482 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6483 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull); 6484 } else { 6485 /* release 0x1b and higher */ 6486 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */ 6487 rx_radr = 6488 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6489 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6490 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6491 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6492 } 6493 6494 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr); 6495 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */ 6496 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 6497 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 6498 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr); 6499 } 6500 6501 /* 6502 * Handle a SMA idle message 6503 * 6504 * This is a work-queue function outside of the interrupt. 6505 */ 6506 void handle_sma_message(struct work_struct *work) 6507 { 6508 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6509 sma_message_work); 6510 struct hfi1_devdata *dd = ppd->dd; 6511 u64 msg; 6512 int ret; 6513 6514 /* 6515 * msg is bytes 1-4 of the 40-bit idle message - the command code 6516 * is stripped off 6517 */ 6518 ret = read_idle_sma(dd, &msg); 6519 if (ret) 6520 return; 6521 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg); 6522 /* 6523 * React to the SMA message. Byte[1] (0 for us) is the command. 6524 */ 6525 switch (msg & 0xff) { 6526 case SMA_IDLE_ARM: 6527 /* 6528 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6529 * State Transitions 6530 * 6531 * Only expected in INIT or ARMED, discard otherwise. 6532 */ 6533 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED)) 6534 ppd->neighbor_normal = 1; 6535 break; 6536 case SMA_IDLE_ACTIVE: 6537 /* 6538 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6539 * State Transitions 6540 * 6541 * Can activate the node. Discard otherwise. 6542 */ 6543 if (ppd->host_link_state == HLS_UP_ARMED && 6544 ppd->is_active_optimize_enabled) { 6545 ppd->neighbor_normal = 1; 6546 ret = set_link_state(ppd, HLS_UP_ACTIVE); 6547 if (ret) 6548 dd_dev_err( 6549 dd, 6550 "%s: received Active SMA idle message, couldn't set link to Active\n", 6551 __func__); 6552 } 6553 break; 6554 default: 6555 dd_dev_err(dd, 6556 "%s: received unexpected SMA idle message 0x%llx\n", 6557 __func__, msg); 6558 break; 6559 } 6560 } 6561 6562 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear) 6563 { 6564 u64 rcvctrl; 6565 unsigned long flags; 6566 6567 spin_lock_irqsave(&dd->rcvctrl_lock, flags); 6568 rcvctrl = read_csr(dd, RCV_CTRL); 6569 rcvctrl |= add; 6570 rcvctrl &= ~clear; 6571 write_csr(dd, RCV_CTRL, rcvctrl); 6572 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags); 6573 } 6574 6575 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add) 6576 { 6577 adjust_rcvctrl(dd, add, 0); 6578 } 6579 6580 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear) 6581 { 6582 adjust_rcvctrl(dd, 0, clear); 6583 } 6584 6585 /* 6586 * Called from all interrupt handlers to start handling an SPC freeze. 6587 */ 6588 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags) 6589 { 6590 struct hfi1_devdata *dd = ppd->dd; 6591 struct send_context *sc; 6592 int i; 6593 6594 if (flags & FREEZE_SELF) 6595 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6596 6597 /* enter frozen mode */ 6598 dd->flags |= HFI1_FROZEN; 6599 6600 /* notify all SDMA engines that they are going into a freeze */ 6601 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN)); 6602 6603 /* do halt pre-handling on all enabled send contexts */ 6604 for (i = 0; i < dd->num_send_contexts; i++) { 6605 sc = dd->send_contexts[i].sc; 6606 if (sc && (sc->flags & SCF_ENABLED)) 6607 sc_stop(sc, SCF_FROZEN | SCF_HALTED); 6608 } 6609 6610 /* Send context are frozen. Notify user space */ 6611 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT); 6612 6613 if (flags & FREEZE_ABORT) { 6614 dd_dev_err(dd, 6615 "Aborted freeze recovery. Please REBOOT system\n"); 6616 return; 6617 } 6618 /* queue non-interrupt handler */ 6619 queue_work(ppd->hfi1_wq, &ppd->freeze_work); 6620 } 6621 6622 /* 6623 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen, 6624 * depending on the "freeze" parameter. 6625 * 6626 * No need to return an error if it times out, our only option 6627 * is to proceed anyway. 6628 */ 6629 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze) 6630 { 6631 unsigned long timeout; 6632 u64 reg; 6633 6634 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT); 6635 while (1) { 6636 reg = read_csr(dd, CCE_STATUS); 6637 if (freeze) { 6638 /* waiting until all indicators are set */ 6639 if ((reg & ALL_FROZE) == ALL_FROZE) 6640 return; /* all done */ 6641 } else { 6642 /* waiting until all indicators are clear */ 6643 if ((reg & ALL_FROZE) == 0) 6644 return; /* all done */ 6645 } 6646 6647 if (time_after(jiffies, timeout)) { 6648 dd_dev_err(dd, 6649 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing", 6650 freeze ? "" : "un", reg & ALL_FROZE, 6651 freeze ? ALL_FROZE : 0ull); 6652 return; 6653 } 6654 usleep_range(80, 120); 6655 } 6656 } 6657 6658 /* 6659 * Do all freeze handling for the RXE block. 6660 */ 6661 static void rxe_freeze(struct hfi1_devdata *dd) 6662 { 6663 int i; 6664 6665 /* disable port */ 6666 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6667 6668 /* disable all receive contexts */ 6669 for (i = 0; i < dd->num_rcv_contexts; i++) 6670 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, i); 6671 } 6672 6673 /* 6674 * Unfreeze handling for the RXE block - kernel contexts only. 6675 * This will also enable the port. User contexts will do unfreeze 6676 * handling on a per-context basis as they call into the driver. 6677 * 6678 */ 6679 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd) 6680 { 6681 u32 rcvmask; 6682 int i; 6683 6684 /* enable all kernel contexts */ 6685 for (i = 0; i < dd->n_krcv_queues; i++) { 6686 rcvmask = HFI1_RCVCTRL_CTXT_ENB; 6687 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */ 6688 rcvmask |= HFI1_CAP_KGET_MASK(dd->rcd[i]->flags, DMA_RTAIL) ? 6689 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS; 6690 hfi1_rcvctrl(dd, rcvmask, i); 6691 } 6692 6693 /* enable port */ 6694 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6695 } 6696 6697 /* 6698 * Non-interrupt SPC freeze handling. 6699 * 6700 * This is a work-queue function outside of the triggering interrupt. 6701 */ 6702 void handle_freeze(struct work_struct *work) 6703 { 6704 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6705 freeze_work); 6706 struct hfi1_devdata *dd = ppd->dd; 6707 6708 /* wait for freeze indicators on all affected blocks */ 6709 wait_for_freeze_status(dd, 1); 6710 6711 /* SPC is now frozen */ 6712 6713 /* do send PIO freeze steps */ 6714 pio_freeze(dd); 6715 6716 /* do send DMA freeze steps */ 6717 sdma_freeze(dd); 6718 6719 /* do send egress freeze steps - nothing to do */ 6720 6721 /* do receive freeze steps */ 6722 rxe_freeze(dd); 6723 6724 /* 6725 * Unfreeze the hardware - clear the freeze, wait for each 6726 * block's frozen bit to clear, then clear the frozen flag. 6727 */ 6728 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6729 wait_for_freeze_status(dd, 0); 6730 6731 if (is_ax(dd)) { 6732 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6733 wait_for_freeze_status(dd, 1); 6734 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6735 wait_for_freeze_status(dd, 0); 6736 } 6737 6738 /* do send PIO unfreeze steps for kernel contexts */ 6739 pio_kernel_unfreeze(dd); 6740 6741 /* do send DMA unfreeze steps */ 6742 sdma_unfreeze(dd); 6743 6744 /* do send egress unfreeze steps - nothing to do */ 6745 6746 /* do receive unfreeze steps for kernel contexts */ 6747 rxe_kernel_unfreeze(dd); 6748 6749 /* 6750 * The unfreeze procedure touches global device registers when 6751 * it disables and re-enables RXE. Mark the device unfrozen 6752 * after all that is done so other parts of the driver waiting 6753 * for the device to unfreeze don't do things out of order. 6754 * 6755 * The above implies that the meaning of HFI1_FROZEN flag is 6756 * "Device has gone into freeze mode and freeze mode handling 6757 * is still in progress." 6758 * 6759 * The flag will be removed when freeze mode processing has 6760 * completed. 6761 */ 6762 dd->flags &= ~HFI1_FROZEN; 6763 wake_up(&dd->event_queue); 6764 6765 /* no longer frozen */ 6766 } 6767 6768 /* 6769 * Handle a link up interrupt from the 8051. 6770 * 6771 * This is a work-queue function outside of the interrupt. 6772 */ 6773 void handle_link_up(struct work_struct *work) 6774 { 6775 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6776 link_up_work); 6777 set_link_state(ppd, HLS_UP_INIT); 6778 6779 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */ 6780 read_ltp_rtt(ppd->dd); 6781 /* 6782 * OPA specifies that certain counters are cleared on a transition 6783 * to link up, so do that. 6784 */ 6785 clear_linkup_counters(ppd->dd); 6786 /* 6787 * And (re)set link up default values. 6788 */ 6789 set_linkup_defaults(ppd); 6790 6791 /* enforce link speed enabled */ 6792 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) { 6793 /* oops - current speed is not enabled, bounce */ 6794 dd_dev_err(ppd->dd, 6795 "Link speed active 0x%x is outside enabled 0x%x, downing link\n", 6796 ppd->link_speed_active, ppd->link_speed_enabled); 6797 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0, 6798 OPA_LINKDOWN_REASON_SPEED_POLICY); 6799 set_link_state(ppd, HLS_DN_OFFLINE); 6800 tune_serdes(ppd); 6801 start_link(ppd); 6802 } 6803 } 6804 6805 /* 6806 * Several pieces of LNI information were cached for SMA in ppd. 6807 * Reset these on link down 6808 */ 6809 static void reset_neighbor_info(struct hfi1_pportdata *ppd) 6810 { 6811 ppd->neighbor_guid = 0; 6812 ppd->neighbor_port_number = 0; 6813 ppd->neighbor_type = 0; 6814 ppd->neighbor_fm_security = 0; 6815 } 6816 6817 static const char * const link_down_reason_strs[] = { 6818 [OPA_LINKDOWN_REASON_NONE] = "None", 6819 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Recive error 0", 6820 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length", 6821 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long", 6822 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short", 6823 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID", 6824 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID", 6825 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2", 6826 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC", 6827 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8", 6828 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail", 6829 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10", 6830 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error", 6831 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15", 6832 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker", 6833 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14", 6834 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15", 6835 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance", 6836 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance", 6837 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance", 6838 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack", 6839 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker", 6840 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt", 6841 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit", 6842 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit", 6843 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24", 6844 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25", 6845 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26", 6846 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27", 6847 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28", 6848 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29", 6849 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30", 6850 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] = 6851 "Excessive buffer overrun", 6852 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown", 6853 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot", 6854 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown", 6855 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce", 6856 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy", 6857 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy", 6858 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected", 6859 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] = 6860 "Local media not installed", 6861 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed", 6862 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config", 6863 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] = 6864 "End to end not installed", 6865 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy", 6866 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy", 6867 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy", 6868 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management", 6869 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled", 6870 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient" 6871 }; 6872 6873 /* return the neighbor link down reason string */ 6874 static const char *link_down_reason_str(u8 reason) 6875 { 6876 const char *str = NULL; 6877 6878 if (reason < ARRAY_SIZE(link_down_reason_strs)) 6879 str = link_down_reason_strs[reason]; 6880 if (!str) 6881 str = "(invalid)"; 6882 6883 return str; 6884 } 6885 6886 /* 6887 * Handle a link down interrupt from the 8051. 6888 * 6889 * This is a work-queue function outside of the interrupt. 6890 */ 6891 void handle_link_down(struct work_struct *work) 6892 { 6893 u8 lcl_reason, neigh_reason = 0; 6894 u8 link_down_reason; 6895 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6896 link_down_work); 6897 int was_up; 6898 static const char ldr_str[] = "Link down reason: "; 6899 6900 if ((ppd->host_link_state & 6901 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) && 6902 ppd->port_type == PORT_TYPE_FIXED) 6903 ppd->offline_disabled_reason = 6904 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED); 6905 6906 /* Go offline first, then deal with reading/writing through 8051 */ 6907 was_up = !!(ppd->host_link_state & HLS_UP); 6908 set_link_state(ppd, HLS_DN_OFFLINE); 6909 6910 if (was_up) { 6911 lcl_reason = 0; 6912 /* link down reason is only valid if the link was up */ 6913 read_link_down_reason(ppd->dd, &link_down_reason); 6914 switch (link_down_reason) { 6915 case LDR_LINK_TRANSFER_ACTIVE_LOW: 6916 /* the link went down, no idle message reason */ 6917 dd_dev_info(ppd->dd, "%sUnexpected link down\n", 6918 ldr_str); 6919 break; 6920 case LDR_RECEIVED_LINKDOWN_IDLE_MSG: 6921 /* 6922 * The neighbor reason is only valid if an idle message 6923 * was received for it. 6924 */ 6925 read_planned_down_reason_code(ppd->dd, &neigh_reason); 6926 dd_dev_info(ppd->dd, 6927 "%sNeighbor link down message %d, %s\n", 6928 ldr_str, neigh_reason, 6929 link_down_reason_str(neigh_reason)); 6930 break; 6931 case LDR_RECEIVED_HOST_OFFLINE_REQ: 6932 dd_dev_info(ppd->dd, 6933 "%sHost requested link to go offline\n", 6934 ldr_str); 6935 break; 6936 default: 6937 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n", 6938 ldr_str, link_down_reason); 6939 break; 6940 } 6941 6942 /* 6943 * If no reason, assume peer-initiated but missed 6944 * LinkGoingDown idle flits. 6945 */ 6946 if (neigh_reason == 0) 6947 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN; 6948 } else { 6949 /* went down while polling or going up */ 6950 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT; 6951 } 6952 6953 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0); 6954 6955 /* inform the SMA when the link transitions from up to down */ 6956 if (was_up && ppd->local_link_down_reason.sma == 0 && 6957 ppd->neigh_link_down_reason.sma == 0) { 6958 ppd->local_link_down_reason.sma = 6959 ppd->local_link_down_reason.latest; 6960 ppd->neigh_link_down_reason.sma = 6961 ppd->neigh_link_down_reason.latest; 6962 } 6963 6964 reset_neighbor_info(ppd); 6965 6966 /* disable the port */ 6967 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6968 6969 /* 6970 * If there is no cable attached, turn the DC off. Otherwise, 6971 * start the link bring up. 6972 */ 6973 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) { 6974 dc_shutdown(ppd->dd); 6975 } else { 6976 tune_serdes(ppd); 6977 start_link(ppd); 6978 } 6979 } 6980 6981 void handle_link_bounce(struct work_struct *work) 6982 { 6983 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6984 link_bounce_work); 6985 6986 /* 6987 * Only do something if the link is currently up. 6988 */ 6989 if (ppd->host_link_state & HLS_UP) { 6990 set_link_state(ppd, HLS_DN_OFFLINE); 6991 tune_serdes(ppd); 6992 start_link(ppd); 6993 } else { 6994 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n", 6995 __func__, link_state_name(ppd->host_link_state)); 6996 } 6997 } 6998 6999 /* 7000 * Mask conversion: Capability exchange to Port LTP. The capability 7001 * exchange has an implicit 16b CRC that is mandatory. 7002 */ 7003 static int cap_to_port_ltp(int cap) 7004 { 7005 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */ 7006 7007 if (cap & CAP_CRC_14B) 7008 port_ltp |= PORT_LTP_CRC_MODE_14; 7009 if (cap & CAP_CRC_48B) 7010 port_ltp |= PORT_LTP_CRC_MODE_48; 7011 if (cap & CAP_CRC_12B_16B_PER_LANE) 7012 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE; 7013 7014 return port_ltp; 7015 } 7016 7017 /* 7018 * Convert an OPA Port LTP mask to capability mask 7019 */ 7020 int port_ltp_to_cap(int port_ltp) 7021 { 7022 int cap_mask = 0; 7023 7024 if (port_ltp & PORT_LTP_CRC_MODE_14) 7025 cap_mask |= CAP_CRC_14B; 7026 if (port_ltp & PORT_LTP_CRC_MODE_48) 7027 cap_mask |= CAP_CRC_48B; 7028 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE) 7029 cap_mask |= CAP_CRC_12B_16B_PER_LANE; 7030 7031 return cap_mask; 7032 } 7033 7034 /* 7035 * Convert a single DC LCB CRC mode to an OPA Port LTP mask. 7036 */ 7037 static int lcb_to_port_ltp(int lcb_crc) 7038 { 7039 int port_ltp = 0; 7040 7041 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE) 7042 port_ltp = PORT_LTP_CRC_MODE_PER_LANE; 7043 else if (lcb_crc == LCB_CRC_48B) 7044 port_ltp = PORT_LTP_CRC_MODE_48; 7045 else if (lcb_crc == LCB_CRC_14B) 7046 port_ltp = PORT_LTP_CRC_MODE_14; 7047 else 7048 port_ltp = PORT_LTP_CRC_MODE_16; 7049 7050 return port_ltp; 7051 } 7052 7053 /* 7054 * Our neighbor has indicated that we are allowed to act as a fabric 7055 * manager, so place the full management partition key in the second 7056 * (0-based) pkey array position (see OPAv1, section 20.2.2.6.8). Note 7057 * that we should already have the limited management partition key in 7058 * array element 1, and also that the port is not yet up when 7059 * add_full_mgmt_pkey() is invoked. 7060 */ 7061 static void add_full_mgmt_pkey(struct hfi1_pportdata *ppd) 7062 { 7063 struct hfi1_devdata *dd = ppd->dd; 7064 7065 /* Sanity check - ppd->pkeys[2] should be 0, or already initalized */ 7066 if (!((ppd->pkeys[2] == 0) || (ppd->pkeys[2] == FULL_MGMT_P_KEY))) 7067 dd_dev_warn(dd, "%s pkey[2] already set to 0x%x, resetting it to 0x%x\n", 7068 __func__, ppd->pkeys[2], FULL_MGMT_P_KEY); 7069 ppd->pkeys[2] = FULL_MGMT_P_KEY; 7070 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); 7071 hfi1_event_pkey_change(ppd->dd, ppd->port); 7072 } 7073 7074 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd) 7075 { 7076 if (ppd->pkeys[2] != 0) { 7077 ppd->pkeys[2] = 0; 7078 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); 7079 hfi1_event_pkey_change(ppd->dd, ppd->port); 7080 } 7081 } 7082 7083 /* 7084 * Convert the given link width to the OPA link width bitmask. 7085 */ 7086 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width) 7087 { 7088 switch (width) { 7089 case 0: 7090 /* 7091 * Simulator and quick linkup do not set the width. 7092 * Just set it to 4x without complaint. 7093 */ 7094 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup) 7095 return OPA_LINK_WIDTH_4X; 7096 return 0; /* no lanes up */ 7097 case 1: return OPA_LINK_WIDTH_1X; 7098 case 2: return OPA_LINK_WIDTH_2X; 7099 case 3: return OPA_LINK_WIDTH_3X; 7100 default: 7101 dd_dev_info(dd, "%s: invalid width %d, using 4\n", 7102 __func__, width); 7103 /* fall through */ 7104 case 4: return OPA_LINK_WIDTH_4X; 7105 } 7106 } 7107 7108 /* 7109 * Do a population count on the bottom nibble. 7110 */ 7111 static const u8 bit_counts[16] = { 7112 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 7113 }; 7114 7115 static inline u8 nibble_to_count(u8 nibble) 7116 { 7117 return bit_counts[nibble & 0xf]; 7118 } 7119 7120 /* 7121 * Read the active lane information from the 8051 registers and return 7122 * their widths. 7123 * 7124 * Active lane information is found in these 8051 registers: 7125 * enable_lane_tx 7126 * enable_lane_rx 7127 */ 7128 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width, 7129 u16 *rx_width) 7130 { 7131 u16 tx, rx; 7132 u8 enable_lane_rx; 7133 u8 enable_lane_tx; 7134 u8 tx_polarity_inversion; 7135 u8 rx_polarity_inversion; 7136 u8 max_rate; 7137 7138 /* read the active lanes */ 7139 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 7140 &rx_polarity_inversion, &max_rate); 7141 read_local_lni(dd, &enable_lane_rx); 7142 7143 /* convert to counts */ 7144 tx = nibble_to_count(enable_lane_tx); 7145 rx = nibble_to_count(enable_lane_rx); 7146 7147 /* 7148 * Set link_speed_active here, overriding what was set in 7149 * handle_verify_cap(). The ASIC 8051 firmware does not correctly 7150 * set the max_rate field in handle_verify_cap until v0.19. 7151 */ 7152 if ((dd->icode == ICODE_RTL_SILICON) && 7153 (dd->dc8051_ver < dc8051_ver(0, 19))) { 7154 /* max_rate: 0 = 12.5G, 1 = 25G */ 7155 switch (max_rate) { 7156 case 0: 7157 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G; 7158 break; 7159 default: 7160 dd_dev_err(dd, 7161 "%s: unexpected max rate %d, using 25Gb\n", 7162 __func__, (int)max_rate); 7163 /* fall through */ 7164 case 1: 7165 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; 7166 break; 7167 } 7168 } 7169 7170 dd_dev_info(dd, 7171 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n", 7172 enable_lane_tx, tx, enable_lane_rx, rx); 7173 *tx_width = link_width_to_bits(dd, tx); 7174 *rx_width = link_width_to_bits(dd, rx); 7175 } 7176 7177 /* 7178 * Read verify_cap_local_fm_link_width[1] to obtain the link widths. 7179 * Valid after the end of VerifyCap and during LinkUp. Does not change 7180 * after link up. I.e. look elsewhere for downgrade information. 7181 * 7182 * Bits are: 7183 * + bits [7:4] contain the number of active transmitters 7184 * + bits [3:0] contain the number of active receivers 7185 * These are numbers 1 through 4 and can be different values if the 7186 * link is asymmetric. 7187 * 7188 * verify_cap_local_fm_link_width[0] retains its original value. 7189 */ 7190 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width, 7191 u16 *rx_width) 7192 { 7193 u16 widths, tx, rx; 7194 u8 misc_bits, local_flags; 7195 u16 active_tx, active_rx; 7196 7197 read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths); 7198 tx = widths >> 12; 7199 rx = (widths >> 8) & 0xf; 7200 7201 *tx_width = link_width_to_bits(dd, tx); 7202 *rx_width = link_width_to_bits(dd, rx); 7203 7204 /* print the active widths */ 7205 get_link_widths(dd, &active_tx, &active_rx); 7206 } 7207 7208 /* 7209 * Set ppd->link_width_active and ppd->link_width_downgrade_active using 7210 * hardware information when the link first comes up. 7211 * 7212 * The link width is not available until after VerifyCap.AllFramesReceived 7213 * (the trigger for handle_verify_cap), so this is outside that routine 7214 * and should be called when the 8051 signals linkup. 7215 */ 7216 void get_linkup_link_widths(struct hfi1_pportdata *ppd) 7217 { 7218 u16 tx_width, rx_width; 7219 7220 /* get end-of-LNI link widths */ 7221 get_linkup_widths(ppd->dd, &tx_width, &rx_width); 7222 7223 /* use tx_width as the link is supposed to be symmetric on link up */ 7224 ppd->link_width_active = tx_width; 7225 /* link width downgrade active (LWD.A) starts out matching LW.A */ 7226 ppd->link_width_downgrade_tx_active = ppd->link_width_active; 7227 ppd->link_width_downgrade_rx_active = ppd->link_width_active; 7228 /* per OPA spec, on link up LWD.E resets to LWD.S */ 7229 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported; 7230 /* cache the active egress rate (units {10^6 bits/sec]) */ 7231 ppd->current_egress_rate = active_egress_rate(ppd); 7232 } 7233 7234 /* 7235 * Handle a verify capabilities interrupt from the 8051. 7236 * 7237 * This is a work-queue function outside of the interrupt. 7238 */ 7239 void handle_verify_cap(struct work_struct *work) 7240 { 7241 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7242 link_vc_work); 7243 struct hfi1_devdata *dd = ppd->dd; 7244 u64 reg; 7245 u8 power_management; 7246 u8 continious; 7247 u8 vcu; 7248 u8 vau; 7249 u8 z; 7250 u16 vl15buf; 7251 u16 link_widths; 7252 u16 crc_mask; 7253 u16 crc_val; 7254 u16 device_id; 7255 u16 active_tx, active_rx; 7256 u8 partner_supported_crc; 7257 u8 remote_tx_rate; 7258 u8 device_rev; 7259 7260 set_link_state(ppd, HLS_VERIFY_CAP); 7261 7262 lcb_shutdown(dd, 0); 7263 adjust_lcb_for_fpga_serdes(dd); 7264 7265 /* 7266 * These are now valid: 7267 * remote VerifyCap fields in the general LNI config 7268 * CSR DC8051_STS_REMOTE_GUID 7269 * CSR DC8051_STS_REMOTE_NODE_TYPE 7270 * CSR DC8051_STS_REMOTE_FM_SECURITY 7271 * CSR DC8051_STS_REMOTE_PORT_NO 7272 */ 7273 7274 read_vc_remote_phy(dd, &power_management, &continious); 7275 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf, 7276 &partner_supported_crc); 7277 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths); 7278 read_remote_device_id(dd, &device_id, &device_rev); 7279 /* 7280 * And the 'MgmtAllowed' information, which is exchanged during 7281 * LNI, is also be available at this point. 7282 */ 7283 read_mgmt_allowed(dd, &ppd->mgmt_allowed); 7284 /* print the active widths */ 7285 get_link_widths(dd, &active_tx, &active_rx); 7286 dd_dev_info(dd, 7287 "Peer PHY: power management 0x%x, continuous updates 0x%x\n", 7288 (int)power_management, (int)continious); 7289 dd_dev_info(dd, 7290 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n", 7291 (int)vau, (int)z, (int)vcu, (int)vl15buf, 7292 (int)partner_supported_crc); 7293 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n", 7294 (u32)remote_tx_rate, (u32)link_widths); 7295 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n", 7296 (u32)device_id, (u32)device_rev); 7297 /* 7298 * The peer vAU value just read is the peer receiver value. HFI does 7299 * not support a transmit vAU of 0 (AU == 8). We advertised that 7300 * with Z=1 in the fabric capabilities sent to the peer. The peer 7301 * will see our Z=1, and, if it advertised a vAU of 0, will move its 7302 * receive to vAU of 1 (AU == 16). Do the same here. We do not care 7303 * about the peer Z value - our sent vAU is 3 (hardwired) and is not 7304 * subject to the Z value exception. 7305 */ 7306 if (vau == 0) 7307 vau = 1; 7308 set_up_vl15(dd, vau, vl15buf); 7309 7310 /* set up the LCB CRC mode */ 7311 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc; 7312 7313 /* order is important: use the lowest bit in common */ 7314 if (crc_mask & CAP_CRC_14B) 7315 crc_val = LCB_CRC_14B; 7316 else if (crc_mask & CAP_CRC_48B) 7317 crc_val = LCB_CRC_48B; 7318 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE) 7319 crc_val = LCB_CRC_12B_16B_PER_LANE; 7320 else 7321 crc_val = LCB_CRC_16B; 7322 7323 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val); 7324 write_csr(dd, DC_LCB_CFG_CRC_MODE, 7325 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT); 7326 7327 /* set (14b only) or clear sideband credit */ 7328 reg = read_csr(dd, SEND_CM_CTRL); 7329 if (crc_val == LCB_CRC_14B && crc_14b_sideband) { 7330 write_csr(dd, SEND_CM_CTRL, 7331 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7332 } else { 7333 write_csr(dd, SEND_CM_CTRL, 7334 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7335 } 7336 7337 ppd->link_speed_active = 0; /* invalid value */ 7338 if (dd->dc8051_ver < dc8051_ver(0, 20)) { 7339 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */ 7340 switch (remote_tx_rate) { 7341 case 0: 7342 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7343 break; 7344 case 1: 7345 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7346 break; 7347 } 7348 } else { 7349 /* actual rate is highest bit of the ANDed rates */ 7350 u8 rate = remote_tx_rate & ppd->local_tx_rate; 7351 7352 if (rate & 2) 7353 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7354 else if (rate & 1) 7355 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7356 } 7357 if (ppd->link_speed_active == 0) { 7358 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n", 7359 __func__, (int)remote_tx_rate); 7360 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7361 } 7362 7363 /* 7364 * Cache the values of the supported, enabled, and active 7365 * LTP CRC modes to return in 'portinfo' queries. But the bit 7366 * flags that are returned in the portinfo query differ from 7367 * what's in the link_crc_mask, crc_sizes, and crc_val 7368 * variables. Convert these here. 7369 */ 7370 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 7371 /* supported crc modes */ 7372 ppd->port_ltp_crc_mode |= 7373 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4; 7374 /* enabled crc modes */ 7375 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val); 7376 /* active crc mode */ 7377 7378 /* set up the remote credit return table */ 7379 assign_remote_cm_au_table(dd, vcu); 7380 7381 /* 7382 * The LCB is reset on entry to handle_verify_cap(), so this must 7383 * be applied on every link up. 7384 * 7385 * Adjust LCB error kill enable to kill the link if 7386 * these RBUF errors are seen: 7387 * REPLAY_BUF_MBE_SMASK 7388 * FLIT_INPUT_BUF_MBE_SMASK 7389 */ 7390 if (is_ax(dd)) { /* fixed in B0 */ 7391 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN); 7392 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK 7393 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK; 7394 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg); 7395 } 7396 7397 /* pull LCB fifos out of reset - all fifo clocks must be stable */ 7398 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 7399 7400 /* give 8051 access to the LCB CSRs */ 7401 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 7402 set_8051_lcb_access(dd); 7403 7404 ppd->neighbor_guid = 7405 read_csr(dd, DC_DC8051_STS_REMOTE_GUID); 7406 ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) & 7407 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK; 7408 ppd->neighbor_type = 7409 read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) & 7410 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK; 7411 ppd->neighbor_fm_security = 7412 read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) & 7413 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK; 7414 dd_dev_info(dd, 7415 "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n", 7416 ppd->neighbor_guid, ppd->neighbor_type, 7417 ppd->mgmt_allowed, ppd->neighbor_fm_security); 7418 if (ppd->mgmt_allowed) 7419 add_full_mgmt_pkey(ppd); 7420 7421 /* tell the 8051 to go to LinkUp */ 7422 set_link_state(ppd, HLS_GOING_UP); 7423 } 7424 7425 /* 7426 * Apply the link width downgrade enabled policy against the current active 7427 * link widths. 7428 * 7429 * Called when the enabled policy changes or the active link widths change. 7430 */ 7431 void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths) 7432 { 7433 int do_bounce = 0; 7434 int tries; 7435 u16 lwde; 7436 u16 tx, rx; 7437 7438 /* use the hls lock to avoid a race with actual link up */ 7439 tries = 0; 7440 retry: 7441 mutex_lock(&ppd->hls_lock); 7442 /* only apply if the link is up */ 7443 if (ppd->host_link_state & HLS_DOWN) { 7444 /* still going up..wait and retry */ 7445 if (ppd->host_link_state & HLS_GOING_UP) { 7446 if (++tries < 1000) { 7447 mutex_unlock(&ppd->hls_lock); 7448 usleep_range(100, 120); /* arbitrary */ 7449 goto retry; 7450 } 7451 dd_dev_err(ppd->dd, 7452 "%s: giving up waiting for link state change\n", 7453 __func__); 7454 } 7455 goto done; 7456 } 7457 7458 lwde = ppd->link_width_downgrade_enabled; 7459 7460 if (refresh_widths) { 7461 get_link_widths(ppd->dd, &tx, &rx); 7462 ppd->link_width_downgrade_tx_active = tx; 7463 ppd->link_width_downgrade_rx_active = rx; 7464 } 7465 7466 if (ppd->link_width_downgrade_tx_active == 0 || 7467 ppd->link_width_downgrade_rx_active == 0) { 7468 /* the 8051 reported a dead link as a downgrade */ 7469 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n"); 7470 } else if (lwde == 0) { 7471 /* downgrade is disabled */ 7472 7473 /* bounce if not at starting active width */ 7474 if ((ppd->link_width_active != 7475 ppd->link_width_downgrade_tx_active) || 7476 (ppd->link_width_active != 7477 ppd->link_width_downgrade_rx_active)) { 7478 dd_dev_err(ppd->dd, 7479 "Link downgrade is disabled and link has downgraded, downing link\n"); 7480 dd_dev_err(ppd->dd, 7481 " original 0x%x, tx active 0x%x, rx active 0x%x\n", 7482 ppd->link_width_active, 7483 ppd->link_width_downgrade_tx_active, 7484 ppd->link_width_downgrade_rx_active); 7485 do_bounce = 1; 7486 } 7487 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 || 7488 (lwde & ppd->link_width_downgrade_rx_active) == 0) { 7489 /* Tx or Rx is outside the enabled policy */ 7490 dd_dev_err(ppd->dd, 7491 "Link is outside of downgrade allowed, downing link\n"); 7492 dd_dev_err(ppd->dd, 7493 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n", 7494 lwde, ppd->link_width_downgrade_tx_active, 7495 ppd->link_width_downgrade_rx_active); 7496 do_bounce = 1; 7497 } 7498 7499 done: 7500 mutex_unlock(&ppd->hls_lock); 7501 7502 if (do_bounce) { 7503 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0, 7504 OPA_LINKDOWN_REASON_WIDTH_POLICY); 7505 set_link_state(ppd, HLS_DN_OFFLINE); 7506 tune_serdes(ppd); 7507 start_link(ppd); 7508 } 7509 } 7510 7511 /* 7512 * Handle a link downgrade interrupt from the 8051. 7513 * 7514 * This is a work-queue function outside of the interrupt. 7515 */ 7516 void handle_link_downgrade(struct work_struct *work) 7517 { 7518 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7519 link_downgrade_work); 7520 7521 dd_dev_info(ppd->dd, "8051: Link width downgrade\n"); 7522 apply_link_downgrade_policy(ppd, 1); 7523 } 7524 7525 static char *dcc_err_string(char *buf, int buf_len, u64 flags) 7526 { 7527 return flag_string(buf, buf_len, flags, dcc_err_flags, 7528 ARRAY_SIZE(dcc_err_flags)); 7529 } 7530 7531 static char *lcb_err_string(char *buf, int buf_len, u64 flags) 7532 { 7533 return flag_string(buf, buf_len, flags, lcb_err_flags, 7534 ARRAY_SIZE(lcb_err_flags)); 7535 } 7536 7537 static char *dc8051_err_string(char *buf, int buf_len, u64 flags) 7538 { 7539 return flag_string(buf, buf_len, flags, dc8051_err_flags, 7540 ARRAY_SIZE(dc8051_err_flags)); 7541 } 7542 7543 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags) 7544 { 7545 return flag_string(buf, buf_len, flags, dc8051_info_err_flags, 7546 ARRAY_SIZE(dc8051_info_err_flags)); 7547 } 7548 7549 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags) 7550 { 7551 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags, 7552 ARRAY_SIZE(dc8051_info_host_msg_flags)); 7553 } 7554 7555 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg) 7556 { 7557 struct hfi1_pportdata *ppd = dd->pport; 7558 u64 info, err, host_msg; 7559 int queue_link_down = 0; 7560 char buf[96]; 7561 7562 /* look at the flags */ 7563 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) { 7564 /* 8051 information set by firmware */ 7565 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */ 7566 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051); 7567 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT) 7568 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK; 7569 host_msg = (info >> 7570 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT) 7571 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK; 7572 7573 /* 7574 * Handle error flags. 7575 */ 7576 if (err & FAILED_LNI) { 7577 /* 7578 * LNI error indications are cleared by the 8051 7579 * only when starting polling. Only pay attention 7580 * to them when in the states that occur during 7581 * LNI. 7582 */ 7583 if (ppd->host_link_state 7584 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 7585 queue_link_down = 1; 7586 dd_dev_info(dd, "Link error: %s\n", 7587 dc8051_info_err_string(buf, 7588 sizeof(buf), 7589 err & 7590 FAILED_LNI)); 7591 } 7592 err &= ~(u64)FAILED_LNI; 7593 } 7594 /* unknown frames can happen durning LNI, just count */ 7595 if (err & UNKNOWN_FRAME) { 7596 ppd->unknown_frame_count++; 7597 err &= ~(u64)UNKNOWN_FRAME; 7598 } 7599 if (err) { 7600 /* report remaining errors, but do not do anything */ 7601 dd_dev_err(dd, "8051 info error: %s\n", 7602 dc8051_info_err_string(buf, sizeof(buf), 7603 err)); 7604 } 7605 7606 /* 7607 * Handle host message flags. 7608 */ 7609 if (host_msg & HOST_REQ_DONE) { 7610 /* 7611 * Presently, the driver does a busy wait for 7612 * host requests to complete. This is only an 7613 * informational message. 7614 * NOTE: The 8051 clears the host message 7615 * information *on the next 8051 command*. 7616 * Therefore, when linkup is achieved, 7617 * this flag will still be set. 7618 */ 7619 host_msg &= ~(u64)HOST_REQ_DONE; 7620 } 7621 if (host_msg & BC_SMA_MSG) { 7622 queue_work(ppd->hfi1_wq, &ppd->sma_message_work); 7623 host_msg &= ~(u64)BC_SMA_MSG; 7624 } 7625 if (host_msg & LINKUP_ACHIEVED) { 7626 dd_dev_info(dd, "8051: Link up\n"); 7627 queue_work(ppd->hfi1_wq, &ppd->link_up_work); 7628 host_msg &= ~(u64)LINKUP_ACHIEVED; 7629 } 7630 if (host_msg & EXT_DEVICE_CFG_REQ) { 7631 handle_8051_request(ppd); 7632 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ; 7633 } 7634 if (host_msg & VERIFY_CAP_FRAME) { 7635 queue_work(ppd->hfi1_wq, &ppd->link_vc_work); 7636 host_msg &= ~(u64)VERIFY_CAP_FRAME; 7637 } 7638 if (host_msg & LINK_GOING_DOWN) { 7639 const char *extra = ""; 7640 /* no downgrade action needed if going down */ 7641 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7642 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7643 extra = " (ignoring downgrade)"; 7644 } 7645 dd_dev_info(dd, "8051: Link down%s\n", extra); 7646 queue_link_down = 1; 7647 host_msg &= ~(u64)LINK_GOING_DOWN; 7648 } 7649 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7650 queue_work(ppd->hfi1_wq, &ppd->link_downgrade_work); 7651 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7652 } 7653 if (host_msg) { 7654 /* report remaining messages, but do not do anything */ 7655 dd_dev_info(dd, "8051 info host message: %s\n", 7656 dc8051_info_host_msg_string(buf, 7657 sizeof(buf), 7658 host_msg)); 7659 } 7660 7661 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK; 7662 } 7663 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) { 7664 /* 7665 * Lost the 8051 heartbeat. If this happens, we 7666 * receive constant interrupts about it. Disable 7667 * the interrupt after the first. 7668 */ 7669 dd_dev_err(dd, "Lost 8051 heartbeat\n"); 7670 write_csr(dd, DC_DC8051_ERR_EN, 7671 read_csr(dd, DC_DC8051_ERR_EN) & 7672 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK); 7673 7674 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK; 7675 } 7676 if (reg) { 7677 /* report the error, but do not do anything */ 7678 dd_dev_err(dd, "8051 error: %s\n", 7679 dc8051_err_string(buf, sizeof(buf), reg)); 7680 } 7681 7682 if (queue_link_down) { 7683 /* 7684 * if the link is already going down or disabled, do not 7685 * queue another 7686 */ 7687 if ((ppd->host_link_state & 7688 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) || 7689 ppd->link_enabled == 0) { 7690 dd_dev_info(dd, "%s: not queuing link down\n", 7691 __func__); 7692 } else { 7693 queue_work(ppd->hfi1_wq, &ppd->link_down_work); 7694 } 7695 } 7696 } 7697 7698 static const char * const fm_config_txt[] = { 7699 [0] = 7700 "BadHeadDist: Distance violation between two head flits", 7701 [1] = 7702 "BadTailDist: Distance violation between two tail flits", 7703 [2] = 7704 "BadCtrlDist: Distance violation between two credit control flits", 7705 [3] = 7706 "BadCrdAck: Credits return for unsupported VL", 7707 [4] = 7708 "UnsupportedVLMarker: Received VL Marker", 7709 [5] = 7710 "BadPreempt: Exceeded the preemption nesting level", 7711 [6] = 7712 "BadControlFlit: Received unsupported control flit", 7713 /* no 7 */ 7714 [8] = 7715 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL", 7716 }; 7717 7718 static const char * const port_rcv_txt[] = { 7719 [1] = 7720 "BadPktLen: Illegal PktLen", 7721 [2] = 7722 "PktLenTooLong: Packet longer than PktLen", 7723 [3] = 7724 "PktLenTooShort: Packet shorter than PktLen", 7725 [4] = 7726 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)", 7727 [5] = 7728 "BadDLID: Illegal DLID (0, doesn't match HFI)", 7729 [6] = 7730 "BadL2: Illegal L2 opcode", 7731 [7] = 7732 "BadSC: Unsupported SC", 7733 [9] = 7734 "BadRC: Illegal RC", 7735 [11] = 7736 "PreemptError: Preempting with same VL", 7737 [12] = 7738 "PreemptVL15: Preempting a VL15 packet", 7739 }; 7740 7741 #define OPA_LDR_FMCONFIG_OFFSET 16 7742 #define OPA_LDR_PORTRCV_OFFSET 0 7743 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 7744 { 7745 u64 info, hdr0, hdr1; 7746 const char *extra; 7747 char buf[96]; 7748 struct hfi1_pportdata *ppd = dd->pport; 7749 u8 lcl_reason = 0; 7750 int do_bounce = 0; 7751 7752 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) { 7753 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) { 7754 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE); 7755 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK; 7756 /* set status bit */ 7757 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK; 7758 } 7759 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK; 7760 } 7761 7762 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) { 7763 struct hfi1_pportdata *ppd = dd->pport; 7764 /* this counter saturates at (2^32) - 1 */ 7765 if (ppd->link_downed < (u32)UINT_MAX) 7766 ppd->link_downed++; 7767 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK; 7768 } 7769 7770 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) { 7771 u8 reason_valid = 1; 7772 7773 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG); 7774 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) { 7775 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK; 7776 /* set status bit */ 7777 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK; 7778 } 7779 switch (info) { 7780 case 0: 7781 case 1: 7782 case 2: 7783 case 3: 7784 case 4: 7785 case 5: 7786 case 6: 7787 extra = fm_config_txt[info]; 7788 break; 7789 case 8: 7790 extra = fm_config_txt[info]; 7791 if (ppd->port_error_action & 7792 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) { 7793 do_bounce = 1; 7794 /* 7795 * lcl_reason cannot be derived from info 7796 * for this error 7797 */ 7798 lcl_reason = 7799 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER; 7800 } 7801 break; 7802 default: 7803 reason_valid = 0; 7804 snprintf(buf, sizeof(buf), "reserved%lld", info); 7805 extra = buf; 7806 break; 7807 } 7808 7809 if (reason_valid && !do_bounce) { 7810 do_bounce = ppd->port_error_action & 7811 (1 << (OPA_LDR_FMCONFIG_OFFSET + info)); 7812 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST; 7813 } 7814 7815 /* just report this */ 7816 dd_dev_info(dd, "DCC Error: fmconfig error: %s\n", extra); 7817 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK; 7818 } 7819 7820 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) { 7821 u8 reason_valid = 1; 7822 7823 info = read_csr(dd, DCC_ERR_INFO_PORTRCV); 7824 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0); 7825 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1); 7826 if (!(dd->err_info_rcvport.status_and_code & 7827 OPA_EI_STATUS_SMASK)) { 7828 dd->err_info_rcvport.status_and_code = 7829 info & OPA_EI_CODE_SMASK; 7830 /* set status bit */ 7831 dd->err_info_rcvport.status_and_code |= 7832 OPA_EI_STATUS_SMASK; 7833 /* 7834 * save first 2 flits in the packet that caused 7835 * the error 7836 */ 7837 dd->err_info_rcvport.packet_flit1 = hdr0; 7838 dd->err_info_rcvport.packet_flit2 = hdr1; 7839 } 7840 switch (info) { 7841 case 1: 7842 case 2: 7843 case 3: 7844 case 4: 7845 case 5: 7846 case 6: 7847 case 7: 7848 case 9: 7849 case 11: 7850 case 12: 7851 extra = port_rcv_txt[info]; 7852 break; 7853 default: 7854 reason_valid = 0; 7855 snprintf(buf, sizeof(buf), "reserved%lld", info); 7856 extra = buf; 7857 break; 7858 } 7859 7860 if (reason_valid && !do_bounce) { 7861 do_bounce = ppd->port_error_action & 7862 (1 << (OPA_LDR_PORTRCV_OFFSET + info)); 7863 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0; 7864 } 7865 7866 /* just report this */ 7867 dd_dev_info(dd, "DCC Error: PortRcv error: %s\n", extra); 7868 dd_dev_info(dd, " hdr0 0x%llx, hdr1 0x%llx\n", 7869 hdr0, hdr1); 7870 7871 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK; 7872 } 7873 7874 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) { 7875 /* informative only */ 7876 dd_dev_info(dd, "8051 access to LCB blocked\n"); 7877 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK; 7878 } 7879 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) { 7880 /* informative only */ 7881 dd_dev_info(dd, "host access to LCB blocked\n"); 7882 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK; 7883 } 7884 7885 /* report any remaining errors */ 7886 if (reg) 7887 dd_dev_info(dd, "DCC Error: %s\n", 7888 dcc_err_string(buf, sizeof(buf), reg)); 7889 7890 if (lcl_reason == 0) 7891 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN; 7892 7893 if (do_bounce) { 7894 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__); 7895 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason); 7896 queue_work(ppd->hfi1_wq, &ppd->link_bounce_work); 7897 } 7898 } 7899 7900 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 7901 { 7902 char buf[96]; 7903 7904 dd_dev_info(dd, "LCB Error: %s\n", 7905 lcb_err_string(buf, sizeof(buf), reg)); 7906 } 7907 7908 /* 7909 * CCE block DC interrupt. Source is < 8. 7910 */ 7911 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source) 7912 { 7913 const struct err_reg_info *eri = &dc_errs[source]; 7914 7915 if (eri->handler) { 7916 interrupt_clear_down(dd, 0, eri); 7917 } else if (source == 3 /* dc_lbm_int */) { 7918 /* 7919 * This indicates that a parity error has occurred on the 7920 * address/control lines presented to the LBM. The error 7921 * is a single pulse, there is no associated error flag, 7922 * and it is non-maskable. This is because if a parity 7923 * error occurs on the request the request is dropped. 7924 * This should never occur, but it is nice to know if it 7925 * ever does. 7926 */ 7927 dd_dev_err(dd, "Parity error in DC LBM block\n"); 7928 } else { 7929 dd_dev_err(dd, "Invalid DC interrupt %u\n", source); 7930 } 7931 } 7932 7933 /* 7934 * TX block send credit interrupt. Source is < 160. 7935 */ 7936 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source) 7937 { 7938 sc_group_release_update(dd, source); 7939 } 7940 7941 /* 7942 * TX block SDMA interrupt. Source is < 48. 7943 * 7944 * SDMA interrupts are grouped by type: 7945 * 7946 * 0 - N-1 = SDma 7947 * N - 2N-1 = SDmaProgress 7948 * 2N - 3N-1 = SDmaIdle 7949 */ 7950 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source) 7951 { 7952 /* what interrupt */ 7953 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 7954 /* which engine */ 7955 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 7956 7957 #ifdef CONFIG_SDMA_VERBOSITY 7958 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which, 7959 slashstrip(__FILE__), __LINE__, __func__); 7960 sdma_dumpstate(&dd->per_sdma[which]); 7961 #endif 7962 7963 if (likely(what < 3 && which < dd->num_sdma)) { 7964 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source); 7965 } else { 7966 /* should not happen */ 7967 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source); 7968 } 7969 } 7970 7971 /* 7972 * RX block receive available interrupt. Source is < 160. 7973 */ 7974 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source) 7975 { 7976 struct hfi1_ctxtdata *rcd; 7977 char *err_detail; 7978 7979 if (likely(source < dd->num_rcv_contexts)) { 7980 rcd = dd->rcd[source]; 7981 if (rcd) { 7982 if (source < dd->first_user_ctxt) 7983 rcd->do_interrupt(rcd, 0); 7984 else 7985 handle_user_interrupt(rcd); 7986 return; /* OK */ 7987 } 7988 /* received an interrupt, but no rcd */ 7989 err_detail = "dataless"; 7990 } else { 7991 /* received an interrupt, but are not using that context */ 7992 err_detail = "out of range"; 7993 } 7994 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n", 7995 err_detail, source); 7996 } 7997 7998 /* 7999 * RX block receive urgent interrupt. Source is < 160. 8000 */ 8001 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source) 8002 { 8003 struct hfi1_ctxtdata *rcd; 8004 char *err_detail; 8005 8006 if (likely(source < dd->num_rcv_contexts)) { 8007 rcd = dd->rcd[source]; 8008 if (rcd) { 8009 /* only pay attention to user urgent interrupts */ 8010 if (source >= dd->first_user_ctxt) 8011 handle_user_interrupt(rcd); 8012 return; /* OK */ 8013 } 8014 /* received an interrupt, but no rcd */ 8015 err_detail = "dataless"; 8016 } else { 8017 /* received an interrupt, but are not using that context */ 8018 err_detail = "out of range"; 8019 } 8020 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n", 8021 err_detail, source); 8022 } 8023 8024 /* 8025 * Reserved range interrupt. Should not be called in normal operation. 8026 */ 8027 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source) 8028 { 8029 char name[64]; 8030 8031 dd_dev_err(dd, "unexpected %s interrupt\n", 8032 is_reserved_name(name, sizeof(name), source)); 8033 } 8034 8035 static const struct is_table is_table[] = { 8036 /* 8037 * start end 8038 * name func interrupt func 8039 */ 8040 { IS_GENERAL_ERR_START, IS_GENERAL_ERR_END, 8041 is_misc_err_name, is_misc_err_int }, 8042 { IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END, 8043 is_sdma_eng_err_name, is_sdma_eng_err_int }, 8044 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END, 8045 is_sendctxt_err_name, is_sendctxt_err_int }, 8046 { IS_SDMA_START, IS_SDMA_END, 8047 is_sdma_eng_name, is_sdma_eng_int }, 8048 { IS_VARIOUS_START, IS_VARIOUS_END, 8049 is_various_name, is_various_int }, 8050 { IS_DC_START, IS_DC_END, 8051 is_dc_name, is_dc_int }, 8052 { IS_RCVAVAIL_START, IS_RCVAVAIL_END, 8053 is_rcv_avail_name, is_rcv_avail_int }, 8054 { IS_RCVURGENT_START, IS_RCVURGENT_END, 8055 is_rcv_urgent_name, is_rcv_urgent_int }, 8056 { IS_SENDCREDIT_START, IS_SENDCREDIT_END, 8057 is_send_credit_name, is_send_credit_int}, 8058 { IS_RESERVED_START, IS_RESERVED_END, 8059 is_reserved_name, is_reserved_int}, 8060 }; 8061 8062 /* 8063 * Interrupt source interrupt - called when the given source has an interrupt. 8064 * Source is a bit index into an array of 64-bit integers. 8065 */ 8066 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source) 8067 { 8068 const struct is_table *entry; 8069 8070 /* avoids a double compare by walking the table in-order */ 8071 for (entry = &is_table[0]; entry->is_name; entry++) { 8072 if (source < entry->end) { 8073 trace_hfi1_interrupt(dd, entry, source); 8074 entry->is_int(dd, source - entry->start); 8075 return; 8076 } 8077 } 8078 /* fell off the end */ 8079 dd_dev_err(dd, "invalid interrupt source %u\n", source); 8080 } 8081 8082 /* 8083 * General interrupt handler. This is able to correctly handle 8084 * all interrupts in case INTx is used. 8085 */ 8086 static irqreturn_t general_interrupt(int irq, void *data) 8087 { 8088 struct hfi1_devdata *dd = data; 8089 u64 regs[CCE_NUM_INT_CSRS]; 8090 u32 bit; 8091 int i; 8092 8093 this_cpu_inc(*dd->int_counter); 8094 8095 /* phase 1: scan and clear all handled interrupts */ 8096 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 8097 if (dd->gi_mask[i] == 0) { 8098 regs[i] = 0; /* used later */ 8099 continue; 8100 } 8101 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) & 8102 dd->gi_mask[i]; 8103 /* only clear if anything is set */ 8104 if (regs[i]) 8105 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]); 8106 } 8107 8108 /* phase 2: call the appropriate handler */ 8109 for_each_set_bit(bit, (unsigned long *)®s[0], 8110 CCE_NUM_INT_CSRS * 64) { 8111 is_interrupt(dd, bit); 8112 } 8113 8114 return IRQ_HANDLED; 8115 } 8116 8117 static irqreturn_t sdma_interrupt(int irq, void *data) 8118 { 8119 struct sdma_engine *sde = data; 8120 struct hfi1_devdata *dd = sde->dd; 8121 u64 status; 8122 8123 #ifdef CONFIG_SDMA_VERBOSITY 8124 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 8125 slashstrip(__FILE__), __LINE__, __func__); 8126 sdma_dumpstate(sde); 8127 #endif 8128 8129 this_cpu_inc(*dd->int_counter); 8130 8131 /* This read_csr is really bad in the hot path */ 8132 status = read_csr(dd, 8133 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64))) 8134 & sde->imask; 8135 if (likely(status)) { 8136 /* clear the interrupt(s) */ 8137 write_csr(dd, 8138 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)), 8139 status); 8140 8141 /* handle the interrupt(s) */ 8142 sdma_engine_interrupt(sde, status); 8143 } else 8144 dd_dev_err(dd, "SDMA engine %u interrupt, but no status bits set\n", 8145 sde->this_idx); 8146 8147 return IRQ_HANDLED; 8148 } 8149 8150 /* 8151 * Clear the receive interrupt. Use a read of the interrupt clear CSR 8152 * to insure that the write completed. This does NOT guarantee that 8153 * queued DMA writes to memory from the chip are pushed. 8154 */ 8155 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd) 8156 { 8157 struct hfi1_devdata *dd = rcd->dd; 8158 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg); 8159 8160 mmiowb(); /* make sure everything before is written */ 8161 write_csr(dd, addr, rcd->imask); 8162 /* force the above write on the chip and get a value back */ 8163 (void)read_csr(dd, addr); 8164 } 8165 8166 /* force the receive interrupt */ 8167 void force_recv_intr(struct hfi1_ctxtdata *rcd) 8168 { 8169 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask); 8170 } 8171 8172 /* 8173 * Return non-zero if a packet is present. 8174 * 8175 * This routine is called when rechecking for packets after the RcvAvail 8176 * interrupt has been cleared down. First, do a quick check of memory for 8177 * a packet present. If not found, use an expensive CSR read of the context 8178 * tail to determine the actual tail. The CSR read is necessary because there 8179 * is no method to push pending DMAs to memory other than an interrupt and we 8180 * are trying to determine if we need to force an interrupt. 8181 */ 8182 static inline int check_packet_present(struct hfi1_ctxtdata *rcd) 8183 { 8184 u32 tail; 8185 int present; 8186 8187 if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) 8188 present = (rcd->seq_cnt == 8189 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd)))); 8190 else /* is RDMA rtail */ 8191 present = (rcd->head != get_rcvhdrtail(rcd)); 8192 8193 if (present) 8194 return 1; 8195 8196 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */ 8197 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 8198 return rcd->head != tail; 8199 } 8200 8201 /* 8202 * Receive packet IRQ handler. This routine expects to be on its own IRQ. 8203 * This routine will try to handle packets immediately (latency), but if 8204 * it finds too many, it will invoke the thread handler (bandwitdh). The 8205 * chip receive interrupt is *not* cleared down until this or the thread (if 8206 * invoked) is finished. The intent is to avoid extra interrupts while we 8207 * are processing packets anyway. 8208 */ 8209 static irqreturn_t receive_context_interrupt(int irq, void *data) 8210 { 8211 struct hfi1_ctxtdata *rcd = data; 8212 struct hfi1_devdata *dd = rcd->dd; 8213 int disposition; 8214 int present; 8215 8216 trace_hfi1_receive_interrupt(dd, rcd->ctxt); 8217 this_cpu_inc(*dd->int_counter); 8218 aspm_ctx_disable(rcd); 8219 8220 /* receive interrupt remains blocked while processing packets */ 8221 disposition = rcd->do_interrupt(rcd, 0); 8222 8223 /* 8224 * Too many packets were seen while processing packets in this 8225 * IRQ handler. Invoke the handler thread. The receive interrupt 8226 * remains blocked. 8227 */ 8228 if (disposition == RCV_PKT_LIMIT) 8229 return IRQ_WAKE_THREAD; 8230 8231 /* 8232 * The packet processor detected no more packets. Clear the receive 8233 * interrupt and recheck for a packet packet that may have arrived 8234 * after the previous check and interrupt clear. If a packet arrived, 8235 * force another interrupt. 8236 */ 8237 clear_recv_intr(rcd); 8238 present = check_packet_present(rcd); 8239 if (present) 8240 force_recv_intr(rcd); 8241 8242 return IRQ_HANDLED; 8243 } 8244 8245 /* 8246 * Receive packet thread handler. This expects to be invoked with the 8247 * receive interrupt still blocked. 8248 */ 8249 static irqreturn_t receive_context_thread(int irq, void *data) 8250 { 8251 struct hfi1_ctxtdata *rcd = data; 8252 int present; 8253 8254 /* receive interrupt is still blocked from the IRQ handler */ 8255 (void)rcd->do_interrupt(rcd, 1); 8256 8257 /* 8258 * The packet processor will only return if it detected no more 8259 * packets. Hold IRQs here so we can safely clear the interrupt and 8260 * recheck for a packet that may have arrived after the previous 8261 * check and the interrupt clear. If a packet arrived, force another 8262 * interrupt. 8263 */ 8264 local_irq_disable(); 8265 clear_recv_intr(rcd); 8266 present = check_packet_present(rcd); 8267 if (present) 8268 force_recv_intr(rcd); 8269 local_irq_enable(); 8270 8271 return IRQ_HANDLED; 8272 } 8273 8274 /* ========================================================================= */ 8275 8276 u32 read_physical_state(struct hfi1_devdata *dd) 8277 { 8278 u64 reg; 8279 8280 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE); 8281 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT) 8282 & DC_DC8051_STS_CUR_STATE_PORT_MASK; 8283 } 8284 8285 u32 read_logical_state(struct hfi1_devdata *dd) 8286 { 8287 u64 reg; 8288 8289 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8290 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT) 8291 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK; 8292 } 8293 8294 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate) 8295 { 8296 u64 reg; 8297 8298 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8299 /* clear current state, set new state */ 8300 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK; 8301 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT; 8302 write_csr(dd, DCC_CFG_PORT_CONFIG, reg); 8303 } 8304 8305 /* 8306 * Use the 8051 to read a LCB CSR. 8307 */ 8308 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data) 8309 { 8310 u32 regno; 8311 int ret; 8312 8313 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 8314 if (acquire_lcb_access(dd, 0) == 0) { 8315 *data = read_csr(dd, addr); 8316 release_lcb_access(dd, 0); 8317 return 0; 8318 } 8319 return -EBUSY; 8320 } 8321 8322 /* register is an index of LCB registers: (offset - base) / 8 */ 8323 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8324 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data); 8325 if (ret != HCMD_SUCCESS) 8326 return -EBUSY; 8327 return 0; 8328 } 8329 8330 /* 8331 * Read an LCB CSR. Access may not be in host control, so check. 8332 * Return 0 on success, -EBUSY on failure. 8333 */ 8334 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data) 8335 { 8336 struct hfi1_pportdata *ppd = dd->pport; 8337 8338 /* if up, go through the 8051 for the value */ 8339 if (ppd->host_link_state & HLS_UP) 8340 return read_lcb_via_8051(dd, addr, data); 8341 /* if going up or down, no access */ 8342 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) 8343 return -EBUSY; 8344 /* otherwise, host has access */ 8345 *data = read_csr(dd, addr); 8346 return 0; 8347 } 8348 8349 /* 8350 * Use the 8051 to write a LCB CSR. 8351 */ 8352 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data) 8353 { 8354 u32 regno; 8355 int ret; 8356 8357 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || 8358 (dd->dc8051_ver < dc8051_ver(0, 20))) { 8359 if (acquire_lcb_access(dd, 0) == 0) { 8360 write_csr(dd, addr, data); 8361 release_lcb_access(dd, 0); 8362 return 0; 8363 } 8364 return -EBUSY; 8365 } 8366 8367 /* register is an index of LCB registers: (offset - base) / 8 */ 8368 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8369 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data); 8370 if (ret != HCMD_SUCCESS) 8371 return -EBUSY; 8372 return 0; 8373 } 8374 8375 /* 8376 * Write an LCB CSR. Access may not be in host control, so check. 8377 * Return 0 on success, -EBUSY on failure. 8378 */ 8379 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data) 8380 { 8381 struct hfi1_pportdata *ppd = dd->pport; 8382 8383 /* if up, go through the 8051 for the value */ 8384 if (ppd->host_link_state & HLS_UP) 8385 return write_lcb_via_8051(dd, addr, data); 8386 /* if going up or down, no access */ 8387 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) 8388 return -EBUSY; 8389 /* otherwise, host has access */ 8390 write_csr(dd, addr, data); 8391 return 0; 8392 } 8393 8394 /* 8395 * Returns: 8396 * < 0 = Linux error, not able to get access 8397 * > 0 = 8051 command RETURN_CODE 8398 */ 8399 static int do_8051_command( 8400 struct hfi1_devdata *dd, 8401 u32 type, 8402 u64 in_data, 8403 u64 *out_data) 8404 { 8405 u64 reg, completed; 8406 int return_code; 8407 unsigned long flags; 8408 unsigned long timeout; 8409 8410 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data); 8411 8412 /* 8413 * Alternative to holding the lock for a long time: 8414 * - keep busy wait - have other users bounce off 8415 */ 8416 spin_lock_irqsave(&dd->dc8051_lock, flags); 8417 8418 /* We can't send any commands to the 8051 if it's in reset */ 8419 if (dd->dc_shutdown) { 8420 return_code = -ENODEV; 8421 goto fail; 8422 } 8423 8424 /* 8425 * If an 8051 host command timed out previously, then the 8051 is 8426 * stuck. 8427 * 8428 * On first timeout, attempt to reset and restart the entire DC 8429 * block (including 8051). (Is this too big of a hammer?) 8430 * 8431 * If the 8051 times out a second time, the reset did not bring it 8432 * back to healthy life. In that case, fail any subsequent commands. 8433 */ 8434 if (dd->dc8051_timed_out) { 8435 if (dd->dc8051_timed_out > 1) { 8436 dd_dev_err(dd, 8437 "Previous 8051 host command timed out, skipping command %u\n", 8438 type); 8439 return_code = -ENXIO; 8440 goto fail; 8441 } 8442 spin_unlock_irqrestore(&dd->dc8051_lock, flags); 8443 dc_shutdown(dd); 8444 dc_start(dd); 8445 spin_lock_irqsave(&dd->dc8051_lock, flags); 8446 } 8447 8448 /* 8449 * If there is no timeout, then the 8051 command interface is 8450 * waiting for a command. 8451 */ 8452 8453 /* 8454 * When writing a LCB CSR, out_data contains the full value to 8455 * to be written, while in_data contains the relative LCB 8456 * address in 7:0. Do the work here, rather than the caller, 8457 * of distrubting the write data to where it needs to go: 8458 * 8459 * Write data 8460 * 39:00 -> in_data[47:8] 8461 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE 8462 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA 8463 */ 8464 if (type == HCMD_WRITE_LCB_CSR) { 8465 in_data |= ((*out_data) & 0xffffffffffull) << 8; 8466 reg = ((((*out_data) >> 40) & 0xff) << 8467 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT) 8468 | ((((*out_data) >> 48) & 0xffff) << 8469 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 8470 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg); 8471 } 8472 8473 /* 8474 * Do two writes: the first to stabilize the type and req_data, the 8475 * second to activate. 8476 */ 8477 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK) 8478 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT 8479 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK) 8480 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT; 8481 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8482 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK; 8483 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8484 8485 /* wait for completion, alternate: interrupt */ 8486 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT); 8487 while (1) { 8488 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1); 8489 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK; 8490 if (completed) 8491 break; 8492 if (time_after(jiffies, timeout)) { 8493 dd->dc8051_timed_out++; 8494 dd_dev_err(dd, "8051 host command %u timeout\n", type); 8495 if (out_data) 8496 *out_data = 0; 8497 return_code = -ETIMEDOUT; 8498 goto fail; 8499 } 8500 udelay(2); 8501 } 8502 8503 if (out_data) { 8504 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT) 8505 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK; 8506 if (type == HCMD_READ_LCB_CSR) { 8507 /* top 16 bits are in a different register */ 8508 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1) 8509 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK) 8510 << (48 8511 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT); 8512 } 8513 } 8514 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT) 8515 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK; 8516 dd->dc8051_timed_out = 0; 8517 /* 8518 * Clear command for next user. 8519 */ 8520 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0); 8521 8522 fail: 8523 spin_unlock_irqrestore(&dd->dc8051_lock, flags); 8524 8525 return return_code; 8526 } 8527 8528 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state) 8529 { 8530 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL); 8531 } 8532 8533 int load_8051_config(struct hfi1_devdata *dd, u8 field_id, 8534 u8 lane_id, u32 config_data) 8535 { 8536 u64 data; 8537 int ret; 8538 8539 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT 8540 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT 8541 | (u64)config_data << LOAD_DATA_DATA_SHIFT; 8542 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL); 8543 if (ret != HCMD_SUCCESS) { 8544 dd_dev_err(dd, 8545 "load 8051 config: field id %d, lane %d, err %d\n", 8546 (int)field_id, (int)lane_id, ret); 8547 } 8548 return ret; 8549 } 8550 8551 /* 8552 * Read the 8051 firmware "registers". Use the RAM directly. Always 8553 * set the result, even on error. 8554 * Return 0 on success, -errno on failure 8555 */ 8556 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id, 8557 u32 *result) 8558 { 8559 u64 big_data; 8560 u32 addr; 8561 int ret; 8562 8563 /* address start depends on the lane_id */ 8564 if (lane_id < 4) 8565 addr = (4 * NUM_GENERAL_FIELDS) 8566 + (lane_id * 4 * NUM_LANE_FIELDS); 8567 else 8568 addr = 0; 8569 addr += field_id * 4; 8570 8571 /* read is in 8-byte chunks, hardware will truncate the address down */ 8572 ret = read_8051_data(dd, addr, 8, &big_data); 8573 8574 if (ret == 0) { 8575 /* extract the 4 bytes we want */ 8576 if (addr & 0x4) 8577 *result = (u32)(big_data >> 32); 8578 else 8579 *result = (u32)big_data; 8580 } else { 8581 *result = 0; 8582 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n", 8583 __func__, lane_id, field_id); 8584 } 8585 8586 return ret; 8587 } 8588 8589 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management, 8590 u8 continuous) 8591 { 8592 u32 frame; 8593 8594 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT 8595 | power_management << POWER_MANAGEMENT_SHIFT; 8596 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY, 8597 GENERAL_CONFIG, frame); 8598 } 8599 8600 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu, 8601 u16 vl15buf, u8 crc_sizes) 8602 { 8603 u32 frame; 8604 8605 frame = (u32)vau << VAU_SHIFT 8606 | (u32)z << Z_SHIFT 8607 | (u32)vcu << VCU_SHIFT 8608 | (u32)vl15buf << VL15BUF_SHIFT 8609 | (u32)crc_sizes << CRC_SIZES_SHIFT; 8610 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC, 8611 GENERAL_CONFIG, frame); 8612 } 8613 8614 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits, 8615 u8 *flag_bits, u16 *link_widths) 8616 { 8617 u32 frame; 8618 8619 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG, 8620 &frame); 8621 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK; 8622 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK; 8623 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 8624 } 8625 8626 static int write_vc_local_link_width(struct hfi1_devdata *dd, 8627 u8 misc_bits, 8628 u8 flag_bits, 8629 u16 link_widths) 8630 { 8631 u32 frame; 8632 8633 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT 8634 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT 8635 | (u32)link_widths << LINK_WIDTH_SHIFT; 8636 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG, 8637 frame); 8638 } 8639 8640 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id, 8641 u8 device_rev) 8642 { 8643 u32 frame; 8644 8645 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT) 8646 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT); 8647 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame); 8648 } 8649 8650 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 8651 u8 *device_rev) 8652 { 8653 u32 frame; 8654 8655 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame); 8656 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK; 8657 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT) 8658 & REMOTE_DEVICE_REV_MASK; 8659 } 8660 8661 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_a, u8 *ver_b) 8662 { 8663 u32 frame; 8664 8665 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame); 8666 *ver_a = (frame >> STS_FM_VERSION_A_SHIFT) & STS_FM_VERSION_A_MASK; 8667 *ver_b = (frame >> STS_FM_VERSION_B_SHIFT) & STS_FM_VERSION_B_MASK; 8668 } 8669 8670 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 8671 u8 *continuous) 8672 { 8673 u32 frame; 8674 8675 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame); 8676 *power_management = (frame >> POWER_MANAGEMENT_SHIFT) 8677 & POWER_MANAGEMENT_MASK; 8678 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT) 8679 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK; 8680 } 8681 8682 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 8683 u8 *vcu, u16 *vl15buf, u8 *crc_sizes) 8684 { 8685 u32 frame; 8686 8687 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame); 8688 *vau = (frame >> VAU_SHIFT) & VAU_MASK; 8689 *z = (frame >> Z_SHIFT) & Z_MASK; 8690 *vcu = (frame >> VCU_SHIFT) & VCU_MASK; 8691 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK; 8692 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK; 8693 } 8694 8695 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 8696 u8 *remote_tx_rate, 8697 u16 *link_widths) 8698 { 8699 u32 frame; 8700 8701 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG, 8702 &frame); 8703 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT) 8704 & REMOTE_TX_RATE_MASK; 8705 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 8706 } 8707 8708 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx) 8709 { 8710 u32 frame; 8711 8712 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame); 8713 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK; 8714 } 8715 8716 static void read_mgmt_allowed(struct hfi1_devdata *dd, u8 *mgmt_allowed) 8717 { 8718 u32 frame; 8719 8720 read_8051_config(dd, REMOTE_LNI_INFO, GENERAL_CONFIG, &frame); 8721 *mgmt_allowed = (frame >> MGMT_ALLOWED_SHIFT) & MGMT_ALLOWED_MASK; 8722 } 8723 8724 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls) 8725 { 8726 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls); 8727 } 8728 8729 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs) 8730 { 8731 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs); 8732 } 8733 8734 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality) 8735 { 8736 u32 frame; 8737 int ret; 8738 8739 *link_quality = 0; 8740 if (dd->pport->host_link_state & HLS_UP) { 8741 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, 8742 &frame); 8743 if (ret == 0) 8744 *link_quality = (frame >> LINK_QUALITY_SHIFT) 8745 & LINK_QUALITY_MASK; 8746 } 8747 } 8748 8749 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc) 8750 { 8751 u32 frame; 8752 8753 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame); 8754 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK; 8755 } 8756 8757 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr) 8758 { 8759 u32 frame; 8760 8761 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame); 8762 *ldr = (frame & 0xff); 8763 } 8764 8765 static int read_tx_settings(struct hfi1_devdata *dd, 8766 u8 *enable_lane_tx, 8767 u8 *tx_polarity_inversion, 8768 u8 *rx_polarity_inversion, 8769 u8 *max_rate) 8770 { 8771 u32 frame; 8772 int ret; 8773 8774 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame); 8775 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT) 8776 & ENABLE_LANE_TX_MASK; 8777 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT) 8778 & TX_POLARITY_INVERSION_MASK; 8779 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT) 8780 & RX_POLARITY_INVERSION_MASK; 8781 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK; 8782 return ret; 8783 } 8784 8785 static int write_tx_settings(struct hfi1_devdata *dd, 8786 u8 enable_lane_tx, 8787 u8 tx_polarity_inversion, 8788 u8 rx_polarity_inversion, 8789 u8 max_rate) 8790 { 8791 u32 frame; 8792 8793 /* no need to mask, all variable sizes match field widths */ 8794 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT 8795 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT 8796 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT 8797 | max_rate << MAX_RATE_SHIFT; 8798 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame); 8799 } 8800 8801 static void check_fabric_firmware_versions(struct hfi1_devdata *dd) 8802 { 8803 u32 frame, version, prod_id; 8804 int ret, lane; 8805 8806 /* 4 lanes */ 8807 for (lane = 0; lane < 4; lane++) { 8808 ret = read_8051_config(dd, SPICO_FW_VERSION, lane, &frame); 8809 if (ret) { 8810 dd_dev_err(dd, 8811 "Unable to read lane %d firmware details\n", 8812 lane); 8813 continue; 8814 } 8815 version = (frame >> SPICO_ROM_VERSION_SHIFT) 8816 & SPICO_ROM_VERSION_MASK; 8817 prod_id = (frame >> SPICO_ROM_PROD_ID_SHIFT) 8818 & SPICO_ROM_PROD_ID_MASK; 8819 dd_dev_info(dd, 8820 "Lane %d firmware: version 0x%04x, prod_id 0x%04x\n", 8821 lane, version, prod_id); 8822 } 8823 } 8824 8825 /* 8826 * Read an idle LCB message. 8827 * 8828 * Returns 0 on success, -EINVAL on error 8829 */ 8830 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out) 8831 { 8832 int ret; 8833 8834 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out); 8835 if (ret != HCMD_SUCCESS) { 8836 dd_dev_err(dd, "read idle message: type %d, err %d\n", 8837 (u32)type, ret); 8838 return -EINVAL; 8839 } 8840 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out); 8841 /* return only the payload as we already know the type */ 8842 *data_out >>= IDLE_PAYLOAD_SHIFT; 8843 return 0; 8844 } 8845 8846 /* 8847 * Read an idle SMA message. To be done in response to a notification from 8848 * the 8051. 8849 * 8850 * Returns 0 on success, -EINVAL on error 8851 */ 8852 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data) 8853 { 8854 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT, 8855 data); 8856 } 8857 8858 /* 8859 * Send an idle LCB message. 8860 * 8861 * Returns 0 on success, -EINVAL on error 8862 */ 8863 static int send_idle_message(struct hfi1_devdata *dd, u64 data) 8864 { 8865 int ret; 8866 8867 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data); 8868 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL); 8869 if (ret != HCMD_SUCCESS) { 8870 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n", 8871 data, ret); 8872 return -EINVAL; 8873 } 8874 return 0; 8875 } 8876 8877 /* 8878 * Send an idle SMA message. 8879 * 8880 * Returns 0 on success, -EINVAL on error 8881 */ 8882 int send_idle_sma(struct hfi1_devdata *dd, u64 message) 8883 { 8884 u64 data; 8885 8886 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) | 8887 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT); 8888 return send_idle_message(dd, data); 8889 } 8890 8891 /* 8892 * Initialize the LCB then do a quick link up. This may or may not be 8893 * in loopback. 8894 * 8895 * return 0 on success, -errno on error 8896 */ 8897 static int do_quick_linkup(struct hfi1_devdata *dd) 8898 { 8899 u64 reg; 8900 unsigned long timeout; 8901 int ret; 8902 8903 lcb_shutdown(dd, 0); 8904 8905 if (loopback) { 8906 /* LCB_CFG_LOOPBACK.VAL = 2 */ 8907 /* LCB_CFG_LANE_WIDTH.VAL = 0 */ 8908 write_csr(dd, DC_LCB_CFG_LOOPBACK, 8909 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT); 8910 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 8911 } 8912 8913 /* start the LCBs */ 8914 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */ 8915 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 8916 8917 /* simulator only loopback steps */ 8918 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 8919 /* LCB_CFG_RUN.EN = 1 */ 8920 write_csr(dd, DC_LCB_CFG_RUN, 8921 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 8922 8923 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */ 8924 timeout = jiffies + msecs_to_jiffies(10); 8925 while (1) { 8926 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE); 8927 if (reg) 8928 break; 8929 if (time_after(jiffies, timeout)) { 8930 dd_dev_err(dd, 8931 "timeout waiting for LINK_TRANSFER_ACTIVE\n"); 8932 return -ETIMEDOUT; 8933 } 8934 udelay(2); 8935 } 8936 8937 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 8938 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT); 8939 } 8940 8941 if (!loopback) { 8942 /* 8943 * When doing quick linkup and not in loopback, both 8944 * sides must be done with LCB set-up before either 8945 * starts the quick linkup. Put a delay here so that 8946 * both sides can be started and have a chance to be 8947 * done with LCB set up before resuming. 8948 */ 8949 dd_dev_err(dd, 8950 "Pausing for peer to be finished with LCB set up\n"); 8951 msleep(5000); 8952 dd_dev_err(dd, "Continuing with quick linkup\n"); 8953 } 8954 8955 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 8956 set_8051_lcb_access(dd); 8957 8958 /* 8959 * State "quick" LinkUp request sets the physical link state to 8960 * LinkUp without a verify capability sequence. 8961 * This state is in simulator v37 and later. 8962 */ 8963 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP); 8964 if (ret != HCMD_SUCCESS) { 8965 dd_dev_err(dd, 8966 "%s: set physical link state to quick LinkUp failed with return %d\n", 8967 __func__, ret); 8968 8969 set_host_lcb_access(dd); 8970 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 8971 8972 if (ret >= 0) 8973 ret = -EINVAL; 8974 return ret; 8975 } 8976 8977 return 0; /* success */ 8978 } 8979 8980 /* 8981 * Set the SerDes to internal loopback mode. 8982 * Returns 0 on success, -errno on error. 8983 */ 8984 static int set_serdes_loopback_mode(struct hfi1_devdata *dd) 8985 { 8986 int ret; 8987 8988 ret = set_physical_link_state(dd, PLS_INTERNAL_SERDES_LOOPBACK); 8989 if (ret == HCMD_SUCCESS) 8990 return 0; 8991 dd_dev_err(dd, 8992 "Set physical link state to SerDes Loopback failed with return %d\n", 8993 ret); 8994 if (ret >= 0) 8995 ret = -EINVAL; 8996 return ret; 8997 } 8998 8999 /* 9000 * Do all special steps to set up loopback. 9001 */ 9002 static int init_loopback(struct hfi1_devdata *dd) 9003 { 9004 dd_dev_info(dd, "Entering loopback mode\n"); 9005 9006 /* all loopbacks should disable self GUID check */ 9007 write_csr(dd, DC_DC8051_CFG_MODE, 9008 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK)); 9009 9010 /* 9011 * The simulator has only one loopback option - LCB. Switch 9012 * to that option, which includes quick link up. 9013 * 9014 * Accept all valid loopback values. 9015 */ 9016 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) && 9017 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB || 9018 loopback == LOOPBACK_CABLE)) { 9019 loopback = LOOPBACK_LCB; 9020 quick_linkup = 1; 9021 return 0; 9022 } 9023 9024 /* handle serdes loopback */ 9025 if (loopback == LOOPBACK_SERDES) { 9026 /* internal serdes loopack needs quick linkup on RTL */ 9027 if (dd->icode == ICODE_RTL_SILICON) 9028 quick_linkup = 1; 9029 return set_serdes_loopback_mode(dd); 9030 } 9031 9032 /* LCB loopback - handled at poll time */ 9033 if (loopback == LOOPBACK_LCB) { 9034 quick_linkup = 1; /* LCB is always quick linkup */ 9035 9036 /* not supported in emulation due to emulation RTL changes */ 9037 if (dd->icode == ICODE_FPGA_EMULATION) { 9038 dd_dev_err(dd, 9039 "LCB loopback not supported in emulation\n"); 9040 return -EINVAL; 9041 } 9042 return 0; 9043 } 9044 9045 /* external cable loopback requires no extra steps */ 9046 if (loopback == LOOPBACK_CABLE) 9047 return 0; 9048 9049 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback); 9050 return -EINVAL; 9051 } 9052 9053 /* 9054 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits 9055 * used in the Verify Capability link width attribute. 9056 */ 9057 static u16 opa_to_vc_link_widths(u16 opa_widths) 9058 { 9059 int i; 9060 u16 result = 0; 9061 9062 static const struct link_bits { 9063 u16 from; 9064 u16 to; 9065 } opa_link_xlate[] = { 9066 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) }, 9067 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) }, 9068 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) }, 9069 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) }, 9070 }; 9071 9072 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) { 9073 if (opa_widths & opa_link_xlate[i].from) 9074 result |= opa_link_xlate[i].to; 9075 } 9076 return result; 9077 } 9078 9079 /* 9080 * Set link attributes before moving to polling. 9081 */ 9082 static int set_local_link_attributes(struct hfi1_pportdata *ppd) 9083 { 9084 struct hfi1_devdata *dd = ppd->dd; 9085 u8 enable_lane_tx; 9086 u8 tx_polarity_inversion; 9087 u8 rx_polarity_inversion; 9088 int ret; 9089 9090 /* reset our fabric serdes to clear any lingering problems */ 9091 fabric_serdes_reset(dd); 9092 9093 /* set the local tx rate - need to read-modify-write */ 9094 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 9095 &rx_polarity_inversion, &ppd->local_tx_rate); 9096 if (ret) 9097 goto set_local_link_attributes_fail; 9098 9099 if (dd->dc8051_ver < dc8051_ver(0, 20)) { 9100 /* set the tx rate to the fastest enabled */ 9101 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9102 ppd->local_tx_rate = 1; 9103 else 9104 ppd->local_tx_rate = 0; 9105 } else { 9106 /* set the tx rate to all enabled */ 9107 ppd->local_tx_rate = 0; 9108 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9109 ppd->local_tx_rate |= 2; 9110 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G) 9111 ppd->local_tx_rate |= 1; 9112 } 9113 9114 enable_lane_tx = 0xF; /* enable all four lanes */ 9115 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion, 9116 rx_polarity_inversion, ppd->local_tx_rate); 9117 if (ret != HCMD_SUCCESS) 9118 goto set_local_link_attributes_fail; 9119 9120 /* 9121 * DC supports continuous updates. 9122 */ 9123 ret = write_vc_local_phy(dd, 9124 0 /* no power management */, 9125 1 /* continuous updates */); 9126 if (ret != HCMD_SUCCESS) 9127 goto set_local_link_attributes_fail; 9128 9129 /* z=1 in the next call: AU of 0 is not supported by the hardware */ 9130 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init, 9131 ppd->port_crc_mode_enabled); 9132 if (ret != HCMD_SUCCESS) 9133 goto set_local_link_attributes_fail; 9134 9135 ret = write_vc_local_link_width(dd, 0, 0, 9136 opa_to_vc_link_widths( 9137 ppd->link_width_enabled)); 9138 if (ret != HCMD_SUCCESS) 9139 goto set_local_link_attributes_fail; 9140 9141 /* let peer know who we are */ 9142 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev); 9143 if (ret == HCMD_SUCCESS) 9144 return 0; 9145 9146 set_local_link_attributes_fail: 9147 dd_dev_err(dd, 9148 "Failed to set local link attributes, return 0x%x\n", 9149 ret); 9150 return ret; 9151 } 9152 9153 /* 9154 * Call this to start the link. 9155 * Do not do anything if the link is disabled. 9156 * Returns 0 if link is disabled, moved to polling, or the driver is not ready. 9157 */ 9158 int start_link(struct hfi1_pportdata *ppd) 9159 { 9160 if (!ppd->link_enabled) { 9161 dd_dev_info(ppd->dd, 9162 "%s: stopping link start because link is disabled\n", 9163 __func__); 9164 return 0; 9165 } 9166 if (!ppd->driver_link_ready) { 9167 dd_dev_info(ppd->dd, 9168 "%s: stopping link start because driver is not ready\n", 9169 __func__); 9170 return 0; 9171 } 9172 9173 /* 9174 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the 9175 * pkey table can be configured properly if the HFI unit is connected 9176 * to switch port with MgmtAllowed=NO 9177 */ 9178 clear_full_mgmt_pkey(ppd); 9179 9180 return set_link_state(ppd, HLS_DN_POLL); 9181 } 9182 9183 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd) 9184 { 9185 struct hfi1_devdata *dd = ppd->dd; 9186 u64 mask; 9187 unsigned long timeout; 9188 9189 /* 9190 * Check for QSFP interrupt for t_init (SFF 8679) 9191 */ 9192 timeout = jiffies + msecs_to_jiffies(2000); 9193 while (1) { 9194 mask = read_csr(dd, dd->hfi1_id ? 9195 ASIC_QSFP2_IN : ASIC_QSFP1_IN); 9196 if (!(mask & QSFP_HFI0_INT_N)) { 9197 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : 9198 ASIC_QSFP1_CLEAR, QSFP_HFI0_INT_N); 9199 break; 9200 } 9201 if (time_after(jiffies, timeout)) { 9202 dd_dev_info(dd, "%s: No IntN detected, reset complete\n", 9203 __func__); 9204 break; 9205 } 9206 udelay(2); 9207 } 9208 } 9209 9210 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable) 9211 { 9212 struct hfi1_devdata *dd = ppd->dd; 9213 u64 mask; 9214 9215 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK); 9216 if (enable) 9217 mask |= (u64)QSFP_HFI0_INT_N; 9218 else 9219 mask &= ~(u64)QSFP_HFI0_INT_N; 9220 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask); 9221 } 9222 9223 void reset_qsfp(struct hfi1_pportdata *ppd) 9224 { 9225 struct hfi1_devdata *dd = ppd->dd; 9226 u64 mask, qsfp_mask; 9227 9228 /* Disable INT_N from triggering QSFP interrupts */ 9229 set_qsfp_int_n(ppd, 0); 9230 9231 /* Reset the QSFP */ 9232 mask = (u64)QSFP_HFI0_RESET_N; 9233 9234 qsfp_mask = read_csr(dd, 9235 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT); 9236 qsfp_mask &= ~mask; 9237 write_csr(dd, 9238 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9239 9240 udelay(10); 9241 9242 qsfp_mask |= mask; 9243 write_csr(dd, 9244 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9245 9246 wait_for_qsfp_init(ppd); 9247 9248 /* 9249 * Allow INT_N to trigger the QSFP interrupt to watch 9250 * for alarms and warnings 9251 */ 9252 set_qsfp_int_n(ppd, 1); 9253 } 9254 9255 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd, 9256 u8 *qsfp_interrupt_status) 9257 { 9258 struct hfi1_devdata *dd = ppd->dd; 9259 9260 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) || 9261 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING)) 9262 dd_dev_info(dd, "%s: QSFP cable on fire\n", 9263 __func__); 9264 9265 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) || 9266 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING)) 9267 dd_dev_info(dd, "%s: QSFP cable temperature too low\n", 9268 __func__); 9269 9270 /* 9271 * The remaining alarms/warnings don't matter if the link is down. 9272 */ 9273 if (ppd->host_link_state & HLS_DOWN) 9274 return 0; 9275 9276 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) || 9277 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING)) 9278 dd_dev_info(dd, "%s: QSFP supply voltage too high\n", 9279 __func__); 9280 9281 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) || 9282 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING)) 9283 dd_dev_info(dd, "%s: QSFP supply voltage too low\n", 9284 __func__); 9285 9286 /* Byte 2 is vendor specific */ 9287 9288 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) || 9289 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING)) 9290 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too high\n", 9291 __func__); 9292 9293 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) || 9294 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING)) 9295 dd_dev_info(dd, "%s: Cable RX channel 1/2 power too low\n", 9296 __func__); 9297 9298 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) || 9299 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING)) 9300 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too high\n", 9301 __func__); 9302 9303 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) || 9304 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING)) 9305 dd_dev_info(dd, "%s: Cable RX channel 3/4 power too low\n", 9306 __func__); 9307 9308 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) || 9309 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING)) 9310 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too high\n", 9311 __func__); 9312 9313 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) || 9314 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING)) 9315 dd_dev_info(dd, "%s: Cable TX channel 1/2 bias too low\n", 9316 __func__); 9317 9318 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) || 9319 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING)) 9320 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too high\n", 9321 __func__); 9322 9323 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) || 9324 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING)) 9325 dd_dev_info(dd, "%s: Cable TX channel 3/4 bias too low\n", 9326 __func__); 9327 9328 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) || 9329 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING)) 9330 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too high\n", 9331 __func__); 9332 9333 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) || 9334 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING)) 9335 dd_dev_info(dd, "%s: Cable TX channel 1/2 power too low\n", 9336 __func__); 9337 9338 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) || 9339 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING)) 9340 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too high\n", 9341 __func__); 9342 9343 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) || 9344 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING)) 9345 dd_dev_info(dd, "%s: Cable TX channel 3/4 power too low\n", 9346 __func__); 9347 9348 /* Bytes 9-10 and 11-12 are reserved */ 9349 /* Bytes 13-15 are vendor specific */ 9350 9351 return 0; 9352 } 9353 9354 /* This routine will only be scheduled if the QSFP module present is asserted */ 9355 void qsfp_event(struct work_struct *work) 9356 { 9357 struct qsfp_data *qd; 9358 struct hfi1_pportdata *ppd; 9359 struct hfi1_devdata *dd; 9360 9361 qd = container_of(work, struct qsfp_data, qsfp_work); 9362 ppd = qd->ppd; 9363 dd = ppd->dd; 9364 9365 /* Sanity check */ 9366 if (!qsfp_mod_present(ppd)) 9367 return; 9368 9369 /* 9370 * Turn DC back on after cable has been re-inserted. Up until 9371 * now, the DC has been in reset to save power. 9372 */ 9373 dc_start(dd); 9374 9375 if (qd->cache_refresh_required) { 9376 set_qsfp_int_n(ppd, 0); 9377 9378 wait_for_qsfp_init(ppd); 9379 9380 /* 9381 * Allow INT_N to trigger the QSFP interrupt to watch 9382 * for alarms and warnings 9383 */ 9384 set_qsfp_int_n(ppd, 1); 9385 9386 tune_serdes(ppd); 9387 9388 start_link(ppd); 9389 } 9390 9391 if (qd->check_interrupt_flags) { 9392 u8 qsfp_interrupt_status[16] = {0,}; 9393 9394 if (one_qsfp_read(ppd, dd->hfi1_id, 6, 9395 &qsfp_interrupt_status[0], 16) != 16) { 9396 dd_dev_info(dd, 9397 "%s: Failed to read status of QSFP module\n", 9398 __func__); 9399 } else { 9400 unsigned long flags; 9401 9402 handle_qsfp_error_conditions( 9403 ppd, qsfp_interrupt_status); 9404 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 9405 ppd->qsfp_info.check_interrupt_flags = 0; 9406 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 9407 flags); 9408 } 9409 } 9410 } 9411 9412 static void init_qsfp_int(struct hfi1_devdata *dd) 9413 { 9414 struct hfi1_pportdata *ppd = dd->pport; 9415 u64 qsfp_mask, cce_int_mask; 9416 const int qsfp1_int_smask = QSFP1_INT % 64; 9417 const int qsfp2_int_smask = QSFP2_INT % 64; 9418 9419 /* 9420 * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0 9421 * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR, 9422 * therefore just one of QSFP1_INT/QSFP2_INT can be used to find 9423 * the index of the appropriate CSR in the CCEIntMask CSR array 9424 */ 9425 cce_int_mask = read_csr(dd, CCE_INT_MASK + 9426 (8 * (QSFP1_INT / 64))); 9427 if (dd->hfi1_id) { 9428 cce_int_mask &= ~((u64)1 << qsfp1_int_smask); 9429 write_csr(dd, CCE_INT_MASK + (8 * (QSFP1_INT / 64)), 9430 cce_int_mask); 9431 } else { 9432 cce_int_mask &= ~((u64)1 << qsfp2_int_smask); 9433 write_csr(dd, CCE_INT_MASK + (8 * (QSFP2_INT / 64)), 9434 cce_int_mask); 9435 } 9436 9437 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 9438 /* Clear current status to avoid spurious interrupts */ 9439 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9440 qsfp_mask); 9441 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, 9442 qsfp_mask); 9443 9444 set_qsfp_int_n(ppd, 0); 9445 9446 /* Handle active low nature of INT_N and MODPRST_N pins */ 9447 if (qsfp_mod_present(ppd)) 9448 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N; 9449 write_csr(dd, 9450 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT, 9451 qsfp_mask); 9452 } 9453 9454 /* 9455 * Do a one-time initialize of the LCB block. 9456 */ 9457 static void init_lcb(struct hfi1_devdata *dd) 9458 { 9459 /* simulator does not correctly handle LCB cclk loopback, skip */ 9460 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 9461 return; 9462 9463 /* the DC has been reset earlier in the driver load */ 9464 9465 /* set LCB for cclk loopback on the port */ 9466 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01); 9467 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00); 9468 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00); 9469 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 9470 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08); 9471 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02); 9472 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00); 9473 } 9474 9475 int bringup_serdes(struct hfi1_pportdata *ppd) 9476 { 9477 struct hfi1_devdata *dd = ppd->dd; 9478 u64 guid; 9479 int ret; 9480 9481 if (HFI1_CAP_IS_KSET(EXTENDED_PSN)) 9482 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK); 9483 9484 guid = ppd->guid; 9485 if (!guid) { 9486 if (dd->base_guid) 9487 guid = dd->base_guid + ppd->port - 1; 9488 ppd->guid = guid; 9489 } 9490 9491 /* Set linkinit_reason on power up per OPA spec */ 9492 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP; 9493 9494 /* one-time init of the LCB */ 9495 init_lcb(dd); 9496 9497 if (loopback) { 9498 ret = init_loopback(dd); 9499 if (ret < 0) 9500 return ret; 9501 } 9502 9503 get_port_type(ppd); 9504 if (ppd->port_type == PORT_TYPE_QSFP) { 9505 set_qsfp_int_n(ppd, 0); 9506 wait_for_qsfp_init(ppd); 9507 set_qsfp_int_n(ppd, 1); 9508 } 9509 9510 /* 9511 * Tune the SerDes to a ballpark setting for 9512 * optimal signal and bit error rate 9513 * Needs to be done before starting the link 9514 */ 9515 tune_serdes(ppd); 9516 9517 return start_link(ppd); 9518 } 9519 9520 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd) 9521 { 9522 struct hfi1_devdata *dd = ppd->dd; 9523 9524 /* 9525 * Shut down the link and keep it down. First turn off that the 9526 * driver wants to allow the link to be up (driver_link_ready). 9527 * Then make sure the link is not automatically restarted 9528 * (link_enabled). Cancel any pending restart. And finally 9529 * go offline. 9530 */ 9531 ppd->driver_link_ready = 0; 9532 ppd->link_enabled = 0; 9533 9534 ppd->offline_disabled_reason = 9535 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_SMA_DISABLED); 9536 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SMA_DISABLED, 0, 9537 OPA_LINKDOWN_REASON_SMA_DISABLED); 9538 set_link_state(ppd, HLS_DN_OFFLINE); 9539 9540 /* disable the port */ 9541 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 9542 } 9543 9544 static inline int init_cpu_counters(struct hfi1_devdata *dd) 9545 { 9546 struct hfi1_pportdata *ppd; 9547 int i; 9548 9549 ppd = (struct hfi1_pportdata *)(dd + 1); 9550 for (i = 0; i < dd->num_pports; i++, ppd++) { 9551 ppd->ibport_data.rvp.rc_acks = NULL; 9552 ppd->ibport_data.rvp.rc_qacks = NULL; 9553 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64); 9554 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64); 9555 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64); 9556 if (!ppd->ibport_data.rvp.rc_acks || 9557 !ppd->ibport_data.rvp.rc_delayed_comp || 9558 !ppd->ibport_data.rvp.rc_qacks) 9559 return -ENOMEM; 9560 } 9561 9562 return 0; 9563 } 9564 9565 static const char * const pt_names[] = { 9566 "expected", 9567 "eager", 9568 "invalid" 9569 }; 9570 9571 static const char *pt_name(u32 type) 9572 { 9573 return type >= ARRAY_SIZE(pt_names) ? "unknown" : pt_names[type]; 9574 } 9575 9576 /* 9577 * index is the index into the receive array 9578 */ 9579 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index, 9580 u32 type, unsigned long pa, u16 order) 9581 { 9582 u64 reg; 9583 void __iomem *base = (dd->rcvarray_wc ? dd->rcvarray_wc : 9584 (dd->kregbase + RCV_ARRAY)); 9585 9586 if (!(dd->flags & HFI1_PRESENT)) 9587 goto done; 9588 9589 if (type == PT_INVALID) { 9590 pa = 0; 9591 } else if (type > PT_INVALID) { 9592 dd_dev_err(dd, 9593 "unexpected receive array type %u for index %u, not handled\n", 9594 type, index); 9595 goto done; 9596 } 9597 9598 hfi1_cdbg(TID, "type %s, index 0x%x, pa 0x%lx, bsize 0x%lx", 9599 pt_name(type), index, pa, (unsigned long)order); 9600 9601 #define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */ 9602 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK 9603 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT 9604 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK) 9605 << RCV_ARRAY_RT_ADDR_SHIFT; 9606 writeq(reg, base + (index * 8)); 9607 9608 if (type == PT_EAGER) 9609 /* 9610 * Eager entries are written one-by-one so we have to push them 9611 * after we write the entry. 9612 */ 9613 flush_wc(); 9614 done: 9615 return; 9616 } 9617 9618 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd) 9619 { 9620 struct hfi1_devdata *dd = rcd->dd; 9621 u32 i; 9622 9623 /* this could be optimized */ 9624 for (i = rcd->eager_base; i < rcd->eager_base + 9625 rcd->egrbufs.alloced; i++) 9626 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9627 9628 for (i = rcd->expected_base; 9629 i < rcd->expected_base + rcd->expected_count; i++) 9630 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9631 } 9632 9633 int hfi1_get_base_kinfo(struct hfi1_ctxtdata *rcd, 9634 struct hfi1_ctxt_info *kinfo) 9635 { 9636 kinfo->runtime_flags = (HFI1_MISC_GET() << HFI1_CAP_USER_SHIFT) | 9637 HFI1_CAP_UGET(MASK) | HFI1_CAP_KGET(K2U); 9638 return 0; 9639 } 9640 9641 struct hfi1_message_header *hfi1_get_msgheader( 9642 struct hfi1_devdata *dd, __le32 *rhf_addr) 9643 { 9644 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr)); 9645 9646 return (struct hfi1_message_header *) 9647 (rhf_addr - dd->rhf_offset + offset); 9648 } 9649 9650 static const char * const ib_cfg_name_strings[] = { 9651 "HFI1_IB_CFG_LIDLMC", 9652 "HFI1_IB_CFG_LWID_DG_ENB", 9653 "HFI1_IB_CFG_LWID_ENB", 9654 "HFI1_IB_CFG_LWID", 9655 "HFI1_IB_CFG_SPD_ENB", 9656 "HFI1_IB_CFG_SPD", 9657 "HFI1_IB_CFG_RXPOL_ENB", 9658 "HFI1_IB_CFG_LREV_ENB", 9659 "HFI1_IB_CFG_LINKLATENCY", 9660 "HFI1_IB_CFG_HRTBT", 9661 "HFI1_IB_CFG_OP_VLS", 9662 "HFI1_IB_CFG_VL_HIGH_CAP", 9663 "HFI1_IB_CFG_VL_LOW_CAP", 9664 "HFI1_IB_CFG_OVERRUN_THRESH", 9665 "HFI1_IB_CFG_PHYERR_THRESH", 9666 "HFI1_IB_CFG_LINKDEFAULT", 9667 "HFI1_IB_CFG_PKEYS", 9668 "HFI1_IB_CFG_MTU", 9669 "HFI1_IB_CFG_LSTATE", 9670 "HFI1_IB_CFG_VL_HIGH_LIMIT", 9671 "HFI1_IB_CFG_PMA_TICKS", 9672 "HFI1_IB_CFG_PORT" 9673 }; 9674 9675 static const char *ib_cfg_name(int which) 9676 { 9677 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings)) 9678 return "invalid"; 9679 return ib_cfg_name_strings[which]; 9680 } 9681 9682 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which) 9683 { 9684 struct hfi1_devdata *dd = ppd->dd; 9685 int val = 0; 9686 9687 switch (which) { 9688 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */ 9689 val = ppd->link_width_enabled; 9690 break; 9691 case HFI1_IB_CFG_LWID: /* currently active Link-width */ 9692 val = ppd->link_width_active; 9693 break; 9694 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 9695 val = ppd->link_speed_enabled; 9696 break; 9697 case HFI1_IB_CFG_SPD: /* current Link speed */ 9698 val = ppd->link_speed_active; 9699 break; 9700 9701 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */ 9702 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */ 9703 case HFI1_IB_CFG_LINKLATENCY: 9704 goto unimplemented; 9705 9706 case HFI1_IB_CFG_OP_VLS: 9707 val = ppd->vls_operational; 9708 break; 9709 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */ 9710 val = VL_ARB_HIGH_PRIO_TABLE_SIZE; 9711 break; 9712 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */ 9713 val = VL_ARB_LOW_PRIO_TABLE_SIZE; 9714 break; 9715 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 9716 val = ppd->overrun_threshold; 9717 break; 9718 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 9719 val = ppd->phy_error_threshold; 9720 break; 9721 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 9722 val = dd->link_default; 9723 break; 9724 9725 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */ 9726 case HFI1_IB_CFG_PMA_TICKS: 9727 default: 9728 unimplemented: 9729 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 9730 dd_dev_info( 9731 dd, 9732 "%s: which %s: not implemented\n", 9733 __func__, 9734 ib_cfg_name(which)); 9735 break; 9736 } 9737 9738 return val; 9739 } 9740 9741 /* 9742 * The largest MAD packet size. 9743 */ 9744 #define MAX_MAD_PACKET 2048 9745 9746 /* 9747 * Return the maximum header bytes that can go on the _wire_ 9748 * for this device. This count includes the ICRC which is 9749 * not part of the packet held in memory but it is appended 9750 * by the HW. 9751 * This is dependent on the device's receive header entry size. 9752 * HFI allows this to be set per-receive context, but the 9753 * driver presently enforces a global value. 9754 */ 9755 u32 lrh_max_header_bytes(struct hfi1_devdata *dd) 9756 { 9757 /* 9758 * The maximum non-payload (MTU) bytes in LRH.PktLen are 9759 * the Receive Header Entry Size minus the PBC (or RHF) size 9760 * plus one DW for the ICRC appended by HW. 9761 * 9762 * dd->rcd[0].rcvhdrqentsize is in DW. 9763 * We use rcd[0] as all context will have the same value. Also, 9764 * the first kernel context would have been allocated by now so 9765 * we are guaranteed a valid value. 9766 */ 9767 return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2; 9768 } 9769 9770 /* 9771 * Set Send Length 9772 * @ppd - per port data 9773 * 9774 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck* 9775 * registers compare against LRH.PktLen, so use the max bytes included 9776 * in the LRH. 9777 * 9778 * This routine changes all VL values except VL15, which it maintains at 9779 * the same value. 9780 */ 9781 static void set_send_length(struct hfi1_pportdata *ppd) 9782 { 9783 struct hfi1_devdata *dd = ppd->dd; 9784 u32 max_hb = lrh_max_header_bytes(dd), dcmtu; 9785 u32 maxvlmtu = dd->vld[15].mtu; 9786 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2) 9787 & SEND_LEN_CHECK1_LEN_VL15_MASK) << 9788 SEND_LEN_CHECK1_LEN_VL15_SHIFT; 9789 int i, j; 9790 u32 thres; 9791 9792 for (i = 0; i < ppd->vls_supported; i++) { 9793 if (dd->vld[i].mtu > maxvlmtu) 9794 maxvlmtu = dd->vld[i].mtu; 9795 if (i <= 3) 9796 len1 |= (((dd->vld[i].mtu + max_hb) >> 2) 9797 & SEND_LEN_CHECK0_LEN_VL0_MASK) << 9798 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT); 9799 else 9800 len2 |= (((dd->vld[i].mtu + max_hb) >> 2) 9801 & SEND_LEN_CHECK1_LEN_VL4_MASK) << 9802 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT); 9803 } 9804 write_csr(dd, SEND_LEN_CHECK0, len1); 9805 write_csr(dd, SEND_LEN_CHECK1, len2); 9806 /* adjust kernel credit return thresholds based on new MTUs */ 9807 /* all kernel receive contexts have the same hdrqentsize */ 9808 for (i = 0; i < ppd->vls_supported; i++) { 9809 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50), 9810 sc_mtu_to_threshold(dd->vld[i].sc, 9811 dd->vld[i].mtu, 9812 dd->rcd[0]->rcvhdrqentsize)); 9813 for (j = 0; j < INIT_SC_PER_VL; j++) 9814 sc_set_cr_threshold( 9815 pio_select_send_context_vl(dd, j, i), 9816 thres); 9817 } 9818 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50), 9819 sc_mtu_to_threshold(dd->vld[15].sc, 9820 dd->vld[15].mtu, 9821 dd->rcd[0]->rcvhdrqentsize)); 9822 sc_set_cr_threshold(dd->vld[15].sc, thres); 9823 9824 /* Adjust maximum MTU for the port in DC */ 9825 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 : 9826 (ilog2(maxvlmtu >> 8) + 1); 9827 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG); 9828 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK; 9829 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) << 9830 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT; 9831 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1); 9832 } 9833 9834 static void set_lidlmc(struct hfi1_pportdata *ppd) 9835 { 9836 int i; 9837 u64 sreg = 0; 9838 struct hfi1_devdata *dd = ppd->dd; 9839 u32 mask = ~((1U << ppd->lmc) - 1); 9840 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1); 9841 9842 if (dd->hfi1_snoop.mode_flag) 9843 dd_dev_info(dd, "Set lid/lmc while snooping"); 9844 9845 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK 9846 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK); 9847 c1 |= ((ppd->lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK) 9848 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) | 9849 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK) 9850 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT); 9851 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1); 9852 9853 /* 9854 * Iterate over all the send contexts and set their SLID check 9855 */ 9856 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) << 9857 SEND_CTXT_CHECK_SLID_MASK_SHIFT) | 9858 (((ppd->lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) << 9859 SEND_CTXT_CHECK_SLID_VALUE_SHIFT); 9860 9861 for (i = 0; i < dd->chip_send_contexts; i++) { 9862 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x", 9863 i, (u32)sreg); 9864 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg); 9865 } 9866 9867 /* Now we have to do the same thing for the sdma engines */ 9868 sdma_update_lmc(dd, mask, ppd->lid); 9869 } 9870 9871 static int wait_phy_linkstate(struct hfi1_devdata *dd, u32 state, u32 msecs) 9872 { 9873 unsigned long timeout; 9874 u32 curr_state; 9875 9876 timeout = jiffies + msecs_to_jiffies(msecs); 9877 while (1) { 9878 curr_state = read_physical_state(dd); 9879 if (curr_state == state) 9880 break; 9881 if (time_after(jiffies, timeout)) { 9882 dd_dev_err(dd, 9883 "timeout waiting for phy link state 0x%x, current state is 0x%x\n", 9884 state, curr_state); 9885 return -ETIMEDOUT; 9886 } 9887 usleep_range(1950, 2050); /* sleep 2ms-ish */ 9888 } 9889 9890 return 0; 9891 } 9892 9893 /* 9894 * Helper for set_link_state(). Do not call except from that routine. 9895 * Expects ppd->hls_mutex to be held. 9896 * 9897 * @rem_reason value to be sent to the neighbor 9898 * 9899 * LinkDownReasons only set if transition succeeds. 9900 */ 9901 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) 9902 { 9903 struct hfi1_devdata *dd = ppd->dd; 9904 u32 pstate, previous_state; 9905 u32 last_local_state; 9906 u32 last_remote_state; 9907 int ret; 9908 int do_transition; 9909 int do_wait; 9910 9911 previous_state = ppd->host_link_state; 9912 ppd->host_link_state = HLS_GOING_OFFLINE; 9913 pstate = read_physical_state(dd); 9914 if (pstate == PLS_OFFLINE) { 9915 do_transition = 0; /* in right state */ 9916 do_wait = 0; /* ...no need to wait */ 9917 } else if ((pstate & 0xff) == PLS_OFFLINE) { 9918 do_transition = 0; /* in an offline transient state */ 9919 do_wait = 1; /* ...wait for it to settle */ 9920 } else { 9921 do_transition = 1; /* need to move to offline */ 9922 do_wait = 1; /* ...will need to wait */ 9923 } 9924 9925 if (do_transition) { 9926 ret = set_physical_link_state(dd, 9927 (rem_reason << 8) | PLS_OFFLINE); 9928 9929 if (ret != HCMD_SUCCESS) { 9930 dd_dev_err(dd, 9931 "Failed to transition to Offline link state, return %d\n", 9932 ret); 9933 return -EINVAL; 9934 } 9935 if (ppd->offline_disabled_reason == 9936 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)) 9937 ppd->offline_disabled_reason = 9938 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 9939 } 9940 9941 if (do_wait) { 9942 /* it can take a while for the link to go down */ 9943 ret = wait_phy_linkstate(dd, PLS_OFFLINE, 10000); 9944 if (ret < 0) 9945 return ret; 9946 } 9947 9948 /* make sure the logical state is also down */ 9949 wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000); 9950 9951 /* 9952 * Now in charge of LCB - must be after the physical state is 9953 * offline.quiet and before host_link_state is changed. 9954 */ 9955 set_host_lcb_access(dd); 9956 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 9957 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */ 9958 9959 if (ppd->port_type == PORT_TYPE_QSFP && 9960 ppd->qsfp_info.limiting_active && 9961 qsfp_mod_present(ppd)) { 9962 int ret; 9963 9964 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT); 9965 if (ret == 0) { 9966 set_qsfp_tx(ppd, 0); 9967 release_chip_resource(dd, qsfp_resource(dd)); 9968 } else { 9969 /* not fatal, but should warn */ 9970 dd_dev_err(dd, 9971 "Unable to acquire lock to turn off QSFP TX\n"); 9972 } 9973 } 9974 9975 /* 9976 * The LNI has a mandatory wait time after the physical state 9977 * moves to Offline.Quiet. The wait time may be different 9978 * depending on how the link went down. The 8051 firmware 9979 * will observe the needed wait time and only move to ready 9980 * when that is completed. The largest of the quiet timeouts 9981 * is 6s, so wait that long and then at least 0.5s more for 9982 * other transitions, and another 0.5s for a buffer. 9983 */ 9984 ret = wait_fm_ready(dd, 7000); 9985 if (ret) { 9986 dd_dev_err(dd, 9987 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n"); 9988 /* state is really offline, so make it so */ 9989 ppd->host_link_state = HLS_DN_OFFLINE; 9990 return ret; 9991 } 9992 9993 /* 9994 * The state is now offline and the 8051 is ready to accept host 9995 * requests. 9996 * - change our state 9997 * - notify others if we were previously in a linkup state 9998 */ 9999 ppd->host_link_state = HLS_DN_OFFLINE; 10000 if (previous_state & HLS_UP) { 10001 /* went down while link was up */ 10002 handle_linkup_change(dd, 0); 10003 } else if (previous_state 10004 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 10005 /* went down while attempting link up */ 10006 /* byte 1 of last_*_state is the failure reason */ 10007 read_last_local_state(dd, &last_local_state); 10008 read_last_remote_state(dd, &last_remote_state); 10009 dd_dev_err(dd, 10010 "LNI failure last states: local 0x%08x, remote 0x%08x\n", 10011 last_local_state, last_remote_state); 10012 } 10013 10014 /* the active link width (downgrade) is 0 on link down */ 10015 ppd->link_width_active = 0; 10016 ppd->link_width_downgrade_tx_active = 0; 10017 ppd->link_width_downgrade_rx_active = 0; 10018 ppd->current_egress_rate = 0; 10019 return 0; 10020 } 10021 10022 /* return the link state name */ 10023 static const char *link_state_name(u32 state) 10024 { 10025 const char *name; 10026 int n = ilog2(state); 10027 static const char * const names[] = { 10028 [__HLS_UP_INIT_BP] = "INIT", 10029 [__HLS_UP_ARMED_BP] = "ARMED", 10030 [__HLS_UP_ACTIVE_BP] = "ACTIVE", 10031 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF", 10032 [__HLS_DN_POLL_BP] = "POLL", 10033 [__HLS_DN_DISABLE_BP] = "DISABLE", 10034 [__HLS_DN_OFFLINE_BP] = "OFFLINE", 10035 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP", 10036 [__HLS_GOING_UP_BP] = "GOING_UP", 10037 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE", 10038 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN" 10039 }; 10040 10041 name = n < ARRAY_SIZE(names) ? names[n] : NULL; 10042 return name ? name : "unknown"; 10043 } 10044 10045 /* return the link state reason name */ 10046 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state) 10047 { 10048 if (state == HLS_UP_INIT) { 10049 switch (ppd->linkinit_reason) { 10050 case OPA_LINKINIT_REASON_LINKUP: 10051 return "(LINKUP)"; 10052 case OPA_LINKINIT_REASON_FLAPPING: 10053 return "(FLAPPING)"; 10054 case OPA_LINKINIT_OUTSIDE_POLICY: 10055 return "(OUTSIDE_POLICY)"; 10056 case OPA_LINKINIT_QUARANTINED: 10057 return "(QUARANTINED)"; 10058 case OPA_LINKINIT_INSUFIC_CAPABILITY: 10059 return "(INSUFIC_CAPABILITY)"; 10060 default: 10061 break; 10062 } 10063 } 10064 return ""; 10065 } 10066 10067 /* 10068 * driver_physical_state - convert the driver's notion of a port's 10069 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*). 10070 * Return -1 (converted to a u32) to indicate error. 10071 */ 10072 u32 driver_physical_state(struct hfi1_pportdata *ppd) 10073 { 10074 switch (ppd->host_link_state) { 10075 case HLS_UP_INIT: 10076 case HLS_UP_ARMED: 10077 case HLS_UP_ACTIVE: 10078 return IB_PORTPHYSSTATE_LINKUP; 10079 case HLS_DN_POLL: 10080 return IB_PORTPHYSSTATE_POLLING; 10081 case HLS_DN_DISABLE: 10082 return IB_PORTPHYSSTATE_DISABLED; 10083 case HLS_DN_OFFLINE: 10084 return OPA_PORTPHYSSTATE_OFFLINE; 10085 case HLS_VERIFY_CAP: 10086 return IB_PORTPHYSSTATE_POLLING; 10087 case HLS_GOING_UP: 10088 return IB_PORTPHYSSTATE_POLLING; 10089 case HLS_GOING_OFFLINE: 10090 return OPA_PORTPHYSSTATE_OFFLINE; 10091 case HLS_LINK_COOLDOWN: 10092 return OPA_PORTPHYSSTATE_OFFLINE; 10093 case HLS_DN_DOWNDEF: 10094 default: 10095 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10096 ppd->host_link_state); 10097 return -1; 10098 } 10099 } 10100 10101 /* 10102 * driver_logical_state - convert the driver's notion of a port's 10103 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1 10104 * (converted to a u32) to indicate error. 10105 */ 10106 u32 driver_logical_state(struct hfi1_pportdata *ppd) 10107 { 10108 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN)) 10109 return IB_PORT_DOWN; 10110 10111 switch (ppd->host_link_state & HLS_UP) { 10112 case HLS_UP_INIT: 10113 return IB_PORT_INIT; 10114 case HLS_UP_ARMED: 10115 return IB_PORT_ARMED; 10116 case HLS_UP_ACTIVE: 10117 return IB_PORT_ACTIVE; 10118 default: 10119 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10120 ppd->host_link_state); 10121 return -1; 10122 } 10123 } 10124 10125 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason, 10126 u8 neigh_reason, u8 rem_reason) 10127 { 10128 if (ppd->local_link_down_reason.latest == 0 && 10129 ppd->neigh_link_down_reason.latest == 0) { 10130 ppd->local_link_down_reason.latest = lcl_reason; 10131 ppd->neigh_link_down_reason.latest = neigh_reason; 10132 ppd->remote_link_down_reason = rem_reason; 10133 } 10134 } 10135 10136 /* 10137 * Change the physical and/or logical link state. 10138 * 10139 * Do not call this routine while inside an interrupt. It contains 10140 * calls to routines that can take multiple seconds to finish. 10141 * 10142 * Returns 0 on success, -errno on failure. 10143 */ 10144 int set_link_state(struct hfi1_pportdata *ppd, u32 state) 10145 { 10146 struct hfi1_devdata *dd = ppd->dd; 10147 struct ib_event event = {.device = NULL}; 10148 int ret1, ret = 0; 10149 int orig_new_state, poll_bounce; 10150 10151 mutex_lock(&ppd->hls_lock); 10152 10153 orig_new_state = state; 10154 if (state == HLS_DN_DOWNDEF) 10155 state = dd->link_default; 10156 10157 /* interpret poll -> poll as a link bounce */ 10158 poll_bounce = ppd->host_link_state == HLS_DN_POLL && 10159 state == HLS_DN_POLL; 10160 10161 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__, 10162 link_state_name(ppd->host_link_state), 10163 link_state_name(orig_new_state), 10164 poll_bounce ? "(bounce) " : "", 10165 link_state_reason_name(ppd, state)); 10166 10167 /* 10168 * If we're going to a (HLS_*) link state that implies the logical 10169 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then 10170 * reset is_sm_config_started to 0. 10171 */ 10172 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE))) 10173 ppd->is_sm_config_started = 0; 10174 10175 /* 10176 * Do nothing if the states match. Let a poll to poll link bounce 10177 * go through. 10178 */ 10179 if (ppd->host_link_state == state && !poll_bounce) 10180 goto done; 10181 10182 switch (state) { 10183 case HLS_UP_INIT: 10184 if (ppd->host_link_state == HLS_DN_POLL && 10185 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) { 10186 /* 10187 * Quick link up jumps from polling to here. 10188 * 10189 * Whether in normal or loopback mode, the 10190 * simulator jumps from polling to link up. 10191 * Accept that here. 10192 */ 10193 /* OK */ 10194 } else if (ppd->host_link_state != HLS_GOING_UP) { 10195 goto unexpected; 10196 } 10197 10198 ppd->host_link_state = HLS_UP_INIT; 10199 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000); 10200 if (ret) { 10201 /* logical state didn't change, stay at going_up */ 10202 ppd->host_link_state = HLS_GOING_UP; 10203 dd_dev_err(dd, 10204 "%s: logical state did not change to INIT\n", 10205 __func__); 10206 } else { 10207 /* clear old transient LINKINIT_REASON code */ 10208 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR) 10209 ppd->linkinit_reason = 10210 OPA_LINKINIT_REASON_LINKUP; 10211 10212 /* enable the port */ 10213 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 10214 10215 handle_linkup_change(dd, 1); 10216 } 10217 break; 10218 case HLS_UP_ARMED: 10219 if (ppd->host_link_state != HLS_UP_INIT) 10220 goto unexpected; 10221 10222 ppd->host_link_state = HLS_UP_ARMED; 10223 set_logical_state(dd, LSTATE_ARMED); 10224 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000); 10225 if (ret) { 10226 /* logical state didn't change, stay at init */ 10227 ppd->host_link_state = HLS_UP_INIT; 10228 dd_dev_err(dd, 10229 "%s: logical state did not change to ARMED\n", 10230 __func__); 10231 } 10232 /* 10233 * The simulator does not currently implement SMA messages, 10234 * so neighbor_normal is not set. Set it here when we first 10235 * move to Armed. 10236 */ 10237 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 10238 ppd->neighbor_normal = 1; 10239 break; 10240 case HLS_UP_ACTIVE: 10241 if (ppd->host_link_state != HLS_UP_ARMED) 10242 goto unexpected; 10243 10244 ppd->host_link_state = HLS_UP_ACTIVE; 10245 set_logical_state(dd, LSTATE_ACTIVE); 10246 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000); 10247 if (ret) { 10248 /* logical state didn't change, stay at armed */ 10249 ppd->host_link_state = HLS_UP_ARMED; 10250 dd_dev_err(dd, 10251 "%s: logical state did not change to ACTIVE\n", 10252 __func__); 10253 } else { 10254 /* tell all engines to go running */ 10255 sdma_all_running(dd); 10256 10257 /* Signal the IB layer that the port has went active */ 10258 event.device = &dd->verbs_dev.rdi.ibdev; 10259 event.element.port_num = ppd->port; 10260 event.event = IB_EVENT_PORT_ACTIVE; 10261 } 10262 break; 10263 case HLS_DN_POLL: 10264 if ((ppd->host_link_state == HLS_DN_DISABLE || 10265 ppd->host_link_state == HLS_DN_OFFLINE) && 10266 dd->dc_shutdown) 10267 dc_start(dd); 10268 /* Hand LED control to the DC */ 10269 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 10270 10271 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10272 u8 tmp = ppd->link_enabled; 10273 10274 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10275 if (ret) { 10276 ppd->link_enabled = tmp; 10277 break; 10278 } 10279 ppd->remote_link_down_reason = 0; 10280 10281 if (ppd->driver_link_ready) 10282 ppd->link_enabled = 1; 10283 } 10284 10285 set_all_slowpath(ppd->dd); 10286 ret = set_local_link_attributes(ppd); 10287 if (ret) 10288 break; 10289 10290 ppd->port_error_action = 0; 10291 ppd->host_link_state = HLS_DN_POLL; 10292 10293 if (quick_linkup) { 10294 /* quick linkup does not go into polling */ 10295 ret = do_quick_linkup(dd); 10296 } else { 10297 ret1 = set_physical_link_state(dd, PLS_POLLING); 10298 if (ret1 != HCMD_SUCCESS) { 10299 dd_dev_err(dd, 10300 "Failed to transition to Polling link state, return 0x%x\n", 10301 ret1); 10302 ret = -EINVAL; 10303 } 10304 } 10305 ppd->offline_disabled_reason = 10306 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE); 10307 /* 10308 * If an error occurred above, go back to offline. The 10309 * caller may reschedule another attempt. 10310 */ 10311 if (ret) 10312 goto_offline(ppd, 0); 10313 break; 10314 case HLS_DN_DISABLE: 10315 /* link is disabled */ 10316 ppd->link_enabled = 0; 10317 10318 /* allow any state to transition to disabled */ 10319 10320 /* must transition to offline first */ 10321 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10322 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10323 if (ret) 10324 break; 10325 ppd->remote_link_down_reason = 0; 10326 } 10327 10328 ret1 = set_physical_link_state(dd, PLS_DISABLED); 10329 if (ret1 != HCMD_SUCCESS) { 10330 dd_dev_err(dd, 10331 "Failed to transition to Disabled link state, return 0x%x\n", 10332 ret1); 10333 ret = -EINVAL; 10334 break; 10335 } 10336 ppd->host_link_state = HLS_DN_DISABLE; 10337 dc_shutdown(dd); 10338 break; 10339 case HLS_DN_OFFLINE: 10340 if (ppd->host_link_state == HLS_DN_DISABLE) 10341 dc_start(dd); 10342 10343 /* allow any state to transition to offline */ 10344 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10345 if (!ret) 10346 ppd->remote_link_down_reason = 0; 10347 break; 10348 case HLS_VERIFY_CAP: 10349 if (ppd->host_link_state != HLS_DN_POLL) 10350 goto unexpected; 10351 ppd->host_link_state = HLS_VERIFY_CAP; 10352 break; 10353 case HLS_GOING_UP: 10354 if (ppd->host_link_state != HLS_VERIFY_CAP) 10355 goto unexpected; 10356 10357 ret1 = set_physical_link_state(dd, PLS_LINKUP); 10358 if (ret1 != HCMD_SUCCESS) { 10359 dd_dev_err(dd, 10360 "Failed to transition to link up state, return 0x%x\n", 10361 ret1); 10362 ret = -EINVAL; 10363 break; 10364 } 10365 ppd->host_link_state = HLS_GOING_UP; 10366 break; 10367 10368 case HLS_GOING_OFFLINE: /* transient within goto_offline() */ 10369 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */ 10370 default: 10371 dd_dev_info(dd, "%s: state 0x%x: not supported\n", 10372 __func__, state); 10373 ret = -EINVAL; 10374 break; 10375 } 10376 10377 goto done; 10378 10379 unexpected: 10380 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n", 10381 __func__, link_state_name(ppd->host_link_state), 10382 link_state_name(state)); 10383 ret = -EINVAL; 10384 10385 done: 10386 mutex_unlock(&ppd->hls_lock); 10387 10388 if (event.device) 10389 ib_dispatch_event(&event); 10390 10391 return ret; 10392 } 10393 10394 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val) 10395 { 10396 u64 reg; 10397 int ret = 0; 10398 10399 switch (which) { 10400 case HFI1_IB_CFG_LIDLMC: 10401 set_lidlmc(ppd); 10402 break; 10403 case HFI1_IB_CFG_VL_HIGH_LIMIT: 10404 /* 10405 * The VL Arbitrator high limit is sent in units of 4k 10406 * bytes, while HFI stores it in units of 64 bytes. 10407 */ 10408 val *= 4096 / 64; 10409 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK) 10410 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT; 10411 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg); 10412 break; 10413 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 10414 /* HFI only supports POLL as the default link down state */ 10415 if (val != HLS_DN_POLL) 10416 ret = -EINVAL; 10417 break; 10418 case HFI1_IB_CFG_OP_VLS: 10419 if (ppd->vls_operational != val) { 10420 ppd->vls_operational = val; 10421 if (!ppd->port) 10422 ret = -EINVAL; 10423 } 10424 break; 10425 /* 10426 * For link width, link width downgrade, and speed enable, always AND 10427 * the setting with what is actually supported. This has two benefits. 10428 * First, enabled can't have unsupported values, no matter what the 10429 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean 10430 * "fill in with your supported value" have all the bits in the 10431 * field set, so simply ANDing with supported has the desired result. 10432 */ 10433 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */ 10434 ppd->link_width_enabled = val & ppd->link_width_supported; 10435 break; 10436 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */ 10437 ppd->link_width_downgrade_enabled = 10438 val & ppd->link_width_downgrade_supported; 10439 break; 10440 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 10441 ppd->link_speed_enabled = val & ppd->link_speed_supported; 10442 break; 10443 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 10444 /* 10445 * HFI does not follow IB specs, save this value 10446 * so we can report it, if asked. 10447 */ 10448 ppd->overrun_threshold = val; 10449 break; 10450 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 10451 /* 10452 * HFI does not follow IB specs, save this value 10453 * so we can report it, if asked. 10454 */ 10455 ppd->phy_error_threshold = val; 10456 break; 10457 10458 case HFI1_IB_CFG_MTU: 10459 set_send_length(ppd); 10460 break; 10461 10462 case HFI1_IB_CFG_PKEYS: 10463 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 10464 set_partition_keys(ppd); 10465 break; 10466 10467 default: 10468 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 10469 dd_dev_info(ppd->dd, 10470 "%s: which %s, val 0x%x: not implemented\n", 10471 __func__, ib_cfg_name(which), val); 10472 break; 10473 } 10474 return ret; 10475 } 10476 10477 /* begin functions related to vl arbitration table caching */ 10478 static void init_vl_arb_caches(struct hfi1_pportdata *ppd) 10479 { 10480 int i; 10481 10482 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 10483 VL_ARB_LOW_PRIO_TABLE_SIZE); 10484 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 10485 VL_ARB_HIGH_PRIO_TABLE_SIZE); 10486 10487 /* 10488 * Note that we always return values directly from the 10489 * 'vl_arb_cache' (and do no CSR reads) in response to a 10490 * 'Get(VLArbTable)'. This is obviously correct after a 10491 * 'Set(VLArbTable)', since the cache will then be up to 10492 * date. But it's also correct prior to any 'Set(VLArbTable)' 10493 * since then both the cache, and the relevant h/w registers 10494 * will be zeroed. 10495 */ 10496 10497 for (i = 0; i < MAX_PRIO_TABLE; i++) 10498 spin_lock_init(&ppd->vl_arb_cache[i].lock); 10499 } 10500 10501 /* 10502 * vl_arb_lock_cache 10503 * 10504 * All other vl_arb_* functions should be called only after locking 10505 * the cache. 10506 */ 10507 static inline struct vl_arb_cache * 10508 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx) 10509 { 10510 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE) 10511 return NULL; 10512 spin_lock(&ppd->vl_arb_cache[idx].lock); 10513 return &ppd->vl_arb_cache[idx]; 10514 } 10515 10516 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx) 10517 { 10518 spin_unlock(&ppd->vl_arb_cache[idx].lock); 10519 } 10520 10521 static void vl_arb_get_cache(struct vl_arb_cache *cache, 10522 struct ib_vl_weight_elem *vl) 10523 { 10524 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl)); 10525 } 10526 10527 static void vl_arb_set_cache(struct vl_arb_cache *cache, 10528 struct ib_vl_weight_elem *vl) 10529 { 10530 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 10531 } 10532 10533 static int vl_arb_match_cache(struct vl_arb_cache *cache, 10534 struct ib_vl_weight_elem *vl) 10535 { 10536 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 10537 } 10538 10539 /* end functions related to vl arbitration table caching */ 10540 10541 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target, 10542 u32 size, struct ib_vl_weight_elem *vl) 10543 { 10544 struct hfi1_devdata *dd = ppd->dd; 10545 u64 reg; 10546 unsigned int i, is_up = 0; 10547 int drain, ret = 0; 10548 10549 mutex_lock(&ppd->hls_lock); 10550 10551 if (ppd->host_link_state & HLS_UP) 10552 is_up = 1; 10553 10554 drain = !is_ax(dd) && is_up; 10555 10556 if (drain) 10557 /* 10558 * Before adjusting VL arbitration weights, empty per-VL 10559 * FIFOs, otherwise a packet whose VL weight is being 10560 * set to 0 could get stuck in a FIFO with no chance to 10561 * egress. 10562 */ 10563 ret = stop_drain_data_vls(dd); 10564 10565 if (ret) { 10566 dd_dev_err( 10567 dd, 10568 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n", 10569 __func__); 10570 goto err; 10571 } 10572 10573 for (i = 0; i < size; i++, vl++) { 10574 /* 10575 * NOTE: The low priority shift and mask are used here, but 10576 * they are the same for both the low and high registers. 10577 */ 10578 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK) 10579 << SEND_LOW_PRIORITY_LIST_VL_SHIFT) 10580 | (((u64)vl->weight 10581 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK) 10582 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT); 10583 write_csr(dd, target + (i * 8), reg); 10584 } 10585 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE); 10586 10587 if (drain) 10588 open_fill_data_vls(dd); /* reopen all VLs */ 10589 10590 err: 10591 mutex_unlock(&ppd->hls_lock); 10592 10593 return ret; 10594 } 10595 10596 /* 10597 * Read one credit merge VL register. 10598 */ 10599 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr, 10600 struct vl_limit *vll) 10601 { 10602 u64 reg = read_csr(dd, csr); 10603 10604 vll->dedicated = cpu_to_be16( 10605 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT) 10606 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK); 10607 vll->shared = cpu_to_be16( 10608 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT) 10609 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK); 10610 } 10611 10612 /* 10613 * Read the current credit merge limits. 10614 */ 10615 static int get_buffer_control(struct hfi1_devdata *dd, 10616 struct buffer_control *bc, u16 *overall_limit) 10617 { 10618 u64 reg; 10619 int i; 10620 10621 /* not all entries are filled in */ 10622 memset(bc, 0, sizeof(*bc)); 10623 10624 /* OPA and HFI have a 1-1 mapping */ 10625 for (i = 0; i < TXE_NUM_DATA_VL; i++) 10626 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]); 10627 10628 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */ 10629 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]); 10630 10631 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 10632 bc->overall_shared_limit = cpu_to_be16( 10633 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) 10634 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK); 10635 if (overall_limit) 10636 *overall_limit = (reg 10637 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) 10638 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK; 10639 return sizeof(struct buffer_control); 10640 } 10641 10642 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 10643 { 10644 u64 reg; 10645 int i; 10646 10647 /* each register contains 16 SC->VLnt mappings, 4 bits each */ 10648 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0); 10649 for (i = 0; i < sizeof(u64); i++) { 10650 u8 byte = *(((u8 *)®) + i); 10651 10652 dp->vlnt[2 * i] = byte & 0xf; 10653 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4; 10654 } 10655 10656 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16); 10657 for (i = 0; i < sizeof(u64); i++) { 10658 u8 byte = *(((u8 *)®) + i); 10659 10660 dp->vlnt[16 + (2 * i)] = byte & 0xf; 10661 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4; 10662 } 10663 return sizeof(struct sc2vlnt); 10664 } 10665 10666 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems, 10667 struct ib_vl_weight_elem *vl) 10668 { 10669 unsigned int i; 10670 10671 for (i = 0; i < nelems; i++, vl++) { 10672 vl->vl = 0xf; 10673 vl->weight = 0; 10674 } 10675 } 10676 10677 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 10678 { 10679 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, 10680 DC_SC_VL_VAL(15_0, 10681 0, dp->vlnt[0] & 0xf, 10682 1, dp->vlnt[1] & 0xf, 10683 2, dp->vlnt[2] & 0xf, 10684 3, dp->vlnt[3] & 0xf, 10685 4, dp->vlnt[4] & 0xf, 10686 5, dp->vlnt[5] & 0xf, 10687 6, dp->vlnt[6] & 0xf, 10688 7, dp->vlnt[7] & 0xf, 10689 8, dp->vlnt[8] & 0xf, 10690 9, dp->vlnt[9] & 0xf, 10691 10, dp->vlnt[10] & 0xf, 10692 11, dp->vlnt[11] & 0xf, 10693 12, dp->vlnt[12] & 0xf, 10694 13, dp->vlnt[13] & 0xf, 10695 14, dp->vlnt[14] & 0xf, 10696 15, dp->vlnt[15] & 0xf)); 10697 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, 10698 DC_SC_VL_VAL(31_16, 10699 16, dp->vlnt[16] & 0xf, 10700 17, dp->vlnt[17] & 0xf, 10701 18, dp->vlnt[18] & 0xf, 10702 19, dp->vlnt[19] & 0xf, 10703 20, dp->vlnt[20] & 0xf, 10704 21, dp->vlnt[21] & 0xf, 10705 22, dp->vlnt[22] & 0xf, 10706 23, dp->vlnt[23] & 0xf, 10707 24, dp->vlnt[24] & 0xf, 10708 25, dp->vlnt[25] & 0xf, 10709 26, dp->vlnt[26] & 0xf, 10710 27, dp->vlnt[27] & 0xf, 10711 28, dp->vlnt[28] & 0xf, 10712 29, dp->vlnt[29] & 0xf, 10713 30, dp->vlnt[30] & 0xf, 10714 31, dp->vlnt[31] & 0xf)); 10715 } 10716 10717 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what, 10718 u16 limit) 10719 { 10720 if (limit != 0) 10721 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n", 10722 what, (int)limit, idx); 10723 } 10724 10725 /* change only the shared limit portion of SendCmGLobalCredit */ 10726 static void set_global_shared(struct hfi1_devdata *dd, u16 limit) 10727 { 10728 u64 reg; 10729 10730 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 10731 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK; 10732 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT; 10733 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 10734 } 10735 10736 /* change only the total credit limit portion of SendCmGLobalCredit */ 10737 static void set_global_limit(struct hfi1_devdata *dd, u16 limit) 10738 { 10739 u64 reg; 10740 10741 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 10742 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK; 10743 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 10744 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 10745 } 10746 10747 /* set the given per-VL shared limit */ 10748 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit) 10749 { 10750 u64 reg; 10751 u32 addr; 10752 10753 if (vl < TXE_NUM_DATA_VL) 10754 addr = SEND_CM_CREDIT_VL + (8 * vl); 10755 else 10756 addr = SEND_CM_CREDIT_VL15; 10757 10758 reg = read_csr(dd, addr); 10759 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK; 10760 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT; 10761 write_csr(dd, addr, reg); 10762 } 10763 10764 /* set the given per-VL dedicated limit */ 10765 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit) 10766 { 10767 u64 reg; 10768 u32 addr; 10769 10770 if (vl < TXE_NUM_DATA_VL) 10771 addr = SEND_CM_CREDIT_VL + (8 * vl); 10772 else 10773 addr = SEND_CM_CREDIT_VL15; 10774 10775 reg = read_csr(dd, addr); 10776 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK; 10777 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT; 10778 write_csr(dd, addr, reg); 10779 } 10780 10781 /* spin until the given per-VL status mask bits clear */ 10782 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask, 10783 const char *which) 10784 { 10785 unsigned long timeout; 10786 u64 reg; 10787 10788 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT); 10789 while (1) { 10790 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask; 10791 10792 if (reg == 0) 10793 return; /* success */ 10794 if (time_after(jiffies, timeout)) 10795 break; /* timed out */ 10796 udelay(1); 10797 } 10798 10799 dd_dev_err(dd, 10800 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n", 10801 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg); 10802 /* 10803 * If this occurs, it is likely there was a credit loss on the link. 10804 * The only recovery from that is a link bounce. 10805 */ 10806 dd_dev_err(dd, 10807 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n"); 10808 } 10809 10810 /* 10811 * The number of credits on the VLs may be changed while everything 10812 * is "live", but the following algorithm must be followed due to 10813 * how the hardware is actually implemented. In particular, 10814 * Return_Credit_Status[] is the only correct status check. 10815 * 10816 * if (reducing Global_Shared_Credit_Limit or any shared limit changing) 10817 * set Global_Shared_Credit_Limit = 0 10818 * use_all_vl = 1 10819 * mask0 = all VLs that are changing either dedicated or shared limits 10820 * set Shared_Limit[mask0] = 0 10821 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0 10822 * if (changing any dedicated limit) 10823 * mask1 = all VLs that are lowering dedicated limits 10824 * lower Dedicated_Limit[mask1] 10825 * spin until Return_Credit_Status[mask1] == 0 10826 * raise Dedicated_Limits 10827 * raise Shared_Limits 10828 * raise Global_Shared_Credit_Limit 10829 * 10830 * lower = if the new limit is lower, set the limit to the new value 10831 * raise = if the new limit is higher than the current value (may be changed 10832 * earlier in the algorithm), set the new limit to the new value 10833 */ 10834 int set_buffer_control(struct hfi1_pportdata *ppd, 10835 struct buffer_control *new_bc) 10836 { 10837 struct hfi1_devdata *dd = ppd->dd; 10838 u64 changing_mask, ld_mask, stat_mask; 10839 int change_count; 10840 int i, use_all_mask; 10841 int this_shared_changing; 10842 int vl_count = 0, ret; 10843 /* 10844 * A0: add the variable any_shared_limit_changing below and in the 10845 * algorithm above. If removing A0 support, it can be removed. 10846 */ 10847 int any_shared_limit_changing; 10848 struct buffer_control cur_bc; 10849 u8 changing[OPA_MAX_VLS]; 10850 u8 lowering_dedicated[OPA_MAX_VLS]; 10851 u16 cur_total; 10852 u32 new_total = 0; 10853 const u64 all_mask = 10854 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK 10855 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK 10856 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK 10857 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK 10858 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK 10859 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK 10860 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK 10861 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK 10862 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK; 10863 10864 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15) 10865 #define NUM_USABLE_VLS 16 /* look at VL15 and less */ 10866 10867 /* find the new total credits, do sanity check on unused VLs */ 10868 for (i = 0; i < OPA_MAX_VLS; i++) { 10869 if (valid_vl(i)) { 10870 new_total += be16_to_cpu(new_bc->vl[i].dedicated); 10871 continue; 10872 } 10873 nonzero_msg(dd, i, "dedicated", 10874 be16_to_cpu(new_bc->vl[i].dedicated)); 10875 nonzero_msg(dd, i, "shared", 10876 be16_to_cpu(new_bc->vl[i].shared)); 10877 new_bc->vl[i].dedicated = 0; 10878 new_bc->vl[i].shared = 0; 10879 } 10880 new_total += be16_to_cpu(new_bc->overall_shared_limit); 10881 10882 /* fetch the current values */ 10883 get_buffer_control(dd, &cur_bc, &cur_total); 10884 10885 /* 10886 * Create the masks we will use. 10887 */ 10888 memset(changing, 0, sizeof(changing)); 10889 memset(lowering_dedicated, 0, sizeof(lowering_dedicated)); 10890 /* 10891 * NOTE: Assumes that the individual VL bits are adjacent and in 10892 * increasing order 10893 */ 10894 stat_mask = 10895 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK; 10896 changing_mask = 0; 10897 ld_mask = 0; 10898 change_count = 0; 10899 any_shared_limit_changing = 0; 10900 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) { 10901 if (!valid_vl(i)) 10902 continue; 10903 this_shared_changing = new_bc->vl[i].shared 10904 != cur_bc.vl[i].shared; 10905 if (this_shared_changing) 10906 any_shared_limit_changing = 1; 10907 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated || 10908 this_shared_changing) { 10909 changing[i] = 1; 10910 changing_mask |= stat_mask; 10911 change_count++; 10912 } 10913 if (be16_to_cpu(new_bc->vl[i].dedicated) < 10914 be16_to_cpu(cur_bc.vl[i].dedicated)) { 10915 lowering_dedicated[i] = 1; 10916 ld_mask |= stat_mask; 10917 } 10918 } 10919 10920 /* bracket the credit change with a total adjustment */ 10921 if (new_total > cur_total) 10922 set_global_limit(dd, new_total); 10923 10924 /* 10925 * Start the credit change algorithm. 10926 */ 10927 use_all_mask = 0; 10928 if ((be16_to_cpu(new_bc->overall_shared_limit) < 10929 be16_to_cpu(cur_bc.overall_shared_limit)) || 10930 (is_ax(dd) && any_shared_limit_changing)) { 10931 set_global_shared(dd, 0); 10932 cur_bc.overall_shared_limit = 0; 10933 use_all_mask = 1; 10934 } 10935 10936 for (i = 0; i < NUM_USABLE_VLS; i++) { 10937 if (!valid_vl(i)) 10938 continue; 10939 10940 if (changing[i]) { 10941 set_vl_shared(dd, i, 0); 10942 cur_bc.vl[i].shared = 0; 10943 } 10944 } 10945 10946 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask, 10947 "shared"); 10948 10949 if (change_count > 0) { 10950 for (i = 0; i < NUM_USABLE_VLS; i++) { 10951 if (!valid_vl(i)) 10952 continue; 10953 10954 if (lowering_dedicated[i]) { 10955 set_vl_dedicated(dd, i, 10956 be16_to_cpu(new_bc-> 10957 vl[i].dedicated)); 10958 cur_bc.vl[i].dedicated = 10959 new_bc->vl[i].dedicated; 10960 } 10961 } 10962 10963 wait_for_vl_status_clear(dd, ld_mask, "dedicated"); 10964 10965 /* now raise all dedicated that are going up */ 10966 for (i = 0; i < NUM_USABLE_VLS; i++) { 10967 if (!valid_vl(i)) 10968 continue; 10969 10970 if (be16_to_cpu(new_bc->vl[i].dedicated) > 10971 be16_to_cpu(cur_bc.vl[i].dedicated)) 10972 set_vl_dedicated(dd, i, 10973 be16_to_cpu(new_bc-> 10974 vl[i].dedicated)); 10975 } 10976 } 10977 10978 /* next raise all shared that are going up */ 10979 for (i = 0; i < NUM_USABLE_VLS; i++) { 10980 if (!valid_vl(i)) 10981 continue; 10982 10983 if (be16_to_cpu(new_bc->vl[i].shared) > 10984 be16_to_cpu(cur_bc.vl[i].shared)) 10985 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared)); 10986 } 10987 10988 /* finally raise the global shared */ 10989 if (be16_to_cpu(new_bc->overall_shared_limit) > 10990 be16_to_cpu(cur_bc.overall_shared_limit)) 10991 set_global_shared(dd, 10992 be16_to_cpu(new_bc->overall_shared_limit)); 10993 10994 /* bracket the credit change with a total adjustment */ 10995 if (new_total < cur_total) 10996 set_global_limit(dd, new_total); 10997 10998 /* 10999 * Determine the actual number of operational VLS using the number of 11000 * dedicated and shared credits for each VL. 11001 */ 11002 if (change_count > 0) { 11003 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11004 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 || 11005 be16_to_cpu(new_bc->vl[i].shared) > 0) 11006 vl_count++; 11007 ppd->actual_vls_operational = vl_count; 11008 ret = sdma_map_init(dd, ppd->port - 1, vl_count ? 11009 ppd->actual_vls_operational : 11010 ppd->vls_operational, 11011 NULL); 11012 if (ret == 0) 11013 ret = pio_map_init(dd, ppd->port - 1, vl_count ? 11014 ppd->actual_vls_operational : 11015 ppd->vls_operational, NULL); 11016 if (ret) 11017 return ret; 11018 } 11019 return 0; 11020 } 11021 11022 /* 11023 * Read the given fabric manager table. Return the size of the 11024 * table (in bytes) on success, and a negative error code on 11025 * failure. 11026 */ 11027 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t) 11028 11029 { 11030 int size; 11031 struct vl_arb_cache *vlc; 11032 11033 switch (which) { 11034 case FM_TBL_VL_HIGH_ARB: 11035 size = 256; 11036 /* 11037 * OPA specifies 128 elements (of 2 bytes each), though 11038 * HFI supports only 16 elements in h/w. 11039 */ 11040 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11041 vl_arb_get_cache(vlc, t); 11042 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11043 break; 11044 case FM_TBL_VL_LOW_ARB: 11045 size = 256; 11046 /* 11047 * OPA specifies 128 elements (of 2 bytes each), though 11048 * HFI supports only 16 elements in h/w. 11049 */ 11050 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11051 vl_arb_get_cache(vlc, t); 11052 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11053 break; 11054 case FM_TBL_BUFFER_CONTROL: 11055 size = get_buffer_control(ppd->dd, t, NULL); 11056 break; 11057 case FM_TBL_SC2VLNT: 11058 size = get_sc2vlnt(ppd->dd, t); 11059 break; 11060 case FM_TBL_VL_PREEMPT_ELEMS: 11061 size = 256; 11062 /* OPA specifies 128 elements, of 2 bytes each */ 11063 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t); 11064 break; 11065 case FM_TBL_VL_PREEMPT_MATRIX: 11066 size = 256; 11067 /* 11068 * OPA specifies that this is the same size as the VL 11069 * arbitration tables (i.e., 256 bytes). 11070 */ 11071 break; 11072 default: 11073 return -EINVAL; 11074 } 11075 return size; 11076 } 11077 11078 /* 11079 * Write the given fabric manager table. 11080 */ 11081 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t) 11082 { 11083 int ret = 0; 11084 struct vl_arb_cache *vlc; 11085 11086 switch (which) { 11087 case FM_TBL_VL_HIGH_ARB: 11088 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11089 if (vl_arb_match_cache(vlc, t)) { 11090 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11091 break; 11092 } 11093 vl_arb_set_cache(vlc, t); 11094 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11095 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST, 11096 VL_ARB_HIGH_PRIO_TABLE_SIZE, t); 11097 break; 11098 case FM_TBL_VL_LOW_ARB: 11099 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11100 if (vl_arb_match_cache(vlc, t)) { 11101 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11102 break; 11103 } 11104 vl_arb_set_cache(vlc, t); 11105 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11106 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST, 11107 VL_ARB_LOW_PRIO_TABLE_SIZE, t); 11108 break; 11109 case FM_TBL_BUFFER_CONTROL: 11110 ret = set_buffer_control(ppd, t); 11111 break; 11112 case FM_TBL_SC2VLNT: 11113 set_sc2vlnt(ppd->dd, t); 11114 break; 11115 default: 11116 ret = -EINVAL; 11117 } 11118 return ret; 11119 } 11120 11121 /* 11122 * Disable all data VLs. 11123 * 11124 * Return 0 if disabled, non-zero if the VLs cannot be disabled. 11125 */ 11126 static int disable_data_vls(struct hfi1_devdata *dd) 11127 { 11128 if (is_ax(dd)) 11129 return 1; 11130 11131 pio_send_control(dd, PSC_DATA_VL_DISABLE); 11132 11133 return 0; 11134 } 11135 11136 /* 11137 * open_fill_data_vls() - the counterpart to stop_drain_data_vls(). 11138 * Just re-enables all data VLs (the "fill" part happens 11139 * automatically - the name was chosen for symmetry with 11140 * stop_drain_data_vls()). 11141 * 11142 * Return 0 if successful, non-zero if the VLs cannot be enabled. 11143 */ 11144 int open_fill_data_vls(struct hfi1_devdata *dd) 11145 { 11146 if (is_ax(dd)) 11147 return 1; 11148 11149 pio_send_control(dd, PSC_DATA_VL_ENABLE); 11150 11151 return 0; 11152 } 11153 11154 /* 11155 * drain_data_vls() - assumes that disable_data_vls() has been called, 11156 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA 11157 * engines to drop to 0. 11158 */ 11159 static void drain_data_vls(struct hfi1_devdata *dd) 11160 { 11161 sc_wait(dd); 11162 sdma_wait(dd); 11163 pause_for_credit_return(dd); 11164 } 11165 11166 /* 11167 * stop_drain_data_vls() - disable, then drain all per-VL fifos. 11168 * 11169 * Use open_fill_data_vls() to resume using data VLs. This pair is 11170 * meant to be used like this: 11171 * 11172 * stop_drain_data_vls(dd); 11173 * // do things with per-VL resources 11174 * open_fill_data_vls(dd); 11175 */ 11176 int stop_drain_data_vls(struct hfi1_devdata *dd) 11177 { 11178 int ret; 11179 11180 ret = disable_data_vls(dd); 11181 if (ret == 0) 11182 drain_data_vls(dd); 11183 11184 return ret; 11185 } 11186 11187 /* 11188 * Convert a nanosecond time to a cclock count. No matter how slow 11189 * the cclock, a non-zero ns will always have a non-zero result. 11190 */ 11191 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns) 11192 { 11193 u32 cclocks; 11194 11195 if (dd->icode == ICODE_FPGA_EMULATION) 11196 cclocks = (ns * 1000) / FPGA_CCLOCK_PS; 11197 else /* simulation pretends to be ASIC */ 11198 cclocks = (ns * 1000) / ASIC_CCLOCK_PS; 11199 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */ 11200 cclocks = 1; 11201 return cclocks; 11202 } 11203 11204 /* 11205 * Convert a cclock count to nanoseconds. Not matter how slow 11206 * the cclock, a non-zero cclocks will always have a non-zero result. 11207 */ 11208 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks) 11209 { 11210 u32 ns; 11211 11212 if (dd->icode == ICODE_FPGA_EMULATION) 11213 ns = (cclocks * FPGA_CCLOCK_PS) / 1000; 11214 else /* simulation pretends to be ASIC */ 11215 ns = (cclocks * ASIC_CCLOCK_PS) / 1000; 11216 if (cclocks && !ns) 11217 ns = 1; 11218 return ns; 11219 } 11220 11221 /* 11222 * Dynamically adjust the receive interrupt timeout for a context based on 11223 * incoming packet rate. 11224 * 11225 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero. 11226 */ 11227 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts) 11228 { 11229 struct hfi1_devdata *dd = rcd->dd; 11230 u32 timeout = rcd->rcvavail_timeout; 11231 11232 /* 11233 * This algorithm doubles or halves the timeout depending on whether 11234 * the number of packets received in this interrupt were less than or 11235 * greater equal the interrupt count. 11236 * 11237 * The calculations below do not allow a steady state to be achieved. 11238 * Only at the endpoints it is possible to have an unchanging 11239 * timeout. 11240 */ 11241 if (npkts < rcv_intr_count) { 11242 /* 11243 * Not enough packets arrived before the timeout, adjust 11244 * timeout downward. 11245 */ 11246 if (timeout < 2) /* already at minimum? */ 11247 return; 11248 timeout >>= 1; 11249 } else { 11250 /* 11251 * More than enough packets arrived before the timeout, adjust 11252 * timeout upward. 11253 */ 11254 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */ 11255 return; 11256 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr); 11257 } 11258 11259 rcd->rcvavail_timeout = timeout; 11260 /* 11261 * timeout cannot be larger than rcv_intr_timeout_csr which has already 11262 * been verified to be in range 11263 */ 11264 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT, 11265 (u64)timeout << 11266 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 11267 } 11268 11269 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, 11270 u32 intr_adjust, u32 npkts) 11271 { 11272 struct hfi1_devdata *dd = rcd->dd; 11273 u64 reg; 11274 u32 ctxt = rcd->ctxt; 11275 11276 /* 11277 * Need to write timeout register before updating RcvHdrHead to ensure 11278 * that a new value is used when the HW decides to restart counting. 11279 */ 11280 if (intr_adjust) 11281 adjust_rcv_timeout(rcd, npkts); 11282 if (updegr) { 11283 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK) 11284 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT; 11285 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg); 11286 } 11287 mmiowb(); 11288 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) | 11289 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK) 11290 << RCV_HDR_HEAD_HEAD_SHIFT); 11291 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 11292 mmiowb(); 11293 } 11294 11295 u32 hdrqempty(struct hfi1_ctxtdata *rcd) 11296 { 11297 u32 head, tail; 11298 11299 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) 11300 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT; 11301 11302 if (rcd->rcvhdrtail_kvaddr) 11303 tail = get_rcvhdrtail(rcd); 11304 else 11305 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 11306 11307 return head == tail; 11308 } 11309 11310 /* 11311 * Context Control and Receive Array encoding for buffer size: 11312 * 0x0 invalid 11313 * 0x1 4 KB 11314 * 0x2 8 KB 11315 * 0x3 16 KB 11316 * 0x4 32 KB 11317 * 0x5 64 KB 11318 * 0x6 128 KB 11319 * 0x7 256 KB 11320 * 0x8 512 KB (Receive Array only) 11321 * 0x9 1 MB (Receive Array only) 11322 * 0xa 2 MB (Receive Array only) 11323 * 11324 * 0xB-0xF - reserved (Receive Array only) 11325 * 11326 * 11327 * This routine assumes that the value has already been sanity checked. 11328 */ 11329 static u32 encoded_size(u32 size) 11330 { 11331 switch (size) { 11332 case 4 * 1024: return 0x1; 11333 case 8 * 1024: return 0x2; 11334 case 16 * 1024: return 0x3; 11335 case 32 * 1024: return 0x4; 11336 case 64 * 1024: return 0x5; 11337 case 128 * 1024: return 0x6; 11338 case 256 * 1024: return 0x7; 11339 case 512 * 1024: return 0x8; 11340 case 1 * 1024 * 1024: return 0x9; 11341 case 2 * 1024 * 1024: return 0xa; 11342 } 11343 return 0x1; /* if invalid, go with the minimum size */ 11344 } 11345 11346 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, int ctxt) 11347 { 11348 struct hfi1_ctxtdata *rcd; 11349 u64 rcvctrl, reg; 11350 int did_enable = 0; 11351 11352 rcd = dd->rcd[ctxt]; 11353 if (!rcd) 11354 return; 11355 11356 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op); 11357 11358 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL); 11359 /* if the context already enabled, don't do the extra steps */ 11360 if ((op & HFI1_RCVCTRL_CTXT_ENB) && 11361 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) { 11362 /* reset the tail and hdr addresses, and sequence count */ 11363 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR, 11364 rcd->rcvhdrq_phys); 11365 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 11366 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11367 rcd->rcvhdrqtailaddr_phys); 11368 rcd->seq_cnt = 1; 11369 11370 /* reset the cached receive header queue head value */ 11371 rcd->head = 0; 11372 11373 /* 11374 * Zero the receive header queue so we don't get false 11375 * positives when checking the sequence number. The 11376 * sequence numbers could land exactly on the same spot. 11377 * E.g. a rcd restart before the receive header wrapped. 11378 */ 11379 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size); 11380 11381 /* starting timeout */ 11382 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr; 11383 11384 /* enable the context */ 11385 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK; 11386 11387 /* clean the egr buffer size first */ 11388 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 11389 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size) 11390 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK) 11391 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT; 11392 11393 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */ 11394 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0); 11395 did_enable = 1; 11396 11397 /* zero RcvEgrIndexHead */ 11398 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0); 11399 11400 /* set eager count and base index */ 11401 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT) 11402 & RCV_EGR_CTRL_EGR_CNT_MASK) 11403 << RCV_EGR_CTRL_EGR_CNT_SHIFT) | 11404 (((rcd->eager_base >> RCV_SHIFT) 11405 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK) 11406 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT); 11407 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg); 11408 11409 /* 11410 * Set TID (expected) count and base index. 11411 * rcd->expected_count is set to individual RcvArray entries, 11412 * not pairs, and the CSR takes a pair-count in groups of 11413 * four, so divide by 8. 11414 */ 11415 reg = (((rcd->expected_count >> RCV_SHIFT) 11416 & RCV_TID_CTRL_TID_PAIR_CNT_MASK) 11417 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) | 11418 (((rcd->expected_base >> RCV_SHIFT) 11419 & RCV_TID_CTRL_TID_BASE_INDEX_MASK) 11420 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT); 11421 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg); 11422 if (ctxt == HFI1_CTRL_CTXT) 11423 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT); 11424 } 11425 if (op & HFI1_RCVCTRL_CTXT_DIS) { 11426 write_csr(dd, RCV_VL15, 0); 11427 /* 11428 * When receive context is being disabled turn on tail 11429 * update with a dummy tail address and then disable 11430 * receive context. 11431 */ 11432 if (dd->rcvhdrtail_dummy_physaddr) { 11433 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11434 dd->rcvhdrtail_dummy_physaddr); 11435 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */ 11436 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 11437 } 11438 11439 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK; 11440 } 11441 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) 11442 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 11443 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) 11444 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 11445 if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_phys) 11446 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 11447 if (op & HFI1_RCVCTRL_TAILUPD_DIS) { 11448 /* See comment on RcvCtxtCtrl.TailUpd above */ 11449 if (!(op & HFI1_RCVCTRL_CTXT_DIS)) 11450 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK; 11451 } 11452 if (op & HFI1_RCVCTRL_TIDFLOW_ENB) 11453 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 11454 if (op & HFI1_RCVCTRL_TIDFLOW_DIS) 11455 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 11456 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) { 11457 /* 11458 * In one-packet-per-eager mode, the size comes from 11459 * the RcvArray entry. 11460 */ 11461 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 11462 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 11463 } 11464 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS) 11465 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 11466 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB) 11467 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 11468 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS) 11469 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 11470 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB) 11471 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 11472 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS) 11473 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 11474 rcd->rcvctrl = rcvctrl; 11475 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl); 11476 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl); 11477 11478 /* work around sticky RcvCtxtStatus.BlockedRHQFull */ 11479 if (did_enable && 11480 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) { 11481 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 11482 if (reg != 0) { 11483 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n", 11484 ctxt, reg); 11485 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 11486 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10); 11487 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00); 11488 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 11489 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 11490 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n", 11491 ctxt, reg, reg == 0 ? "not" : "still"); 11492 } 11493 } 11494 11495 if (did_enable) { 11496 /* 11497 * The interrupt timeout and count must be set after 11498 * the context is enabled to take effect. 11499 */ 11500 /* set interrupt timeout */ 11501 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT, 11502 (u64)rcd->rcvavail_timeout << 11503 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 11504 11505 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */ 11506 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT; 11507 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 11508 } 11509 11510 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS)) 11511 /* 11512 * If the context has been disabled and the Tail Update has 11513 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address 11514 * so it doesn't contain an address that is invalid. 11515 */ 11516 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11517 dd->rcvhdrtail_dummy_physaddr); 11518 } 11519 11520 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp) 11521 { 11522 int ret; 11523 u64 val = 0; 11524 11525 if (namep) { 11526 ret = dd->cntrnameslen; 11527 *namep = dd->cntrnames; 11528 } else { 11529 const struct cntr_entry *entry; 11530 int i, j; 11531 11532 ret = (dd->ndevcntrs) * sizeof(u64); 11533 11534 /* Get the start of the block of counters */ 11535 *cntrp = dd->cntrs; 11536 11537 /* 11538 * Now go and fill in each counter in the block. 11539 */ 11540 for (i = 0; i < DEV_CNTR_LAST; i++) { 11541 entry = &dev_cntrs[i]; 11542 hfi1_cdbg(CNTR, "reading %s", entry->name); 11543 if (entry->flags & CNTR_DISABLED) { 11544 /* Nothing */ 11545 hfi1_cdbg(CNTR, "\tDisabled\n"); 11546 } else { 11547 if (entry->flags & CNTR_VL) { 11548 hfi1_cdbg(CNTR, "\tPer VL\n"); 11549 for (j = 0; j < C_VL_COUNT; j++) { 11550 val = entry->rw_cntr(entry, 11551 dd, j, 11552 CNTR_MODE_R, 11553 0); 11554 hfi1_cdbg( 11555 CNTR, 11556 "\t\tRead 0x%llx for %d\n", 11557 val, j); 11558 dd->cntrs[entry->offset + j] = 11559 val; 11560 } 11561 } else if (entry->flags & CNTR_SDMA) { 11562 hfi1_cdbg(CNTR, 11563 "\t Per SDMA Engine\n"); 11564 for (j = 0; j < dd->chip_sdma_engines; 11565 j++) { 11566 val = 11567 entry->rw_cntr(entry, dd, j, 11568 CNTR_MODE_R, 0); 11569 hfi1_cdbg(CNTR, 11570 "\t\tRead 0x%llx for %d\n", 11571 val, j); 11572 dd->cntrs[entry->offset + j] = 11573 val; 11574 } 11575 } else { 11576 val = entry->rw_cntr(entry, dd, 11577 CNTR_INVALID_VL, 11578 CNTR_MODE_R, 0); 11579 dd->cntrs[entry->offset] = val; 11580 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 11581 } 11582 } 11583 } 11584 } 11585 return ret; 11586 } 11587 11588 /* 11589 * Used by sysfs to create files for hfi stats to read 11590 */ 11591 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp) 11592 { 11593 int ret; 11594 u64 val = 0; 11595 11596 if (namep) { 11597 ret = ppd->dd->portcntrnameslen; 11598 *namep = ppd->dd->portcntrnames; 11599 } else { 11600 const struct cntr_entry *entry; 11601 int i, j; 11602 11603 ret = ppd->dd->nportcntrs * sizeof(u64); 11604 *cntrp = ppd->cntrs; 11605 11606 for (i = 0; i < PORT_CNTR_LAST; i++) { 11607 entry = &port_cntrs[i]; 11608 hfi1_cdbg(CNTR, "reading %s", entry->name); 11609 if (entry->flags & CNTR_DISABLED) { 11610 /* Nothing */ 11611 hfi1_cdbg(CNTR, "\tDisabled\n"); 11612 continue; 11613 } 11614 11615 if (entry->flags & CNTR_VL) { 11616 hfi1_cdbg(CNTR, "\tPer VL"); 11617 for (j = 0; j < C_VL_COUNT; j++) { 11618 val = entry->rw_cntr(entry, ppd, j, 11619 CNTR_MODE_R, 11620 0); 11621 hfi1_cdbg( 11622 CNTR, 11623 "\t\tRead 0x%llx for %d", 11624 val, j); 11625 ppd->cntrs[entry->offset + j] = val; 11626 } 11627 } else { 11628 val = entry->rw_cntr(entry, ppd, 11629 CNTR_INVALID_VL, 11630 CNTR_MODE_R, 11631 0); 11632 ppd->cntrs[entry->offset] = val; 11633 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 11634 } 11635 } 11636 } 11637 return ret; 11638 } 11639 11640 static void free_cntrs(struct hfi1_devdata *dd) 11641 { 11642 struct hfi1_pportdata *ppd; 11643 int i; 11644 11645 if (dd->synth_stats_timer.data) 11646 del_timer_sync(&dd->synth_stats_timer); 11647 dd->synth_stats_timer.data = 0; 11648 ppd = (struct hfi1_pportdata *)(dd + 1); 11649 for (i = 0; i < dd->num_pports; i++, ppd++) { 11650 kfree(ppd->cntrs); 11651 kfree(ppd->scntrs); 11652 free_percpu(ppd->ibport_data.rvp.rc_acks); 11653 free_percpu(ppd->ibport_data.rvp.rc_qacks); 11654 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp); 11655 ppd->cntrs = NULL; 11656 ppd->scntrs = NULL; 11657 ppd->ibport_data.rvp.rc_acks = NULL; 11658 ppd->ibport_data.rvp.rc_qacks = NULL; 11659 ppd->ibport_data.rvp.rc_delayed_comp = NULL; 11660 } 11661 kfree(dd->portcntrnames); 11662 dd->portcntrnames = NULL; 11663 kfree(dd->cntrs); 11664 dd->cntrs = NULL; 11665 kfree(dd->scntrs); 11666 dd->scntrs = NULL; 11667 kfree(dd->cntrnames); 11668 dd->cntrnames = NULL; 11669 } 11670 11671 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL 11672 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF 11673 11674 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry, 11675 u64 *psval, void *context, int vl) 11676 { 11677 u64 val; 11678 u64 sval = *psval; 11679 11680 if (entry->flags & CNTR_DISABLED) { 11681 dd_dev_err(dd, "Counter %s not enabled", entry->name); 11682 return 0; 11683 } 11684 11685 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 11686 11687 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0); 11688 11689 /* If its a synthetic counter there is more work we need to do */ 11690 if (entry->flags & CNTR_SYNTH) { 11691 if (sval == CNTR_MAX) { 11692 /* No need to read already saturated */ 11693 return CNTR_MAX; 11694 } 11695 11696 if (entry->flags & CNTR_32BIT) { 11697 /* 32bit counters can wrap multiple times */ 11698 u64 upper = sval >> 32; 11699 u64 lower = (sval << 32) >> 32; 11700 11701 if (lower > val) { /* hw wrapped */ 11702 if (upper == CNTR_32BIT_MAX) 11703 val = CNTR_MAX; 11704 else 11705 upper++; 11706 } 11707 11708 if (val != CNTR_MAX) 11709 val = (upper << 32) | val; 11710 11711 } else { 11712 /* If we rolled we are saturated */ 11713 if ((val < sval) || (val > CNTR_MAX)) 11714 val = CNTR_MAX; 11715 } 11716 } 11717 11718 *psval = val; 11719 11720 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 11721 11722 return val; 11723 } 11724 11725 static u64 write_dev_port_cntr(struct hfi1_devdata *dd, 11726 struct cntr_entry *entry, 11727 u64 *psval, void *context, int vl, u64 data) 11728 { 11729 u64 val; 11730 11731 if (entry->flags & CNTR_DISABLED) { 11732 dd_dev_err(dd, "Counter %s not enabled", entry->name); 11733 return 0; 11734 } 11735 11736 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 11737 11738 if (entry->flags & CNTR_SYNTH) { 11739 *psval = data; 11740 if (entry->flags & CNTR_32BIT) { 11741 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 11742 (data << 32) >> 32); 11743 val = data; /* return the full 64bit value */ 11744 } else { 11745 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 11746 data); 11747 } 11748 } else { 11749 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data); 11750 } 11751 11752 *psval = val; 11753 11754 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 11755 11756 return val; 11757 } 11758 11759 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl) 11760 { 11761 struct cntr_entry *entry; 11762 u64 *sval; 11763 11764 entry = &dev_cntrs[index]; 11765 sval = dd->scntrs + entry->offset; 11766 11767 if (vl != CNTR_INVALID_VL) 11768 sval += vl; 11769 11770 return read_dev_port_cntr(dd, entry, sval, dd, vl); 11771 } 11772 11773 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data) 11774 { 11775 struct cntr_entry *entry; 11776 u64 *sval; 11777 11778 entry = &dev_cntrs[index]; 11779 sval = dd->scntrs + entry->offset; 11780 11781 if (vl != CNTR_INVALID_VL) 11782 sval += vl; 11783 11784 return write_dev_port_cntr(dd, entry, sval, dd, vl, data); 11785 } 11786 11787 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl) 11788 { 11789 struct cntr_entry *entry; 11790 u64 *sval; 11791 11792 entry = &port_cntrs[index]; 11793 sval = ppd->scntrs + entry->offset; 11794 11795 if (vl != CNTR_INVALID_VL) 11796 sval += vl; 11797 11798 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 11799 (index <= C_RCV_HDR_OVF_LAST)) { 11800 /* We do not want to bother for disabled contexts */ 11801 return 0; 11802 } 11803 11804 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl); 11805 } 11806 11807 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data) 11808 { 11809 struct cntr_entry *entry; 11810 u64 *sval; 11811 11812 entry = &port_cntrs[index]; 11813 sval = ppd->scntrs + entry->offset; 11814 11815 if (vl != CNTR_INVALID_VL) 11816 sval += vl; 11817 11818 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 11819 (index <= C_RCV_HDR_OVF_LAST)) { 11820 /* We do not want to bother for disabled contexts */ 11821 return 0; 11822 } 11823 11824 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data); 11825 } 11826 11827 static void update_synth_timer(unsigned long opaque) 11828 { 11829 u64 cur_tx; 11830 u64 cur_rx; 11831 u64 total_flits; 11832 u8 update = 0; 11833 int i, j, vl; 11834 struct hfi1_pportdata *ppd; 11835 struct cntr_entry *entry; 11836 11837 struct hfi1_devdata *dd = (struct hfi1_devdata *)opaque; 11838 11839 /* 11840 * Rather than keep beating on the CSRs pick a minimal set that we can 11841 * check to watch for potential roll over. We can do this by looking at 11842 * the number of flits sent/recv. If the total flits exceeds 32bits then 11843 * we have to iterate all the counters and update. 11844 */ 11845 entry = &dev_cntrs[C_DC_RCV_FLITS]; 11846 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 11847 11848 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 11849 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 11850 11851 hfi1_cdbg( 11852 CNTR, 11853 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n", 11854 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx); 11855 11856 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) { 11857 /* 11858 * May not be strictly necessary to update but it won't hurt and 11859 * simplifies the logic here. 11860 */ 11861 update = 1; 11862 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating", 11863 dd->unit); 11864 } else { 11865 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx); 11866 hfi1_cdbg(CNTR, 11867 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit, 11868 total_flits, (u64)CNTR_32BIT_MAX); 11869 if (total_flits >= CNTR_32BIT_MAX) { 11870 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating", 11871 dd->unit); 11872 update = 1; 11873 } 11874 } 11875 11876 if (update) { 11877 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit); 11878 for (i = 0; i < DEV_CNTR_LAST; i++) { 11879 entry = &dev_cntrs[i]; 11880 if (entry->flags & CNTR_VL) { 11881 for (vl = 0; vl < C_VL_COUNT; vl++) 11882 read_dev_cntr(dd, i, vl); 11883 } else { 11884 read_dev_cntr(dd, i, CNTR_INVALID_VL); 11885 } 11886 } 11887 ppd = (struct hfi1_pportdata *)(dd + 1); 11888 for (i = 0; i < dd->num_pports; i++, ppd++) { 11889 for (j = 0; j < PORT_CNTR_LAST; j++) { 11890 entry = &port_cntrs[j]; 11891 if (entry->flags & CNTR_VL) { 11892 for (vl = 0; vl < C_VL_COUNT; vl++) 11893 read_port_cntr(ppd, j, vl); 11894 } else { 11895 read_port_cntr(ppd, j, CNTR_INVALID_VL); 11896 } 11897 } 11898 } 11899 11900 /* 11901 * We want the value in the register. The goal is to keep track 11902 * of the number of "ticks" not the counter value. In other 11903 * words if the register rolls we want to notice it and go ahead 11904 * and force an update. 11905 */ 11906 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 11907 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 11908 CNTR_MODE_R, 0); 11909 11910 entry = &dev_cntrs[C_DC_RCV_FLITS]; 11911 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 11912 CNTR_MODE_R, 0); 11913 11914 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx", 11915 dd->unit, dd->last_tx, dd->last_rx); 11916 11917 } else { 11918 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit); 11919 } 11920 11921 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 11922 } 11923 11924 #define C_MAX_NAME 13 /* 12 chars + one for /0 */ 11925 static int init_cntrs(struct hfi1_devdata *dd) 11926 { 11927 int i, rcv_ctxts, j; 11928 size_t sz; 11929 char *p; 11930 char name[C_MAX_NAME]; 11931 struct hfi1_pportdata *ppd; 11932 const char *bit_type_32 = ",32"; 11933 const int bit_type_32_sz = strlen(bit_type_32); 11934 11935 /* set up the stats timer; the add_timer is done at the end */ 11936 setup_timer(&dd->synth_stats_timer, update_synth_timer, 11937 (unsigned long)dd); 11938 11939 /***********************/ 11940 /* per device counters */ 11941 /***********************/ 11942 11943 /* size names and determine how many we have*/ 11944 dd->ndevcntrs = 0; 11945 sz = 0; 11946 11947 for (i = 0; i < DEV_CNTR_LAST; i++) { 11948 if (dev_cntrs[i].flags & CNTR_DISABLED) { 11949 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name); 11950 continue; 11951 } 11952 11953 if (dev_cntrs[i].flags & CNTR_VL) { 11954 dev_cntrs[i].offset = dd->ndevcntrs; 11955 for (j = 0; j < C_VL_COUNT; j++) { 11956 snprintf(name, C_MAX_NAME, "%s%d", 11957 dev_cntrs[i].name, vl_from_idx(j)); 11958 sz += strlen(name); 11959 /* Add ",32" for 32-bit counters */ 11960 if (dev_cntrs[i].flags & CNTR_32BIT) 11961 sz += bit_type_32_sz; 11962 sz++; 11963 dd->ndevcntrs++; 11964 } 11965 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 11966 dev_cntrs[i].offset = dd->ndevcntrs; 11967 for (j = 0; j < dd->chip_sdma_engines; j++) { 11968 snprintf(name, C_MAX_NAME, "%s%d", 11969 dev_cntrs[i].name, j); 11970 sz += strlen(name); 11971 /* Add ",32" for 32-bit counters */ 11972 if (dev_cntrs[i].flags & CNTR_32BIT) 11973 sz += bit_type_32_sz; 11974 sz++; 11975 dd->ndevcntrs++; 11976 } 11977 } else { 11978 /* +1 for newline. */ 11979 sz += strlen(dev_cntrs[i].name) + 1; 11980 /* Add ",32" for 32-bit counters */ 11981 if (dev_cntrs[i].flags & CNTR_32BIT) 11982 sz += bit_type_32_sz; 11983 dev_cntrs[i].offset = dd->ndevcntrs; 11984 dd->ndevcntrs++; 11985 } 11986 } 11987 11988 /* allocate space for the counter values */ 11989 dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL); 11990 if (!dd->cntrs) 11991 goto bail; 11992 11993 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL); 11994 if (!dd->scntrs) 11995 goto bail; 11996 11997 /* allocate space for the counter names */ 11998 dd->cntrnameslen = sz; 11999 dd->cntrnames = kmalloc(sz, GFP_KERNEL); 12000 if (!dd->cntrnames) 12001 goto bail; 12002 12003 /* fill in the names */ 12004 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) { 12005 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12006 /* Nothing */ 12007 } else if (dev_cntrs[i].flags & CNTR_VL) { 12008 for (j = 0; j < C_VL_COUNT; j++) { 12009 snprintf(name, C_MAX_NAME, "%s%d", 12010 dev_cntrs[i].name, 12011 vl_from_idx(j)); 12012 memcpy(p, name, strlen(name)); 12013 p += strlen(name); 12014 12015 /* Counter is 32 bits */ 12016 if (dev_cntrs[i].flags & CNTR_32BIT) { 12017 memcpy(p, bit_type_32, bit_type_32_sz); 12018 p += bit_type_32_sz; 12019 } 12020 12021 *p++ = '\n'; 12022 } 12023 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12024 for (j = 0; j < dd->chip_sdma_engines; j++) { 12025 snprintf(name, C_MAX_NAME, "%s%d", 12026 dev_cntrs[i].name, j); 12027 memcpy(p, name, strlen(name)); 12028 p += strlen(name); 12029 12030 /* Counter is 32 bits */ 12031 if (dev_cntrs[i].flags & CNTR_32BIT) { 12032 memcpy(p, bit_type_32, bit_type_32_sz); 12033 p += bit_type_32_sz; 12034 } 12035 12036 *p++ = '\n'; 12037 } 12038 } else { 12039 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name)); 12040 p += strlen(dev_cntrs[i].name); 12041 12042 /* Counter is 32 bits */ 12043 if (dev_cntrs[i].flags & CNTR_32BIT) { 12044 memcpy(p, bit_type_32, bit_type_32_sz); 12045 p += bit_type_32_sz; 12046 } 12047 12048 *p++ = '\n'; 12049 } 12050 } 12051 12052 /*********************/ 12053 /* per port counters */ 12054 /*********************/ 12055 12056 /* 12057 * Go through the counters for the overflows and disable the ones we 12058 * don't need. This varies based on platform so we need to do it 12059 * dynamically here. 12060 */ 12061 rcv_ctxts = dd->num_rcv_contexts; 12062 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts; 12063 i <= C_RCV_HDR_OVF_LAST; i++) { 12064 port_cntrs[i].flags |= CNTR_DISABLED; 12065 } 12066 12067 /* size port counter names and determine how many we have*/ 12068 sz = 0; 12069 dd->nportcntrs = 0; 12070 for (i = 0; i < PORT_CNTR_LAST; i++) { 12071 if (port_cntrs[i].flags & CNTR_DISABLED) { 12072 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name); 12073 continue; 12074 } 12075 12076 if (port_cntrs[i].flags & CNTR_VL) { 12077 port_cntrs[i].offset = dd->nportcntrs; 12078 for (j = 0; j < C_VL_COUNT; j++) { 12079 snprintf(name, C_MAX_NAME, "%s%d", 12080 port_cntrs[i].name, vl_from_idx(j)); 12081 sz += strlen(name); 12082 /* Add ",32" for 32-bit counters */ 12083 if (port_cntrs[i].flags & CNTR_32BIT) 12084 sz += bit_type_32_sz; 12085 sz++; 12086 dd->nportcntrs++; 12087 } 12088 } else { 12089 /* +1 for newline */ 12090 sz += strlen(port_cntrs[i].name) + 1; 12091 /* Add ",32" for 32-bit counters */ 12092 if (port_cntrs[i].flags & CNTR_32BIT) 12093 sz += bit_type_32_sz; 12094 port_cntrs[i].offset = dd->nportcntrs; 12095 dd->nportcntrs++; 12096 } 12097 } 12098 12099 /* allocate space for the counter names */ 12100 dd->portcntrnameslen = sz; 12101 dd->portcntrnames = kmalloc(sz, GFP_KERNEL); 12102 if (!dd->portcntrnames) 12103 goto bail; 12104 12105 /* fill in port cntr names */ 12106 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) { 12107 if (port_cntrs[i].flags & CNTR_DISABLED) 12108 continue; 12109 12110 if (port_cntrs[i].flags & CNTR_VL) { 12111 for (j = 0; j < C_VL_COUNT; j++) { 12112 snprintf(name, C_MAX_NAME, "%s%d", 12113 port_cntrs[i].name, vl_from_idx(j)); 12114 memcpy(p, name, strlen(name)); 12115 p += strlen(name); 12116 12117 /* Counter is 32 bits */ 12118 if (port_cntrs[i].flags & CNTR_32BIT) { 12119 memcpy(p, bit_type_32, bit_type_32_sz); 12120 p += bit_type_32_sz; 12121 } 12122 12123 *p++ = '\n'; 12124 } 12125 } else { 12126 memcpy(p, port_cntrs[i].name, 12127 strlen(port_cntrs[i].name)); 12128 p += strlen(port_cntrs[i].name); 12129 12130 /* Counter is 32 bits */ 12131 if (port_cntrs[i].flags & CNTR_32BIT) { 12132 memcpy(p, bit_type_32, bit_type_32_sz); 12133 p += bit_type_32_sz; 12134 } 12135 12136 *p++ = '\n'; 12137 } 12138 } 12139 12140 /* allocate per port storage for counter values */ 12141 ppd = (struct hfi1_pportdata *)(dd + 1); 12142 for (i = 0; i < dd->num_pports; i++, ppd++) { 12143 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12144 if (!ppd->cntrs) 12145 goto bail; 12146 12147 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12148 if (!ppd->scntrs) 12149 goto bail; 12150 } 12151 12152 /* CPU counters need to be allocated and zeroed */ 12153 if (init_cpu_counters(dd)) 12154 goto bail; 12155 12156 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12157 return 0; 12158 bail: 12159 free_cntrs(dd); 12160 return -ENOMEM; 12161 } 12162 12163 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) 12164 { 12165 switch (chip_lstate) { 12166 default: 12167 dd_dev_err(dd, 12168 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n", 12169 chip_lstate); 12170 /* fall through */ 12171 case LSTATE_DOWN: 12172 return IB_PORT_DOWN; 12173 case LSTATE_INIT: 12174 return IB_PORT_INIT; 12175 case LSTATE_ARMED: 12176 return IB_PORT_ARMED; 12177 case LSTATE_ACTIVE: 12178 return IB_PORT_ACTIVE; 12179 } 12180 } 12181 12182 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) 12183 { 12184 /* look at the HFI meta-states only */ 12185 switch (chip_pstate & 0xf0) { 12186 default: 12187 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n", 12188 chip_pstate); 12189 /* fall through */ 12190 case PLS_DISABLED: 12191 return IB_PORTPHYSSTATE_DISABLED; 12192 case PLS_OFFLINE: 12193 return OPA_PORTPHYSSTATE_OFFLINE; 12194 case PLS_POLLING: 12195 return IB_PORTPHYSSTATE_POLLING; 12196 case PLS_CONFIGPHY: 12197 return IB_PORTPHYSSTATE_TRAINING; 12198 case PLS_LINKUP: 12199 return IB_PORTPHYSSTATE_LINKUP; 12200 case PLS_PHYTEST: 12201 return IB_PORTPHYSSTATE_PHY_TEST; 12202 } 12203 } 12204 12205 /* return the OPA port logical state name */ 12206 const char *opa_lstate_name(u32 lstate) 12207 { 12208 static const char * const port_logical_names[] = { 12209 "PORT_NOP", 12210 "PORT_DOWN", 12211 "PORT_INIT", 12212 "PORT_ARMED", 12213 "PORT_ACTIVE", 12214 "PORT_ACTIVE_DEFER", 12215 }; 12216 if (lstate < ARRAY_SIZE(port_logical_names)) 12217 return port_logical_names[lstate]; 12218 return "unknown"; 12219 } 12220 12221 /* return the OPA port physical state name */ 12222 const char *opa_pstate_name(u32 pstate) 12223 { 12224 static const char * const port_physical_names[] = { 12225 "PHYS_NOP", 12226 "reserved1", 12227 "PHYS_POLL", 12228 "PHYS_DISABLED", 12229 "PHYS_TRAINING", 12230 "PHYS_LINKUP", 12231 "PHYS_LINK_ERR_RECOVER", 12232 "PHYS_PHY_TEST", 12233 "reserved8", 12234 "PHYS_OFFLINE", 12235 "PHYS_GANGED", 12236 "PHYS_TEST", 12237 }; 12238 if (pstate < ARRAY_SIZE(port_physical_names)) 12239 return port_physical_names[pstate]; 12240 return "unknown"; 12241 } 12242 12243 /* 12244 * Read the hardware link state and set the driver's cached value of it. 12245 * Return the (new) current value. 12246 */ 12247 u32 get_logical_state(struct hfi1_pportdata *ppd) 12248 { 12249 u32 new_state; 12250 12251 new_state = chip_to_opa_lstate(ppd->dd, read_logical_state(ppd->dd)); 12252 if (new_state != ppd->lstate) { 12253 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n", 12254 opa_lstate_name(new_state), new_state); 12255 ppd->lstate = new_state; 12256 } 12257 /* 12258 * Set port status flags in the page mapped into userspace 12259 * memory. Do it here to ensure a reliable state - this is 12260 * the only function called by all state handling code. 12261 * Always set the flags due to the fact that the cache value 12262 * might have been changed explicitly outside of this 12263 * function. 12264 */ 12265 if (ppd->statusp) { 12266 switch (ppd->lstate) { 12267 case IB_PORT_DOWN: 12268 case IB_PORT_INIT: 12269 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF | 12270 HFI1_STATUS_IB_READY); 12271 break; 12272 case IB_PORT_ARMED: 12273 *ppd->statusp |= HFI1_STATUS_IB_CONF; 12274 break; 12275 case IB_PORT_ACTIVE: 12276 *ppd->statusp |= HFI1_STATUS_IB_READY; 12277 break; 12278 } 12279 } 12280 return ppd->lstate; 12281 } 12282 12283 /** 12284 * wait_logical_linkstate - wait for an IB link state change to occur 12285 * @ppd: port device 12286 * @state: the state to wait for 12287 * @msecs: the number of milliseconds to wait 12288 * 12289 * Wait up to msecs milliseconds for IB link state change to occur. 12290 * For now, take the easy polling route. 12291 * Returns 0 if state reached, otherwise -ETIMEDOUT. 12292 */ 12293 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 12294 int msecs) 12295 { 12296 unsigned long timeout; 12297 12298 timeout = jiffies + msecs_to_jiffies(msecs); 12299 while (1) { 12300 if (get_logical_state(ppd) == state) 12301 return 0; 12302 if (time_after(jiffies, timeout)) 12303 break; 12304 msleep(20); 12305 } 12306 dd_dev_err(ppd->dd, "timeout waiting for link state 0x%x\n", state); 12307 12308 return -ETIMEDOUT; 12309 } 12310 12311 u8 hfi1_ibphys_portstate(struct hfi1_pportdata *ppd) 12312 { 12313 u32 pstate; 12314 u32 ib_pstate; 12315 12316 pstate = read_physical_state(ppd->dd); 12317 ib_pstate = chip_to_opa_pstate(ppd->dd, pstate); 12318 if (ppd->last_pstate != ib_pstate) { 12319 dd_dev_info(ppd->dd, 12320 "%s: physical state changed to %s (0x%x), phy 0x%x\n", 12321 __func__, opa_pstate_name(ib_pstate), ib_pstate, 12322 pstate); 12323 ppd->last_pstate = ib_pstate; 12324 } 12325 return ib_pstate; 12326 } 12327 12328 /* 12329 * Read/modify/write ASIC_QSFP register bits as selected by mask 12330 * data: 0 or 1 in the positions depending on what needs to be written 12331 * dir: 0 for read, 1 for write 12332 * mask: select by setting 12333 * I2CCLK (bit 0) 12334 * I2CDATA (bit 1) 12335 */ 12336 u64 hfi1_gpio_mod(struct hfi1_devdata *dd, u32 target, u32 data, u32 dir, 12337 u32 mask) 12338 { 12339 u64 qsfp_oe, target_oe; 12340 12341 target_oe = target ? ASIC_QSFP2_OE : ASIC_QSFP1_OE; 12342 if (mask) { 12343 /* We are writing register bits, so lock access */ 12344 dir &= mask; 12345 data &= mask; 12346 12347 qsfp_oe = read_csr(dd, target_oe); 12348 qsfp_oe = (qsfp_oe & ~(u64)mask) | (u64)dir; 12349 write_csr(dd, target_oe, qsfp_oe); 12350 } 12351 /* We are exclusively reading bits here, but it is unlikely 12352 * we'll get valid data when we set the direction of the pin 12353 * in the same call, so read should call this function again 12354 * to get valid data 12355 */ 12356 return read_csr(dd, target ? ASIC_QSFP2_IN : ASIC_QSFP1_IN); 12357 } 12358 12359 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \ 12360 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 12361 12362 #define SET_STATIC_RATE_CONTROL_SMASK(r) \ 12363 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 12364 12365 int hfi1_init_ctxt(struct send_context *sc) 12366 { 12367 if (sc) { 12368 struct hfi1_devdata *dd = sc->dd; 12369 u64 reg; 12370 u8 set = (sc->type == SC_USER ? 12371 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) : 12372 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL)); 12373 reg = read_kctxt_csr(dd, sc->hw_context, 12374 SEND_CTXT_CHECK_ENABLE); 12375 if (set) 12376 CLEAR_STATIC_RATE_CONTROL_SMASK(reg); 12377 else 12378 SET_STATIC_RATE_CONTROL_SMASK(reg); 12379 write_kctxt_csr(dd, sc->hw_context, 12380 SEND_CTXT_CHECK_ENABLE, reg); 12381 } 12382 return 0; 12383 } 12384 12385 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp) 12386 { 12387 int ret = 0; 12388 u64 reg; 12389 12390 if (dd->icode != ICODE_RTL_SILICON) { 12391 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 12392 dd_dev_info(dd, "%s: tempsense not supported by HW\n", 12393 __func__); 12394 return -EINVAL; 12395 } 12396 reg = read_csr(dd, ASIC_STS_THERM); 12397 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) & 12398 ASIC_STS_THERM_CURR_TEMP_MASK); 12399 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) & 12400 ASIC_STS_THERM_LO_TEMP_MASK); 12401 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) & 12402 ASIC_STS_THERM_HI_TEMP_MASK); 12403 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) & 12404 ASIC_STS_THERM_CRIT_TEMP_MASK); 12405 /* triggers is a 3-bit value - 1 bit per trigger. */ 12406 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7); 12407 12408 return ret; 12409 } 12410 12411 /* ========================================================================= */ 12412 12413 /* 12414 * Enable/disable chip from delivering interrupts. 12415 */ 12416 void set_intr_state(struct hfi1_devdata *dd, u32 enable) 12417 { 12418 int i; 12419 12420 /* 12421 * In HFI, the mask needs to be 1 to allow interrupts. 12422 */ 12423 if (enable) { 12424 /* enable all interrupts */ 12425 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 12426 write_csr(dd, CCE_INT_MASK + (8 * i), ~(u64)0); 12427 12428 init_qsfp_int(dd); 12429 } else { 12430 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 12431 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull); 12432 } 12433 } 12434 12435 /* 12436 * Clear all interrupt sources on the chip. 12437 */ 12438 static void clear_all_interrupts(struct hfi1_devdata *dd) 12439 { 12440 int i; 12441 12442 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 12443 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0); 12444 12445 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0); 12446 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0); 12447 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0); 12448 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0); 12449 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0); 12450 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0); 12451 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0); 12452 for (i = 0; i < dd->chip_send_contexts; i++) 12453 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0); 12454 for (i = 0; i < dd->chip_sdma_engines; i++) 12455 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0); 12456 12457 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0); 12458 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0); 12459 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0); 12460 } 12461 12462 /* Move to pcie.c? */ 12463 static void disable_intx(struct pci_dev *pdev) 12464 { 12465 pci_intx(pdev, 0); 12466 } 12467 12468 static void clean_up_interrupts(struct hfi1_devdata *dd) 12469 { 12470 int i; 12471 12472 /* remove irqs - must happen before disabling/turning off */ 12473 if (dd->num_msix_entries) { 12474 /* MSI-X */ 12475 struct hfi1_msix_entry *me = dd->msix_entries; 12476 12477 for (i = 0; i < dd->num_msix_entries; i++, me++) { 12478 if (!me->arg) /* => no irq, no affinity */ 12479 continue; 12480 hfi1_put_irq_affinity(dd, &dd->msix_entries[i]); 12481 free_irq(me->msix.vector, me->arg); 12482 } 12483 } else { 12484 /* INTx */ 12485 if (dd->requested_intx_irq) { 12486 free_irq(dd->pcidev->irq, dd); 12487 dd->requested_intx_irq = 0; 12488 } 12489 } 12490 12491 /* turn off interrupts */ 12492 if (dd->num_msix_entries) { 12493 /* MSI-X */ 12494 pci_disable_msix(dd->pcidev); 12495 } else { 12496 /* INTx */ 12497 disable_intx(dd->pcidev); 12498 } 12499 12500 /* clean structures */ 12501 kfree(dd->msix_entries); 12502 dd->msix_entries = NULL; 12503 dd->num_msix_entries = 0; 12504 } 12505 12506 /* 12507 * Remap the interrupt source from the general handler to the given MSI-X 12508 * interrupt. 12509 */ 12510 static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr) 12511 { 12512 u64 reg; 12513 int m, n; 12514 12515 /* clear from the handled mask of the general interrupt */ 12516 m = isrc / 64; 12517 n = isrc % 64; 12518 dd->gi_mask[m] &= ~((u64)1 << n); 12519 12520 /* direct the chip source to the given MSI-X interrupt */ 12521 m = isrc / 8; 12522 n = isrc % 8; 12523 reg = read_csr(dd, CCE_INT_MAP + (8 * m)); 12524 reg &= ~((u64)0xff << (8 * n)); 12525 reg |= ((u64)msix_intr & 0xff) << (8 * n); 12526 write_csr(dd, CCE_INT_MAP + (8 * m), reg); 12527 } 12528 12529 static void remap_sdma_interrupts(struct hfi1_devdata *dd, 12530 int engine, int msix_intr) 12531 { 12532 /* 12533 * SDMA engine interrupt sources grouped by type, rather than 12534 * engine. Per-engine interrupts are as follows: 12535 * SDMA 12536 * SDMAProgress 12537 * SDMAIdle 12538 */ 12539 remap_intr(dd, IS_SDMA_START + 0 * TXE_NUM_SDMA_ENGINES + engine, 12540 msix_intr); 12541 remap_intr(dd, IS_SDMA_START + 1 * TXE_NUM_SDMA_ENGINES + engine, 12542 msix_intr); 12543 remap_intr(dd, IS_SDMA_START + 2 * TXE_NUM_SDMA_ENGINES + engine, 12544 msix_intr); 12545 } 12546 12547 static int request_intx_irq(struct hfi1_devdata *dd) 12548 { 12549 int ret; 12550 12551 snprintf(dd->intx_name, sizeof(dd->intx_name), DRIVER_NAME "_%d", 12552 dd->unit); 12553 ret = request_irq(dd->pcidev->irq, general_interrupt, 12554 IRQF_SHARED, dd->intx_name, dd); 12555 if (ret) 12556 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n", 12557 ret); 12558 else 12559 dd->requested_intx_irq = 1; 12560 return ret; 12561 } 12562 12563 static int request_msix_irqs(struct hfi1_devdata *dd) 12564 { 12565 int first_general, last_general; 12566 int first_sdma, last_sdma; 12567 int first_rx, last_rx; 12568 int i, ret = 0; 12569 12570 /* calculate the ranges we are going to use */ 12571 first_general = 0; 12572 last_general = first_general + 1; 12573 first_sdma = last_general; 12574 last_sdma = first_sdma + dd->num_sdma; 12575 first_rx = last_sdma; 12576 last_rx = first_rx + dd->n_krcv_queues; 12577 12578 /* 12579 * Sanity check - the code expects all SDMA chip source 12580 * interrupts to be in the same CSR, starting at bit 0. Verify 12581 * that this is true by checking the bit location of the start. 12582 */ 12583 BUILD_BUG_ON(IS_SDMA_START % 64); 12584 12585 for (i = 0; i < dd->num_msix_entries; i++) { 12586 struct hfi1_msix_entry *me = &dd->msix_entries[i]; 12587 const char *err_info; 12588 irq_handler_t handler; 12589 irq_handler_t thread = NULL; 12590 void *arg; 12591 int idx; 12592 struct hfi1_ctxtdata *rcd = NULL; 12593 struct sdma_engine *sde = NULL; 12594 12595 /* obtain the arguments to request_irq */ 12596 if (first_general <= i && i < last_general) { 12597 idx = i - first_general; 12598 handler = general_interrupt; 12599 arg = dd; 12600 snprintf(me->name, sizeof(me->name), 12601 DRIVER_NAME "_%d", dd->unit); 12602 err_info = "general"; 12603 me->type = IRQ_GENERAL; 12604 } else if (first_sdma <= i && i < last_sdma) { 12605 idx = i - first_sdma; 12606 sde = &dd->per_sdma[idx]; 12607 handler = sdma_interrupt; 12608 arg = sde; 12609 snprintf(me->name, sizeof(me->name), 12610 DRIVER_NAME "_%d sdma%d", dd->unit, idx); 12611 err_info = "sdma"; 12612 remap_sdma_interrupts(dd, idx, i); 12613 me->type = IRQ_SDMA; 12614 } else if (first_rx <= i && i < last_rx) { 12615 idx = i - first_rx; 12616 rcd = dd->rcd[idx]; 12617 /* no interrupt if no rcd */ 12618 if (!rcd) 12619 continue; 12620 /* 12621 * Set the interrupt register and mask for this 12622 * context's interrupt. 12623 */ 12624 rcd->ireg = (IS_RCVAVAIL_START + idx) / 64; 12625 rcd->imask = ((u64)1) << 12626 ((IS_RCVAVAIL_START + idx) % 64); 12627 handler = receive_context_interrupt; 12628 thread = receive_context_thread; 12629 arg = rcd; 12630 snprintf(me->name, sizeof(me->name), 12631 DRIVER_NAME "_%d kctxt%d", dd->unit, idx); 12632 err_info = "receive context"; 12633 remap_intr(dd, IS_RCVAVAIL_START + idx, i); 12634 me->type = IRQ_RCVCTXT; 12635 } else { 12636 /* not in our expected range - complain, then 12637 * ignore it 12638 */ 12639 dd_dev_err(dd, 12640 "Unexpected extra MSI-X interrupt %d\n", i); 12641 continue; 12642 } 12643 /* no argument, no interrupt */ 12644 if (!arg) 12645 continue; 12646 /* make sure the name is terminated */ 12647 me->name[sizeof(me->name) - 1] = 0; 12648 12649 ret = request_threaded_irq(me->msix.vector, handler, thread, 0, 12650 me->name, arg); 12651 if (ret) { 12652 dd_dev_err(dd, 12653 "unable to allocate %s interrupt, vector %d, index %d, err %d\n", 12654 err_info, me->msix.vector, idx, ret); 12655 return ret; 12656 } 12657 /* 12658 * assign arg after request_irq call, so it will be 12659 * cleaned up 12660 */ 12661 me->arg = arg; 12662 12663 ret = hfi1_get_irq_affinity(dd, me); 12664 if (ret) 12665 dd_dev_err(dd, 12666 "unable to pin IRQ %d\n", ret); 12667 } 12668 12669 return ret; 12670 } 12671 12672 /* 12673 * Set the general handler to accept all interrupts, remap all 12674 * chip interrupts back to MSI-X 0. 12675 */ 12676 static void reset_interrupts(struct hfi1_devdata *dd) 12677 { 12678 int i; 12679 12680 /* all interrupts handled by the general handler */ 12681 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 12682 dd->gi_mask[i] = ~(u64)0; 12683 12684 /* all chip interrupts map to MSI-X 0 */ 12685 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 12686 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 12687 } 12688 12689 static int set_up_interrupts(struct hfi1_devdata *dd) 12690 { 12691 struct hfi1_msix_entry *entries; 12692 u32 total, request; 12693 int i, ret; 12694 int single_interrupt = 0; /* we expect to have all the interrupts */ 12695 12696 /* 12697 * Interrupt count: 12698 * 1 general, "slow path" interrupt (includes the SDMA engines 12699 * slow source, SDMACleanupDone) 12700 * N interrupts - one per used SDMA engine 12701 * M interrupt - one per kernel receive context 12702 */ 12703 total = 1 + dd->num_sdma + dd->n_krcv_queues; 12704 12705 entries = kcalloc(total, sizeof(*entries), GFP_KERNEL); 12706 if (!entries) { 12707 ret = -ENOMEM; 12708 goto fail; 12709 } 12710 /* 1-1 MSI-X entry assignment */ 12711 for (i = 0; i < total; i++) 12712 entries[i].msix.entry = i; 12713 12714 /* ask for MSI-X interrupts */ 12715 request = total; 12716 request_msix(dd, &request, entries); 12717 12718 if (request == 0) { 12719 /* using INTx */ 12720 /* dd->num_msix_entries already zero */ 12721 kfree(entries); 12722 single_interrupt = 1; 12723 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n"); 12724 } else { 12725 /* using MSI-X */ 12726 dd->num_msix_entries = request; 12727 dd->msix_entries = entries; 12728 12729 if (request != total) { 12730 /* using MSI-X, with reduced interrupts */ 12731 dd_dev_err( 12732 dd, 12733 "cannot handle reduced interrupt case, want %u, got %u\n", 12734 total, request); 12735 ret = -EINVAL; 12736 goto fail; 12737 } 12738 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total); 12739 } 12740 12741 /* mask all interrupts */ 12742 set_intr_state(dd, 0); 12743 /* clear all pending interrupts */ 12744 clear_all_interrupts(dd); 12745 12746 /* reset general handler mask, chip MSI-X mappings */ 12747 reset_interrupts(dd); 12748 12749 if (single_interrupt) 12750 ret = request_intx_irq(dd); 12751 else 12752 ret = request_msix_irqs(dd); 12753 if (ret) 12754 goto fail; 12755 12756 return 0; 12757 12758 fail: 12759 clean_up_interrupts(dd); 12760 return ret; 12761 } 12762 12763 /* 12764 * Set up context values in dd. Sets: 12765 * 12766 * num_rcv_contexts - number of contexts being used 12767 * n_krcv_queues - number of kernel contexts 12768 * first_user_ctxt - first non-kernel context in array of contexts 12769 * freectxts - number of free user contexts 12770 * num_send_contexts - number of PIO send contexts being used 12771 */ 12772 static int set_up_context_variables(struct hfi1_devdata *dd) 12773 { 12774 int num_kernel_contexts; 12775 int total_contexts; 12776 int ret; 12777 unsigned ngroups; 12778 int qos_rmt_count; 12779 int user_rmt_reduced; 12780 12781 /* 12782 * Kernel receive contexts: 12783 * - min of 2 or 1 context/numa (excluding control context) 12784 * - Context 0 - control context (VL15/multicast/error) 12785 * - Context 1 - first kernel context 12786 * - Context 2 - second kernel context 12787 * ... 12788 */ 12789 if (n_krcvqs) 12790 /* 12791 * n_krcvqs is the sum of module parameter kernel receive 12792 * contexts, krcvqs[]. It does not include the control 12793 * context, so add that. 12794 */ 12795 num_kernel_contexts = n_krcvqs + 1; 12796 else 12797 num_kernel_contexts = num_online_nodes() + 1; 12798 num_kernel_contexts = 12799 max_t(int, MIN_KERNEL_KCTXTS, num_kernel_contexts); 12800 /* 12801 * Every kernel receive context needs an ACK send context. 12802 * one send context is allocated for each VL{0-7} and VL15 12803 */ 12804 if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) { 12805 dd_dev_err(dd, 12806 "Reducing # kernel rcv contexts to: %d, from %d\n", 12807 (int)(dd->chip_send_contexts - num_vls - 1), 12808 (int)num_kernel_contexts); 12809 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1; 12810 } 12811 /* 12812 * User contexts: 12813 * - default to 1 user context per real (non-HT) CPU core if 12814 * num_user_contexts is negative 12815 */ 12816 if (num_user_contexts < 0) 12817 num_user_contexts = 12818 cpumask_weight(&dd->affinity->real_cpu_mask); 12819 12820 total_contexts = num_kernel_contexts + num_user_contexts; 12821 12822 /* 12823 * Adjust the counts given a global max. 12824 */ 12825 if (total_contexts > dd->chip_rcv_contexts) { 12826 dd_dev_err(dd, 12827 "Reducing # user receive contexts to: %d, from %d\n", 12828 (int)(dd->chip_rcv_contexts - num_kernel_contexts), 12829 (int)num_user_contexts); 12830 num_user_contexts = dd->chip_rcv_contexts - num_kernel_contexts; 12831 /* recalculate */ 12832 total_contexts = num_kernel_contexts + num_user_contexts; 12833 } 12834 12835 /* each user context requires an entry in the RMT */ 12836 qos_rmt_count = qos_rmt_entries(dd, NULL, NULL); 12837 if (qos_rmt_count + num_user_contexts > NUM_MAP_ENTRIES) { 12838 user_rmt_reduced = NUM_MAP_ENTRIES - qos_rmt_count; 12839 dd_dev_err(dd, 12840 "RMT size is reducing the number of user receive contexts from %d to %d\n", 12841 (int)num_user_contexts, 12842 user_rmt_reduced); 12843 /* recalculate */ 12844 num_user_contexts = user_rmt_reduced; 12845 total_contexts = num_kernel_contexts + num_user_contexts; 12846 } 12847 12848 /* the first N are kernel contexts, the rest are user contexts */ 12849 dd->num_rcv_contexts = total_contexts; 12850 dd->n_krcv_queues = num_kernel_contexts; 12851 dd->first_user_ctxt = num_kernel_contexts; 12852 dd->num_user_contexts = num_user_contexts; 12853 dd->freectxts = num_user_contexts; 12854 dd_dev_info(dd, 12855 "rcv contexts: chip %d, used %d (kernel %d, user %d)\n", 12856 (int)dd->chip_rcv_contexts, 12857 (int)dd->num_rcv_contexts, 12858 (int)dd->n_krcv_queues, 12859 (int)dd->num_rcv_contexts - dd->n_krcv_queues); 12860 12861 /* 12862 * Receive array allocation: 12863 * All RcvArray entries are divided into groups of 8. This 12864 * is required by the hardware and will speed up writes to 12865 * consecutive entries by using write-combining of the entire 12866 * cacheline. 12867 * 12868 * The number of groups are evenly divided among all contexts. 12869 * any left over groups will be given to the first N user 12870 * contexts. 12871 */ 12872 dd->rcv_entries.group_size = RCV_INCREMENT; 12873 ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size; 12874 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts; 12875 dd->rcv_entries.nctxt_extra = ngroups - 12876 (dd->num_rcv_contexts * dd->rcv_entries.ngroups); 12877 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n", 12878 dd->rcv_entries.ngroups, 12879 dd->rcv_entries.nctxt_extra); 12880 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size > 12881 MAX_EAGER_ENTRIES * 2) { 12882 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) / 12883 dd->rcv_entries.group_size; 12884 dd_dev_info(dd, 12885 "RcvArray group count too high, change to %u\n", 12886 dd->rcv_entries.ngroups); 12887 dd->rcv_entries.nctxt_extra = 0; 12888 } 12889 /* 12890 * PIO send contexts 12891 */ 12892 ret = init_sc_pools_and_sizes(dd); 12893 if (ret >= 0) { /* success */ 12894 dd->num_send_contexts = ret; 12895 dd_dev_info( 12896 dd, 12897 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n", 12898 dd->chip_send_contexts, 12899 dd->num_send_contexts, 12900 dd->sc_sizes[SC_KERNEL].count, 12901 dd->sc_sizes[SC_ACK].count, 12902 dd->sc_sizes[SC_USER].count, 12903 dd->sc_sizes[SC_VL15].count); 12904 ret = 0; /* success */ 12905 } 12906 12907 return ret; 12908 } 12909 12910 /* 12911 * Set the device/port partition key table. The MAD code 12912 * will ensure that, at least, the partial management 12913 * partition key is present in the table. 12914 */ 12915 static void set_partition_keys(struct hfi1_pportdata *ppd) 12916 { 12917 struct hfi1_devdata *dd = ppd->dd; 12918 u64 reg = 0; 12919 int i; 12920 12921 dd_dev_info(dd, "Setting partition keys\n"); 12922 for (i = 0; i < hfi1_get_npkeys(dd); i++) { 12923 reg |= (ppd->pkeys[i] & 12924 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) << 12925 ((i % 4) * 12926 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT); 12927 /* Each register holds 4 PKey values. */ 12928 if ((i % 4) == 3) { 12929 write_csr(dd, RCV_PARTITION_KEY + 12930 ((i - 3) * 2), reg); 12931 reg = 0; 12932 } 12933 } 12934 12935 /* Always enable HW pkeys check when pkeys table is set */ 12936 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK); 12937 } 12938 12939 /* 12940 * These CSRs and memories are uninitialized on reset and must be 12941 * written before reading to set the ECC/parity bits. 12942 * 12943 * NOTE: All user context CSRs that are not mmaped write-only 12944 * (e.g. the TID flows) must be initialized even if the driver never 12945 * reads them. 12946 */ 12947 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd) 12948 { 12949 int i, j; 12950 12951 /* CceIntMap */ 12952 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 12953 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 12954 12955 /* SendCtxtCreditReturnAddr */ 12956 for (i = 0; i < dd->chip_send_contexts; i++) 12957 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 12958 12959 /* PIO Send buffers */ 12960 /* SDMA Send buffers */ 12961 /* 12962 * These are not normally read, and (presently) have no method 12963 * to be read, so are not pre-initialized 12964 */ 12965 12966 /* RcvHdrAddr */ 12967 /* RcvHdrTailAddr */ 12968 /* RcvTidFlowTable */ 12969 for (i = 0; i < dd->chip_rcv_contexts; i++) { 12970 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 12971 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 12972 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) 12973 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0); 12974 } 12975 12976 /* RcvArray */ 12977 for (i = 0; i < dd->chip_rcv_array_count; i++) 12978 write_csr(dd, RCV_ARRAY + (8 * i), 12979 RCV_ARRAY_RT_WRITE_ENABLE_SMASK); 12980 12981 /* RcvQPMapTable */ 12982 for (i = 0; i < 32; i++) 12983 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 12984 } 12985 12986 /* 12987 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus. 12988 */ 12989 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits, 12990 u64 ctrl_bits) 12991 { 12992 unsigned long timeout; 12993 u64 reg; 12994 12995 /* is the condition present? */ 12996 reg = read_csr(dd, CCE_STATUS); 12997 if ((reg & status_bits) == 0) 12998 return; 12999 13000 /* clear the condition */ 13001 write_csr(dd, CCE_CTRL, ctrl_bits); 13002 13003 /* wait for the condition to clear */ 13004 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT); 13005 while (1) { 13006 reg = read_csr(dd, CCE_STATUS); 13007 if ((reg & status_bits) == 0) 13008 return; 13009 if (time_after(jiffies, timeout)) { 13010 dd_dev_err(dd, 13011 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n", 13012 status_bits, reg & status_bits); 13013 return; 13014 } 13015 udelay(1); 13016 } 13017 } 13018 13019 /* set CCE CSRs to chip reset defaults */ 13020 static void reset_cce_csrs(struct hfi1_devdata *dd) 13021 { 13022 int i; 13023 13024 /* CCE_REVISION read-only */ 13025 /* CCE_REVISION2 read-only */ 13026 /* CCE_CTRL - bits clear automatically */ 13027 /* CCE_STATUS read-only, use CceCtrl to clear */ 13028 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK); 13029 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK); 13030 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK); 13031 for (i = 0; i < CCE_NUM_SCRATCH; i++) 13032 write_csr(dd, CCE_SCRATCH + (8 * i), 0); 13033 /* CCE_ERR_STATUS read-only */ 13034 write_csr(dd, CCE_ERR_MASK, 0); 13035 write_csr(dd, CCE_ERR_CLEAR, ~0ull); 13036 /* CCE_ERR_FORCE leave alone */ 13037 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++) 13038 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0); 13039 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR); 13040 /* CCE_PCIE_CTRL leave alone */ 13041 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) { 13042 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0); 13043 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i), 13044 CCE_MSIX_TABLE_UPPER_RESETCSR); 13045 } 13046 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) { 13047 /* CCE_MSIX_PBA read-only */ 13048 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull); 13049 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull); 13050 } 13051 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13052 write_csr(dd, CCE_INT_MAP, 0); 13053 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 13054 /* CCE_INT_STATUS read-only */ 13055 write_csr(dd, CCE_INT_MASK + (8 * i), 0); 13056 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull); 13057 /* CCE_INT_FORCE leave alone */ 13058 /* CCE_INT_BLOCKED read-only */ 13059 } 13060 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++) 13061 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0); 13062 } 13063 13064 /* set MISC CSRs to chip reset defaults */ 13065 static void reset_misc_csrs(struct hfi1_devdata *dd) 13066 { 13067 int i; 13068 13069 for (i = 0; i < 32; i++) { 13070 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0); 13071 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0); 13072 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0); 13073 } 13074 /* 13075 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can 13076 * only be written 128-byte chunks 13077 */ 13078 /* init RSA engine to clear lingering errors */ 13079 write_csr(dd, MISC_CFG_RSA_CMD, 1); 13080 write_csr(dd, MISC_CFG_RSA_MU, 0); 13081 write_csr(dd, MISC_CFG_FW_CTRL, 0); 13082 /* MISC_STS_8051_DIGEST read-only */ 13083 /* MISC_STS_SBM_DIGEST read-only */ 13084 /* MISC_STS_PCIE_DIGEST read-only */ 13085 /* MISC_STS_FAB_DIGEST read-only */ 13086 /* MISC_ERR_STATUS read-only */ 13087 write_csr(dd, MISC_ERR_MASK, 0); 13088 write_csr(dd, MISC_ERR_CLEAR, ~0ull); 13089 /* MISC_ERR_FORCE leave alone */ 13090 } 13091 13092 /* set TXE CSRs to chip reset defaults */ 13093 static void reset_txe_csrs(struct hfi1_devdata *dd) 13094 { 13095 int i; 13096 13097 /* 13098 * TXE Kernel CSRs 13099 */ 13100 write_csr(dd, SEND_CTRL, 0); 13101 __cm_reset(dd, 0); /* reset CM internal state */ 13102 /* SEND_CONTEXTS read-only */ 13103 /* SEND_DMA_ENGINES read-only */ 13104 /* SEND_PIO_MEM_SIZE read-only */ 13105 /* SEND_DMA_MEM_SIZE read-only */ 13106 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0); 13107 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */ 13108 /* SEND_PIO_ERR_STATUS read-only */ 13109 write_csr(dd, SEND_PIO_ERR_MASK, 0); 13110 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull); 13111 /* SEND_PIO_ERR_FORCE leave alone */ 13112 /* SEND_DMA_ERR_STATUS read-only */ 13113 write_csr(dd, SEND_DMA_ERR_MASK, 0); 13114 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull); 13115 /* SEND_DMA_ERR_FORCE leave alone */ 13116 /* SEND_EGRESS_ERR_STATUS read-only */ 13117 write_csr(dd, SEND_EGRESS_ERR_MASK, 0); 13118 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull); 13119 /* SEND_EGRESS_ERR_FORCE leave alone */ 13120 write_csr(dd, SEND_BTH_QP, 0); 13121 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0); 13122 write_csr(dd, SEND_SC2VLT0, 0); 13123 write_csr(dd, SEND_SC2VLT1, 0); 13124 write_csr(dd, SEND_SC2VLT2, 0); 13125 write_csr(dd, SEND_SC2VLT3, 0); 13126 write_csr(dd, SEND_LEN_CHECK0, 0); 13127 write_csr(dd, SEND_LEN_CHECK1, 0); 13128 /* SEND_ERR_STATUS read-only */ 13129 write_csr(dd, SEND_ERR_MASK, 0); 13130 write_csr(dd, SEND_ERR_CLEAR, ~0ull); 13131 /* SEND_ERR_FORCE read-only */ 13132 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++) 13133 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0); 13134 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++) 13135 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0); 13136 for (i = 0; i < dd->chip_send_contexts / NUM_CONTEXTS_PER_SET; i++) 13137 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0); 13138 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++) 13139 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0); 13140 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++) 13141 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0); 13142 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR); 13143 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR); 13144 /* SEND_CM_CREDIT_USED_STATUS read-only */ 13145 write_csr(dd, SEND_CM_TIMER_CTRL, 0); 13146 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0); 13147 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0); 13148 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0); 13149 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0); 13150 for (i = 0; i < TXE_NUM_DATA_VL; i++) 13151 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 13152 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 13153 /* SEND_CM_CREDIT_USED_VL read-only */ 13154 /* SEND_CM_CREDIT_USED_VL15 read-only */ 13155 /* SEND_EGRESS_CTXT_STATUS read-only */ 13156 /* SEND_EGRESS_SEND_DMA_STATUS read-only */ 13157 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull); 13158 /* SEND_EGRESS_ERR_INFO read-only */ 13159 /* SEND_EGRESS_ERR_SOURCE read-only */ 13160 13161 /* 13162 * TXE Per-Context CSRs 13163 */ 13164 for (i = 0; i < dd->chip_send_contexts; i++) { 13165 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 13166 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0); 13167 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13168 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0); 13169 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0); 13170 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull); 13171 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0); 13172 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0); 13173 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0); 13174 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0); 13175 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0); 13176 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0); 13177 } 13178 13179 /* 13180 * TXE Per-SDMA CSRs 13181 */ 13182 for (i = 0; i < dd->chip_sdma_engines; i++) { 13183 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 13184 /* SEND_DMA_STATUS read-only */ 13185 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0); 13186 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0); 13187 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0); 13188 /* SEND_DMA_HEAD read-only */ 13189 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0); 13190 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0); 13191 /* SEND_DMA_IDLE_CNT read-only */ 13192 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0); 13193 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0); 13194 /* SEND_DMA_DESC_FETCHED_CNT read-only */ 13195 /* SEND_DMA_ENG_ERR_STATUS read-only */ 13196 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0); 13197 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull); 13198 /* SEND_DMA_ENG_ERR_FORCE leave alone */ 13199 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0); 13200 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0); 13201 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0); 13202 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0); 13203 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0); 13204 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0); 13205 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0); 13206 } 13207 } 13208 13209 /* 13210 * Expect on entry: 13211 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0 13212 */ 13213 static void init_rbufs(struct hfi1_devdata *dd) 13214 { 13215 u64 reg; 13216 int count; 13217 13218 /* 13219 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are 13220 * clear. 13221 */ 13222 count = 0; 13223 while (1) { 13224 reg = read_csr(dd, RCV_STATUS); 13225 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK 13226 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0) 13227 break; 13228 /* 13229 * Give up after 1ms - maximum wait time. 13230 * 13231 * RBuf size is 148KiB. Slowest possible is PCIe Gen1 x1 at 13232 * 250MB/s bandwidth. Lower rate to 66% for overhead to get: 13233 * 148 KB / (66% * 250MB/s) = 920us 13234 */ 13235 if (count++ > 500) { 13236 dd_dev_err(dd, 13237 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n", 13238 __func__, reg); 13239 break; 13240 } 13241 udelay(2); /* do not busy-wait the CSR */ 13242 } 13243 13244 /* start the init - expect RcvCtrl to be 0 */ 13245 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK); 13246 13247 /* 13248 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief 13249 * period after the write before RcvStatus.RxRbufInitDone is valid. 13250 * The delay in the first run through the loop below is sufficient and 13251 * required before the first read of RcvStatus.RxRbufInintDone. 13252 */ 13253 read_csr(dd, RCV_CTRL); 13254 13255 /* wait for the init to finish */ 13256 count = 0; 13257 while (1) { 13258 /* delay is required first time through - see above */ 13259 udelay(2); /* do not busy-wait the CSR */ 13260 reg = read_csr(dd, RCV_STATUS); 13261 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK)) 13262 break; 13263 13264 /* give up after 100us - slowest possible at 33MHz is 73us */ 13265 if (count++ > 50) { 13266 dd_dev_err(dd, 13267 "%s: RcvStatus.RxRbufInit not set, continuing\n", 13268 __func__); 13269 break; 13270 } 13271 } 13272 } 13273 13274 /* set RXE CSRs to chip reset defaults */ 13275 static void reset_rxe_csrs(struct hfi1_devdata *dd) 13276 { 13277 int i, j; 13278 13279 /* 13280 * RXE Kernel CSRs 13281 */ 13282 write_csr(dd, RCV_CTRL, 0); 13283 init_rbufs(dd); 13284 /* RCV_STATUS read-only */ 13285 /* RCV_CONTEXTS read-only */ 13286 /* RCV_ARRAY_CNT read-only */ 13287 /* RCV_BUF_SIZE read-only */ 13288 write_csr(dd, RCV_BTH_QP, 0); 13289 write_csr(dd, RCV_MULTICAST, 0); 13290 write_csr(dd, RCV_BYPASS, 0); 13291 write_csr(dd, RCV_VL15, 0); 13292 /* this is a clear-down */ 13293 write_csr(dd, RCV_ERR_INFO, 13294 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK); 13295 /* RCV_ERR_STATUS read-only */ 13296 write_csr(dd, RCV_ERR_MASK, 0); 13297 write_csr(dd, RCV_ERR_CLEAR, ~0ull); 13298 /* RCV_ERR_FORCE leave alone */ 13299 for (i = 0; i < 32; i++) 13300 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13301 for (i = 0; i < 4; i++) 13302 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0); 13303 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++) 13304 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0); 13305 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++) 13306 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0); 13307 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) { 13308 write_csr(dd, RCV_RSM_CFG + (8 * i), 0); 13309 write_csr(dd, RCV_RSM_SELECT + (8 * i), 0); 13310 write_csr(dd, RCV_RSM_MATCH + (8 * i), 0); 13311 } 13312 for (i = 0; i < 32; i++) 13313 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0); 13314 13315 /* 13316 * RXE Kernel and User Per-Context CSRs 13317 */ 13318 for (i = 0; i < dd->chip_rcv_contexts; i++) { 13319 /* kernel */ 13320 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0); 13321 /* RCV_CTXT_STATUS read-only */ 13322 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0); 13323 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0); 13324 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0); 13325 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13326 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0); 13327 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0); 13328 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0); 13329 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13330 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0); 13331 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0); 13332 13333 /* user */ 13334 /* RCV_HDR_TAIL read-only */ 13335 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0); 13336 /* RCV_EGR_INDEX_TAIL read-only */ 13337 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0); 13338 /* RCV_EGR_OFFSET_TAIL read-only */ 13339 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) { 13340 write_uctxt_csr(dd, i, 13341 RCV_TID_FLOW_TABLE + (8 * j), 0); 13342 } 13343 } 13344 } 13345 13346 /* 13347 * Set sc2vl tables. 13348 * 13349 * They power on to zeros, so to avoid send context errors 13350 * they need to be set: 13351 * 13352 * SC 0-7 -> VL 0-7 (respectively) 13353 * SC 15 -> VL 15 13354 * otherwise 13355 * -> VL 0 13356 */ 13357 static void init_sc2vl_tables(struct hfi1_devdata *dd) 13358 { 13359 int i; 13360 /* init per architecture spec, constrained by hardware capability */ 13361 13362 /* HFI maps sent packets */ 13363 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL( 13364 0, 13365 0, 0, 1, 1, 13366 2, 2, 3, 3, 13367 4, 4, 5, 5, 13368 6, 6, 7, 7)); 13369 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL( 13370 1, 13371 8, 0, 9, 0, 13372 10, 0, 11, 0, 13373 12, 0, 13, 0, 13374 14, 0, 15, 15)); 13375 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL( 13376 2, 13377 16, 0, 17, 0, 13378 18, 0, 19, 0, 13379 20, 0, 21, 0, 13380 22, 0, 23, 0)); 13381 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL( 13382 3, 13383 24, 0, 25, 0, 13384 26, 0, 27, 0, 13385 28, 0, 29, 0, 13386 30, 0, 31, 0)); 13387 13388 /* DC maps received packets */ 13389 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL( 13390 15_0, 13391 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 13392 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15)); 13393 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL( 13394 31_16, 13395 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 13396 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0)); 13397 13398 /* initialize the cached sc2vl values consistently with h/w */ 13399 for (i = 0; i < 32; i++) { 13400 if (i < 8 || i == 15) 13401 *((u8 *)(dd->sc2vl) + i) = (u8)i; 13402 else 13403 *((u8 *)(dd->sc2vl) + i) = 0; 13404 } 13405 } 13406 13407 /* 13408 * Read chip sizes and then reset parts to sane, disabled, values. We cannot 13409 * depend on the chip going through a power-on reset - a driver may be loaded 13410 * and unloaded many times. 13411 * 13412 * Do not write any CSR values to the chip in this routine - there may be 13413 * a reset following the (possible) FLR in this routine. 13414 * 13415 */ 13416 static void init_chip(struct hfi1_devdata *dd) 13417 { 13418 int i; 13419 13420 /* 13421 * Put the HFI CSRs in a known state. 13422 * Combine this with a DC reset. 13423 * 13424 * Stop the device from doing anything while we do a 13425 * reset. We know there are no other active users of 13426 * the device since we are now in charge. Turn off 13427 * off all outbound and inbound traffic and make sure 13428 * the device does not generate any interrupts. 13429 */ 13430 13431 /* disable send contexts and SDMA engines */ 13432 write_csr(dd, SEND_CTRL, 0); 13433 for (i = 0; i < dd->chip_send_contexts; i++) 13434 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 13435 for (i = 0; i < dd->chip_sdma_engines; i++) 13436 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 13437 /* disable port (turn off RXE inbound traffic) and contexts */ 13438 write_csr(dd, RCV_CTRL, 0); 13439 for (i = 0; i < dd->chip_rcv_contexts; i++) 13440 write_csr(dd, RCV_CTXT_CTRL, 0); 13441 /* mask all interrupt sources */ 13442 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 13443 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull); 13444 13445 /* 13446 * DC Reset: do a full DC reset before the register clear. 13447 * A recommended length of time to hold is one CSR read, 13448 * so reread the CceDcCtrl. Then, hold the DC in reset 13449 * across the clear. 13450 */ 13451 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK); 13452 (void)read_csr(dd, CCE_DC_CTRL); 13453 13454 if (use_flr) { 13455 /* 13456 * A FLR will reset the SPC core and part of the PCIe. 13457 * The parts that need to be restored have already been 13458 * saved. 13459 */ 13460 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 13461 13462 /* do the FLR, the DC reset will remain */ 13463 hfi1_pcie_flr(dd); 13464 13465 /* restore command and BARs */ 13466 restore_pci_variables(dd); 13467 13468 if (is_ax(dd)) { 13469 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 13470 hfi1_pcie_flr(dd); 13471 restore_pci_variables(dd); 13472 } 13473 } else { 13474 dd_dev_info(dd, "Resetting CSRs with writes\n"); 13475 reset_cce_csrs(dd); 13476 reset_txe_csrs(dd); 13477 reset_rxe_csrs(dd); 13478 reset_misc_csrs(dd); 13479 } 13480 /* clear the DC reset */ 13481 write_csr(dd, CCE_DC_CTRL, 0); 13482 13483 /* Set the LED off */ 13484 setextled(dd, 0); 13485 13486 /* 13487 * Clear the QSFP reset. 13488 * An FLR enforces a 0 on all out pins. The driver does not touch 13489 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and 13490 * anything plugged constantly in reset, if it pays attention 13491 * to RESET_N. 13492 * Prime examples of this are optical cables. Set all pins high. 13493 * I2CCLK and I2CDAT will change per direction, and INT_N and 13494 * MODPRS_N are input only and their value is ignored. 13495 */ 13496 write_csr(dd, ASIC_QSFP1_OUT, 0x1f); 13497 write_csr(dd, ASIC_QSFP2_OUT, 0x1f); 13498 init_chip_resources(dd); 13499 } 13500 13501 static void init_early_variables(struct hfi1_devdata *dd) 13502 { 13503 int i; 13504 13505 /* assign link credit variables */ 13506 dd->vau = CM_VAU; 13507 dd->link_credits = CM_GLOBAL_CREDITS; 13508 if (is_ax(dd)) 13509 dd->link_credits--; 13510 dd->vcu = cu_to_vcu(hfi1_cu); 13511 /* enough room for 8 MAD packets plus header - 17K */ 13512 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau); 13513 if (dd->vl15_init > dd->link_credits) 13514 dd->vl15_init = dd->link_credits; 13515 13516 write_uninitialized_csrs_and_memories(dd); 13517 13518 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 13519 for (i = 0; i < dd->num_pports; i++) { 13520 struct hfi1_pportdata *ppd = &dd->pport[i]; 13521 13522 set_partition_keys(ppd); 13523 } 13524 init_sc2vl_tables(dd); 13525 } 13526 13527 static void init_kdeth_qp(struct hfi1_devdata *dd) 13528 { 13529 /* user changed the KDETH_QP */ 13530 if (kdeth_qp != 0 && kdeth_qp >= 0xff) { 13531 /* out of range or illegal value */ 13532 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring"); 13533 kdeth_qp = 0; 13534 } 13535 if (kdeth_qp == 0) /* not set, or failed range check */ 13536 kdeth_qp = DEFAULT_KDETH_QP; 13537 13538 write_csr(dd, SEND_BTH_QP, 13539 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) << 13540 SEND_BTH_QP_KDETH_QP_SHIFT); 13541 13542 write_csr(dd, RCV_BTH_QP, 13543 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) << 13544 RCV_BTH_QP_KDETH_QP_SHIFT); 13545 } 13546 13547 /** 13548 * init_qpmap_table 13549 * @dd - device data 13550 * @first_ctxt - first context 13551 * @last_ctxt - first context 13552 * 13553 * This return sets the qpn mapping table that 13554 * is indexed by qpn[8:1]. 13555 * 13556 * The routine will round robin the 256 settings 13557 * from first_ctxt to last_ctxt. 13558 * 13559 * The first/last looks ahead to having specialized 13560 * receive contexts for mgmt and bypass. Normal 13561 * verbs traffic will assumed to be on a range 13562 * of receive contexts. 13563 */ 13564 static void init_qpmap_table(struct hfi1_devdata *dd, 13565 u32 first_ctxt, 13566 u32 last_ctxt) 13567 { 13568 u64 reg = 0; 13569 u64 regno = RCV_QP_MAP_TABLE; 13570 int i; 13571 u64 ctxt = first_ctxt; 13572 13573 for (i = 0; i < 256; i++) { 13574 reg |= ctxt << (8 * (i % 8)); 13575 ctxt++; 13576 if (ctxt > last_ctxt) 13577 ctxt = first_ctxt; 13578 if (i % 8 == 7) { 13579 write_csr(dd, regno, reg); 13580 reg = 0; 13581 regno += 8; 13582 } 13583 } 13584 13585 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK 13586 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK); 13587 } 13588 13589 struct rsm_map_table { 13590 u64 map[NUM_MAP_REGS]; 13591 unsigned int used; 13592 }; 13593 13594 struct rsm_rule_data { 13595 u8 offset; 13596 u8 pkt_type; 13597 u32 field1_off; 13598 u32 field2_off; 13599 u32 index1_off; 13600 u32 index1_width; 13601 u32 index2_off; 13602 u32 index2_width; 13603 u32 mask1; 13604 u32 value1; 13605 u32 mask2; 13606 u32 value2; 13607 }; 13608 13609 /* 13610 * Return an initialized RMT map table for users to fill in. OK if it 13611 * returns NULL, indicating no table. 13612 */ 13613 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd) 13614 { 13615 struct rsm_map_table *rmt; 13616 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */ 13617 13618 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL); 13619 if (rmt) { 13620 memset(rmt->map, rxcontext, sizeof(rmt->map)); 13621 rmt->used = 0; 13622 } 13623 13624 return rmt; 13625 } 13626 13627 /* 13628 * Write the final RMT map table to the chip and free the table. OK if 13629 * table is NULL. 13630 */ 13631 static void complete_rsm_map_table(struct hfi1_devdata *dd, 13632 struct rsm_map_table *rmt) 13633 { 13634 int i; 13635 13636 if (rmt) { 13637 /* write table to chip */ 13638 for (i = 0; i < NUM_MAP_REGS; i++) 13639 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]); 13640 13641 /* enable RSM */ 13642 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 13643 } 13644 } 13645 13646 /* 13647 * Add a receive side mapping rule. 13648 */ 13649 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index, 13650 struct rsm_rule_data *rrd) 13651 { 13652 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 13653 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT | 13654 1ull << rule_index | /* enable bit */ 13655 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT); 13656 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 13657 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT | 13658 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT | 13659 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT | 13660 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT | 13661 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT | 13662 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT); 13663 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 13664 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT | 13665 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT | 13666 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT | 13667 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT); 13668 } 13669 13670 /* return the number of RSM map table entries that will be used for QOS */ 13671 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 13672 unsigned int *np) 13673 { 13674 int i; 13675 unsigned int m, n; 13676 u8 max_by_vl = 0; 13677 13678 /* is QOS active at all? */ 13679 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS || 13680 num_vls == 1 || 13681 krcvqsset <= 1) 13682 goto no_qos; 13683 13684 /* determine bits for qpn */ 13685 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++) 13686 if (krcvqs[i] > max_by_vl) 13687 max_by_vl = krcvqs[i]; 13688 if (max_by_vl > 32) 13689 goto no_qos; 13690 m = ilog2(__roundup_pow_of_two(max_by_vl)); 13691 13692 /* determine bits for vl */ 13693 n = ilog2(__roundup_pow_of_two(num_vls)); 13694 13695 /* reject if too much is used */ 13696 if ((m + n) > 7) 13697 goto no_qos; 13698 13699 if (mp) 13700 *mp = m; 13701 if (np) 13702 *np = n; 13703 13704 return 1 << (m + n); 13705 13706 no_qos: 13707 if (mp) 13708 *mp = 0; 13709 if (np) 13710 *np = 0; 13711 return 0; 13712 } 13713 13714 /** 13715 * init_qos - init RX qos 13716 * @dd - device data 13717 * @rmt - RSM map table 13718 * 13719 * This routine initializes Rule 0 and the RSM map table to implement 13720 * quality of service (qos). 13721 * 13722 * If all of the limit tests succeed, qos is applied based on the array 13723 * interpretation of krcvqs where entry 0 is VL0. 13724 * 13725 * The number of vl bits (n) and the number of qpn bits (m) are computed to 13726 * feed both the RSM map table and the single rule. 13727 */ 13728 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt) 13729 { 13730 struct rsm_rule_data rrd; 13731 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m; 13732 unsigned int rmt_entries; 13733 u64 reg; 13734 13735 if (!rmt) 13736 goto bail; 13737 rmt_entries = qos_rmt_entries(dd, &m, &n); 13738 if (rmt_entries == 0) 13739 goto bail; 13740 qpns_per_vl = 1 << m; 13741 13742 /* enough room in the map table? */ 13743 rmt_entries = 1 << (m + n); 13744 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES) 13745 goto bail; 13746 13747 /* add qos entries to the the RSM map table */ 13748 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) { 13749 unsigned tctxt; 13750 13751 for (qpn = 0, tctxt = ctxt; 13752 krcvqs[i] && qpn < qpns_per_vl; qpn++) { 13753 unsigned idx, regoff, regidx; 13754 13755 /* generate the index the hardware will produce */ 13756 idx = rmt->used + ((qpn << n) ^ i); 13757 regoff = (idx % 8) * 8; 13758 regidx = idx / 8; 13759 /* replace default with context number */ 13760 reg = rmt->map[regidx]; 13761 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK 13762 << regoff); 13763 reg |= (u64)(tctxt++) << regoff; 13764 rmt->map[regidx] = reg; 13765 if (tctxt == ctxt + krcvqs[i]) 13766 tctxt = ctxt; 13767 } 13768 ctxt += krcvqs[i]; 13769 } 13770 13771 rrd.offset = rmt->used; 13772 rrd.pkt_type = 2; 13773 rrd.field1_off = LRH_BTH_MATCH_OFFSET; 13774 rrd.field2_off = LRH_SC_MATCH_OFFSET; 13775 rrd.index1_off = LRH_SC_SELECT_OFFSET; 13776 rrd.index1_width = n; 13777 rrd.index2_off = QPN_SELECT_OFFSET; 13778 rrd.index2_width = m + n; 13779 rrd.mask1 = LRH_BTH_MASK; 13780 rrd.value1 = LRH_BTH_VALUE; 13781 rrd.mask2 = LRH_SC_MASK; 13782 rrd.value2 = LRH_SC_VALUE; 13783 13784 /* add rule 0 */ 13785 add_rsm_rule(dd, 0, &rrd); 13786 13787 /* mark RSM map entries as used */ 13788 rmt->used += rmt_entries; 13789 /* map everything else to the mcast/err/vl15 context */ 13790 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT); 13791 dd->qos_shift = n + 1; 13792 return; 13793 bail: 13794 dd->qos_shift = 1; 13795 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1); 13796 } 13797 13798 static void init_user_fecn_handling(struct hfi1_devdata *dd, 13799 struct rsm_map_table *rmt) 13800 { 13801 struct rsm_rule_data rrd; 13802 u64 reg; 13803 int i, idx, regoff, regidx; 13804 u8 offset; 13805 13806 /* there needs to be enough room in the map table */ 13807 if (rmt->used + dd->num_user_contexts >= NUM_MAP_ENTRIES) { 13808 dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n"); 13809 return; 13810 } 13811 13812 /* 13813 * RSM will extract the destination context as an index into the 13814 * map table. The destination contexts are a sequential block 13815 * in the range first_user_ctxt...num_rcv_contexts-1 (inclusive). 13816 * Map entries are accessed as offset + extracted value. Adjust 13817 * the added offset so this sequence can be placed anywhere in 13818 * the table - as long as the entries themselves do not wrap. 13819 * There are only enough bits in offset for the table size, so 13820 * start with that to allow for a "negative" offset. 13821 */ 13822 offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used - 13823 (int)dd->first_user_ctxt); 13824 13825 for (i = dd->first_user_ctxt, idx = rmt->used; 13826 i < dd->num_rcv_contexts; i++, idx++) { 13827 /* replace with identity mapping */ 13828 regoff = (idx % 8) * 8; 13829 regidx = idx / 8; 13830 reg = rmt->map[regidx]; 13831 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff); 13832 reg |= (u64)i << regoff; 13833 rmt->map[regidx] = reg; 13834 } 13835 13836 /* 13837 * For RSM intercept of Expected FECN packets: 13838 * o packet type 0 - expected 13839 * o match on F (bit 95), using select/match 1, and 13840 * o match on SH (bit 133), using select/match 2. 13841 * 13842 * Use index 1 to extract the 8-bit receive context from DestQP 13843 * (start at bit 64). Use that as the RSM map table index. 13844 */ 13845 rrd.offset = offset; 13846 rrd.pkt_type = 0; 13847 rrd.field1_off = 95; 13848 rrd.field2_off = 133; 13849 rrd.index1_off = 64; 13850 rrd.index1_width = 8; 13851 rrd.index2_off = 0; 13852 rrd.index2_width = 0; 13853 rrd.mask1 = 1; 13854 rrd.value1 = 1; 13855 rrd.mask2 = 1; 13856 rrd.value2 = 1; 13857 13858 /* add rule 1 */ 13859 add_rsm_rule(dd, 1, &rrd); 13860 13861 rmt->used += dd->num_user_contexts; 13862 } 13863 13864 static void init_rxe(struct hfi1_devdata *dd) 13865 { 13866 struct rsm_map_table *rmt; 13867 13868 /* enable all receive errors */ 13869 write_csr(dd, RCV_ERR_MASK, ~0ull); 13870 13871 rmt = alloc_rsm_map_table(dd); 13872 /* set up QOS, including the QPN map table */ 13873 init_qos(dd, rmt); 13874 init_user_fecn_handling(dd, rmt); 13875 complete_rsm_map_table(dd, rmt); 13876 kfree(rmt); 13877 13878 /* 13879 * make sure RcvCtrl.RcvWcb <= PCIe Device Control 13880 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config 13881 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one 13882 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and 13883 * Max_PayLoad_Size set to its minimum of 128. 13884 * 13885 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0 13886 * (64 bytes). Max_Payload_Size is possibly modified upward in 13887 * tune_pcie_caps() which is called after this routine. 13888 */ 13889 } 13890 13891 static void init_other(struct hfi1_devdata *dd) 13892 { 13893 /* enable all CCE errors */ 13894 write_csr(dd, CCE_ERR_MASK, ~0ull); 13895 /* enable *some* Misc errors */ 13896 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK); 13897 /* enable all DC errors, except LCB */ 13898 write_csr(dd, DCC_ERR_FLG_EN, ~0ull); 13899 write_csr(dd, DC_DC8051_ERR_EN, ~0ull); 13900 } 13901 13902 /* 13903 * Fill out the given AU table using the given CU. A CU is defined in terms 13904 * AUs. The table is a an encoding: given the index, how many AUs does that 13905 * represent? 13906 * 13907 * NOTE: Assumes that the register layout is the same for the 13908 * local and remote tables. 13909 */ 13910 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu, 13911 u32 csr0to3, u32 csr4to7) 13912 { 13913 write_csr(dd, csr0to3, 13914 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT | 13915 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT | 13916 2ull * cu << 13917 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT | 13918 4ull * cu << 13919 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT); 13920 write_csr(dd, csr4to7, 13921 8ull * cu << 13922 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT | 13923 16ull * cu << 13924 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT | 13925 32ull * cu << 13926 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT | 13927 64ull * cu << 13928 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT); 13929 } 13930 13931 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 13932 { 13933 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3, 13934 SEND_CM_LOCAL_AU_TABLE4_TO7); 13935 } 13936 13937 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 13938 { 13939 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3, 13940 SEND_CM_REMOTE_AU_TABLE4_TO7); 13941 } 13942 13943 static void init_txe(struct hfi1_devdata *dd) 13944 { 13945 int i; 13946 13947 /* enable all PIO, SDMA, general, and Egress errors */ 13948 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull); 13949 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull); 13950 write_csr(dd, SEND_ERR_MASK, ~0ull); 13951 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull); 13952 13953 /* enable all per-context and per-SDMA engine errors */ 13954 for (i = 0; i < dd->chip_send_contexts; i++) 13955 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull); 13956 for (i = 0; i < dd->chip_sdma_engines; i++) 13957 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull); 13958 13959 /* set the local CU to AU mapping */ 13960 assign_local_cm_au_table(dd, dd->vcu); 13961 13962 /* 13963 * Set reasonable default for Credit Return Timer 13964 * Don't set on Simulator - causes it to choke. 13965 */ 13966 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR) 13967 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE); 13968 } 13969 13970 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt, u16 jkey) 13971 { 13972 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt]; 13973 unsigned sctxt; 13974 int ret = 0; 13975 u64 reg; 13976 13977 if (!rcd || !rcd->sc) { 13978 ret = -EINVAL; 13979 goto done; 13980 } 13981 sctxt = rcd->sc->hw_context; 13982 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */ 13983 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) << 13984 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT); 13985 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */ 13986 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY)) 13987 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK; 13988 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, reg); 13989 /* 13990 * Enable send-side J_KEY integrity check, unless this is A0 h/w 13991 */ 13992 if (!is_ax(dd)) { 13993 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE); 13994 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 13995 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg); 13996 } 13997 13998 /* Enable J_KEY check on receive context. */ 13999 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK | 14000 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) << 14001 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT); 14002 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, reg); 14003 done: 14004 return ret; 14005 } 14006 14007 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, unsigned ctxt) 14008 { 14009 struct hfi1_ctxtdata *rcd = dd->rcd[ctxt]; 14010 unsigned sctxt; 14011 int ret = 0; 14012 u64 reg; 14013 14014 if (!rcd || !rcd->sc) { 14015 ret = -EINVAL; 14016 goto done; 14017 } 14018 sctxt = rcd->sc->hw_context; 14019 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_JOB_KEY, 0); 14020 /* 14021 * Disable send-side J_KEY integrity check, unless this is A0 h/w. 14022 * This check would not have been enabled for A0 h/w, see 14023 * set_ctxt_jkey(). 14024 */ 14025 if (!is_ax(dd)) { 14026 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE); 14027 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14028 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg); 14029 } 14030 /* Turn off the J_KEY on the receive side */ 14031 write_kctxt_csr(dd, ctxt, RCV_KEY_CTRL, 0); 14032 done: 14033 return ret; 14034 } 14035 14036 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt, u16 pkey) 14037 { 14038 struct hfi1_ctxtdata *rcd; 14039 unsigned sctxt; 14040 int ret = 0; 14041 u64 reg; 14042 14043 if (ctxt < dd->num_rcv_contexts) { 14044 rcd = dd->rcd[ctxt]; 14045 } else { 14046 ret = -EINVAL; 14047 goto done; 14048 } 14049 if (!rcd || !rcd->sc) { 14050 ret = -EINVAL; 14051 goto done; 14052 } 14053 sctxt = rcd->sc->hw_context; 14054 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) << 14055 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT; 14056 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg); 14057 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE); 14058 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14059 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK; 14060 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg); 14061 done: 14062 return ret; 14063 } 14064 14065 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, unsigned ctxt) 14066 { 14067 struct hfi1_ctxtdata *rcd; 14068 unsigned sctxt; 14069 int ret = 0; 14070 u64 reg; 14071 14072 if (ctxt < dd->num_rcv_contexts) { 14073 rcd = dd->rcd[ctxt]; 14074 } else { 14075 ret = -EINVAL; 14076 goto done; 14077 } 14078 if (!rcd || !rcd->sc) { 14079 ret = -EINVAL; 14080 goto done; 14081 } 14082 sctxt = rcd->sc->hw_context; 14083 reg = read_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE); 14084 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14085 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_ENABLE, reg); 14086 write_kctxt_csr(dd, sctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0); 14087 done: 14088 return ret; 14089 } 14090 14091 /* 14092 * Start doing the clean up the the chip. Our clean up happens in multiple 14093 * stages and this is just the first. 14094 */ 14095 void hfi1_start_cleanup(struct hfi1_devdata *dd) 14096 { 14097 aspm_exit(dd); 14098 free_cntrs(dd); 14099 free_rcverr(dd); 14100 clean_up_interrupts(dd); 14101 finish_chip_resources(dd); 14102 } 14103 14104 #define HFI_BASE_GUID(dev) \ 14105 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT)) 14106 14107 /* 14108 * Information can be shared between the two HFIs on the same ASIC 14109 * in the same OS. This function finds the peer device and sets 14110 * up a shared structure. 14111 */ 14112 static int init_asic_data(struct hfi1_devdata *dd) 14113 { 14114 unsigned long flags; 14115 struct hfi1_devdata *tmp, *peer = NULL; 14116 struct hfi1_asic_data *asic_data; 14117 int ret = 0; 14118 14119 /* pre-allocate the asic structure in case we are the first device */ 14120 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL); 14121 if (!asic_data) 14122 return -ENOMEM; 14123 14124 spin_lock_irqsave(&hfi1_devs_lock, flags); 14125 /* Find our peer device */ 14126 list_for_each_entry(tmp, &hfi1_dev_list, list) { 14127 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) && 14128 dd->unit != tmp->unit) { 14129 peer = tmp; 14130 break; 14131 } 14132 } 14133 14134 if (peer) { 14135 /* use already allocated structure */ 14136 dd->asic_data = peer->asic_data; 14137 kfree(asic_data); 14138 } else { 14139 dd->asic_data = asic_data; 14140 mutex_init(&dd->asic_data->asic_resource_mutex); 14141 } 14142 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */ 14143 spin_unlock_irqrestore(&hfi1_devs_lock, flags); 14144 return ret; 14145 } 14146 14147 /* 14148 * Set dd->boardname. Use a generic name if a name is not returned from 14149 * EFI variable space. 14150 * 14151 * Return 0 on success, -ENOMEM if space could not be allocated. 14152 */ 14153 static int obtain_boardname(struct hfi1_devdata *dd) 14154 { 14155 /* generic board description */ 14156 const char generic[] = 14157 "Intel Omni-Path Host Fabric Interface Adapter 100 Series"; 14158 unsigned long size; 14159 int ret; 14160 14161 ret = read_hfi1_efi_var(dd, "description", &size, 14162 (void **)&dd->boardname); 14163 if (ret) { 14164 dd_dev_info(dd, "Board description not found\n"); 14165 /* use generic description */ 14166 dd->boardname = kstrdup(generic, GFP_KERNEL); 14167 if (!dd->boardname) 14168 return -ENOMEM; 14169 } 14170 return 0; 14171 } 14172 14173 /* 14174 * Check the interrupt registers to make sure that they are mapped correctly. 14175 * It is intended to help user identify any mismapping by VMM when the driver 14176 * is running in a VM. This function should only be called before interrupt 14177 * is set up properly. 14178 * 14179 * Return 0 on success, -EINVAL on failure. 14180 */ 14181 static int check_int_registers(struct hfi1_devdata *dd) 14182 { 14183 u64 reg; 14184 u64 all_bits = ~(u64)0; 14185 u64 mask; 14186 14187 /* Clear CceIntMask[0] to avoid raising any interrupts */ 14188 mask = read_csr(dd, CCE_INT_MASK); 14189 write_csr(dd, CCE_INT_MASK, 0ull); 14190 reg = read_csr(dd, CCE_INT_MASK); 14191 if (reg) 14192 goto err_exit; 14193 14194 /* Clear all interrupt status bits */ 14195 write_csr(dd, CCE_INT_CLEAR, all_bits); 14196 reg = read_csr(dd, CCE_INT_STATUS); 14197 if (reg) 14198 goto err_exit; 14199 14200 /* Set all interrupt status bits */ 14201 write_csr(dd, CCE_INT_FORCE, all_bits); 14202 reg = read_csr(dd, CCE_INT_STATUS); 14203 if (reg != all_bits) 14204 goto err_exit; 14205 14206 /* Restore the interrupt mask */ 14207 write_csr(dd, CCE_INT_CLEAR, all_bits); 14208 write_csr(dd, CCE_INT_MASK, mask); 14209 14210 return 0; 14211 err_exit: 14212 write_csr(dd, CCE_INT_MASK, mask); 14213 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n"); 14214 return -EINVAL; 14215 } 14216 14217 /** 14218 * Allocate and initialize the device structure for the hfi. 14219 * @dev: the pci_dev for hfi1_ib device 14220 * @ent: pci_device_id struct for this dev 14221 * 14222 * Also allocates, initializes, and returns the devdata struct for this 14223 * device instance 14224 * 14225 * This is global, and is called directly at init to set up the 14226 * chip-specific function pointers for later use. 14227 */ 14228 struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev, 14229 const struct pci_device_id *ent) 14230 { 14231 struct hfi1_devdata *dd; 14232 struct hfi1_pportdata *ppd; 14233 u64 reg; 14234 int i, ret; 14235 static const char * const inames[] = { /* implementation names */ 14236 "RTL silicon", 14237 "RTL VCS simulation", 14238 "RTL FPGA emulation", 14239 "Functional simulator" 14240 }; 14241 struct pci_dev *parent = pdev->bus->self; 14242 14243 dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS * 14244 sizeof(struct hfi1_pportdata)); 14245 if (IS_ERR(dd)) 14246 goto bail; 14247 ppd = dd->pport; 14248 for (i = 0; i < dd->num_pports; i++, ppd++) { 14249 int vl; 14250 /* init common fields */ 14251 hfi1_init_pportdata(pdev, ppd, dd, 0, 1); 14252 /* DC supports 4 link widths */ 14253 ppd->link_width_supported = 14254 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X | 14255 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X; 14256 ppd->link_width_downgrade_supported = 14257 ppd->link_width_supported; 14258 /* start out enabling only 4X */ 14259 ppd->link_width_enabled = OPA_LINK_WIDTH_4X; 14260 ppd->link_width_downgrade_enabled = 14261 ppd->link_width_downgrade_supported; 14262 /* link width active is 0 when link is down */ 14263 /* link width downgrade active is 0 when link is down */ 14264 14265 if (num_vls < HFI1_MIN_VLS_SUPPORTED || 14266 num_vls > HFI1_MAX_VLS_SUPPORTED) { 14267 hfi1_early_err(&pdev->dev, 14268 "Invalid num_vls %u, using %u VLs\n", 14269 num_vls, HFI1_MAX_VLS_SUPPORTED); 14270 num_vls = HFI1_MAX_VLS_SUPPORTED; 14271 } 14272 ppd->vls_supported = num_vls; 14273 ppd->vls_operational = ppd->vls_supported; 14274 ppd->actual_vls_operational = ppd->vls_supported; 14275 /* Set the default MTU. */ 14276 for (vl = 0; vl < num_vls; vl++) 14277 dd->vld[vl].mtu = hfi1_max_mtu; 14278 dd->vld[15].mtu = MAX_MAD_PACKET; 14279 /* 14280 * Set the initial values to reasonable default, will be set 14281 * for real when link is up. 14282 */ 14283 ppd->lstate = IB_PORT_DOWN; 14284 ppd->overrun_threshold = 0x4; 14285 ppd->phy_error_threshold = 0xf; 14286 ppd->port_crc_mode_enabled = link_crc_mask; 14287 /* initialize supported LTP CRC mode */ 14288 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 14289 /* initialize enabled LTP CRC mode */ 14290 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4; 14291 /* start in offline */ 14292 ppd->host_link_state = HLS_DN_OFFLINE; 14293 init_vl_arb_caches(ppd); 14294 ppd->last_pstate = 0xff; /* invalid value */ 14295 } 14296 14297 dd->link_default = HLS_DN_POLL; 14298 14299 /* 14300 * Do remaining PCIe setup and save PCIe values in dd. 14301 * Any error printing is already done by the init code. 14302 * On return, we have the chip mapped. 14303 */ 14304 ret = hfi1_pcie_ddinit(dd, pdev, ent); 14305 if (ret < 0) 14306 goto bail_free; 14307 14308 /* verify that reads actually work, save revision for reset check */ 14309 dd->revision = read_csr(dd, CCE_REVISION); 14310 if (dd->revision == ~(u64)0) { 14311 dd_dev_err(dd, "cannot read chip CSRs\n"); 14312 ret = -EINVAL; 14313 goto bail_cleanup; 14314 } 14315 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT) 14316 & CCE_REVISION_CHIP_REV_MAJOR_MASK; 14317 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT) 14318 & CCE_REVISION_CHIP_REV_MINOR_MASK; 14319 14320 /* 14321 * Check interrupt registers mapping if the driver has no access to 14322 * the upstream component. In this case, it is likely that the driver 14323 * is running in a VM. 14324 */ 14325 if (!parent) { 14326 ret = check_int_registers(dd); 14327 if (ret) 14328 goto bail_cleanup; 14329 } 14330 14331 /* 14332 * obtain the hardware ID - NOT related to unit, which is a 14333 * software enumeration 14334 */ 14335 reg = read_csr(dd, CCE_REVISION2); 14336 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT) 14337 & CCE_REVISION2_HFI_ID_MASK; 14338 /* the variable size will remove unwanted bits */ 14339 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT; 14340 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT; 14341 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n", 14342 dd->icode < ARRAY_SIZE(inames) ? 14343 inames[dd->icode] : "unknown", (int)dd->irev); 14344 14345 /* speeds the hardware can support */ 14346 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G; 14347 /* speeds allowed to run at */ 14348 dd->pport->link_speed_enabled = dd->pport->link_speed_supported; 14349 /* give a reasonable active value, will be set on link up */ 14350 dd->pport->link_speed_active = OPA_LINK_SPEED_25G; 14351 14352 dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS); 14353 dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS); 14354 dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES); 14355 dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE); 14356 dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE); 14357 /* fix up link widths for emulation _p */ 14358 ppd = dd->pport; 14359 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) { 14360 ppd->link_width_supported = 14361 ppd->link_width_enabled = 14362 ppd->link_width_downgrade_supported = 14363 ppd->link_width_downgrade_enabled = 14364 OPA_LINK_WIDTH_1X; 14365 } 14366 /* insure num_vls isn't larger than number of sdma engines */ 14367 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) { 14368 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n", 14369 num_vls, dd->chip_sdma_engines); 14370 num_vls = dd->chip_sdma_engines; 14371 ppd->vls_supported = dd->chip_sdma_engines; 14372 ppd->vls_operational = ppd->vls_supported; 14373 } 14374 14375 /* 14376 * Convert the ns parameter to the 64 * cclocks used in the CSR. 14377 * Limit the max if larger than the field holds. If timeout is 14378 * non-zero, then the calculated field will be at least 1. 14379 * 14380 * Must be after icode is set up - the cclock rate depends 14381 * on knowing the hardware being used. 14382 */ 14383 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64; 14384 if (dd->rcv_intr_timeout_csr > 14385 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK) 14386 dd->rcv_intr_timeout_csr = 14387 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK; 14388 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout) 14389 dd->rcv_intr_timeout_csr = 1; 14390 14391 /* needs to be done before we look for the peer device */ 14392 read_guid(dd); 14393 14394 /* set up shared ASIC data with peer device */ 14395 ret = init_asic_data(dd); 14396 if (ret) 14397 goto bail_cleanup; 14398 14399 /* obtain chip sizes, reset chip CSRs */ 14400 init_chip(dd); 14401 14402 /* read in the PCIe link speed information */ 14403 ret = pcie_speeds(dd); 14404 if (ret) 14405 goto bail_cleanup; 14406 14407 /* Needs to be called before hfi1_firmware_init */ 14408 get_platform_config(dd); 14409 14410 /* read in firmware */ 14411 ret = hfi1_firmware_init(dd); 14412 if (ret) 14413 goto bail_cleanup; 14414 14415 /* 14416 * In general, the PCIe Gen3 transition must occur after the 14417 * chip has been idled (so it won't initiate any PCIe transactions 14418 * e.g. an interrupt) and before the driver changes any registers 14419 * (the transition will reset the registers). 14420 * 14421 * In particular, place this call after: 14422 * - init_chip() - the chip will not initiate any PCIe transactions 14423 * - pcie_speeds() - reads the current link speed 14424 * - hfi1_firmware_init() - the needed firmware is ready to be 14425 * downloaded 14426 */ 14427 ret = do_pcie_gen3_transition(dd); 14428 if (ret) 14429 goto bail_cleanup; 14430 14431 /* start setting dd values and adjusting CSRs */ 14432 init_early_variables(dd); 14433 14434 parse_platform_config(dd); 14435 14436 ret = obtain_boardname(dd); 14437 if (ret) 14438 goto bail_cleanup; 14439 14440 snprintf(dd->boardversion, BOARD_VERS_MAX, 14441 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n", 14442 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN, 14443 (u32)dd->majrev, 14444 (u32)dd->minrev, 14445 (dd->revision >> CCE_REVISION_SW_SHIFT) 14446 & CCE_REVISION_SW_MASK); 14447 14448 /* 14449 * The real cpu mask is part of the affinity struct but has to be 14450 * initialized earlier than the rest of the affinity struct because it 14451 * is needed to calculate the number of user contexts in 14452 * set_up_context_variables(). However, hfi1_dev_affinity_init(), 14453 * which initializes the rest of the affinity struct members, 14454 * depends on set_up_context_variables() for the number of kernel 14455 * contexts, so it cannot be called before set_up_context_variables(). 14456 */ 14457 ret = init_real_cpu_mask(dd); 14458 if (ret) 14459 goto bail_cleanup; 14460 14461 ret = set_up_context_variables(dd); 14462 if (ret) 14463 goto bail_cleanup; 14464 14465 /* set initial RXE CSRs */ 14466 init_rxe(dd); 14467 /* set initial TXE CSRs */ 14468 init_txe(dd); 14469 /* set initial non-RXE, non-TXE CSRs */ 14470 init_other(dd); 14471 /* set up KDETH QP prefix in both RX and TX CSRs */ 14472 init_kdeth_qp(dd); 14473 14474 hfi1_dev_affinity_init(dd); 14475 14476 /* send contexts must be set up before receive contexts */ 14477 ret = init_send_contexts(dd); 14478 if (ret) 14479 goto bail_cleanup; 14480 14481 ret = hfi1_create_ctxts(dd); 14482 if (ret) 14483 goto bail_cleanup; 14484 14485 dd->rcvhdrsize = DEFAULT_RCVHDRSIZE; 14486 /* 14487 * rcd[0] is guaranteed to be valid by this point. Also, all 14488 * context are using the same value, as per the module parameter. 14489 */ 14490 dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32); 14491 14492 ret = init_pervl_scs(dd); 14493 if (ret) 14494 goto bail_cleanup; 14495 14496 /* sdma init */ 14497 for (i = 0; i < dd->num_pports; ++i) { 14498 ret = sdma_init(dd, i); 14499 if (ret) 14500 goto bail_cleanup; 14501 } 14502 14503 /* use contexts created by hfi1_create_ctxts */ 14504 ret = set_up_interrupts(dd); 14505 if (ret) 14506 goto bail_cleanup; 14507 14508 /* set up LCB access - must be after set_up_interrupts() */ 14509 init_lcb_access(dd); 14510 14511 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n", 14512 dd->base_guid & 0xFFFFFF); 14513 14514 dd->oui1 = dd->base_guid >> 56 & 0xFF; 14515 dd->oui2 = dd->base_guid >> 48 & 0xFF; 14516 dd->oui3 = dd->base_guid >> 40 & 0xFF; 14517 14518 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */ 14519 if (ret) 14520 goto bail_clear_intr; 14521 check_fabric_firmware_versions(dd); 14522 14523 thermal_init(dd); 14524 14525 ret = init_cntrs(dd); 14526 if (ret) 14527 goto bail_clear_intr; 14528 14529 ret = init_rcverr(dd); 14530 if (ret) 14531 goto bail_free_cntrs; 14532 14533 ret = eprom_init(dd); 14534 if (ret) 14535 goto bail_free_rcverr; 14536 14537 goto bail; 14538 14539 bail_free_rcverr: 14540 free_rcverr(dd); 14541 bail_free_cntrs: 14542 free_cntrs(dd); 14543 bail_clear_intr: 14544 clean_up_interrupts(dd); 14545 bail_cleanup: 14546 hfi1_pcie_ddcleanup(dd); 14547 bail_free: 14548 hfi1_free_devdata(dd); 14549 dd = ERR_PTR(ret); 14550 bail: 14551 return dd; 14552 } 14553 14554 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate, 14555 u32 dw_len) 14556 { 14557 u32 delta_cycles; 14558 u32 current_egress_rate = ppd->current_egress_rate; 14559 /* rates here are in units of 10^6 bits/sec */ 14560 14561 if (desired_egress_rate == -1) 14562 return 0; /* shouldn't happen */ 14563 14564 if (desired_egress_rate >= current_egress_rate) 14565 return 0; /* we can't help go faster, only slower */ 14566 14567 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) - 14568 egress_cycles(dw_len * 4, current_egress_rate); 14569 14570 return (u16)delta_cycles; 14571 } 14572 14573 /** 14574 * create_pbc - build a pbc for transmission 14575 * @flags: special case flags or-ed in built pbc 14576 * @srate: static rate 14577 * @vl: vl 14578 * @dwlen: dword length (header words + data words + pbc words) 14579 * 14580 * Create a PBC with the given flags, rate, VL, and length. 14581 * 14582 * NOTE: The PBC created will not insert any HCRC - all callers but one are 14583 * for verbs, which does not use this PSM feature. The lone other caller 14584 * is for the diagnostic interface which calls this if the user does not 14585 * supply their own PBC. 14586 */ 14587 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, 14588 u32 dw_len) 14589 { 14590 u64 pbc, delay = 0; 14591 14592 if (unlikely(srate_mbs)) 14593 delay = delay_cycles(ppd, srate_mbs, dw_len); 14594 14595 pbc = flags 14596 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT) 14597 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT) 14598 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT 14599 | (dw_len & PBC_LENGTH_DWS_MASK) 14600 << PBC_LENGTH_DWS_SHIFT; 14601 14602 return pbc; 14603 } 14604 14605 #define SBUS_THERMAL 0x4f 14606 #define SBUS_THERM_MONITOR_MODE 0x1 14607 14608 #define THERM_FAILURE(dev, ret, reason) \ 14609 dd_dev_err((dd), \ 14610 "Thermal sensor initialization failed: %s (%d)\n", \ 14611 (reason), (ret)) 14612 14613 /* 14614 * Initialize the thermal sensor. 14615 * 14616 * After initialization, enable polling of thermal sensor through 14617 * SBus interface. In order for this to work, the SBus Master 14618 * firmware has to be loaded due to the fact that the HW polling 14619 * logic uses SBus interrupts, which are not supported with 14620 * default firmware. Otherwise, no data will be returned through 14621 * the ASIC_STS_THERM CSR. 14622 */ 14623 static int thermal_init(struct hfi1_devdata *dd) 14624 { 14625 int ret = 0; 14626 14627 if (dd->icode != ICODE_RTL_SILICON || 14628 check_chip_resource(dd, CR_THERM_INIT, NULL)) 14629 return ret; 14630 14631 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT); 14632 if (ret) { 14633 THERM_FAILURE(dd, ret, "Acquire SBus"); 14634 return ret; 14635 } 14636 14637 dd_dev_info(dd, "Initializing thermal sensor\n"); 14638 /* Disable polling of thermal readings */ 14639 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0); 14640 msleep(100); 14641 /* Thermal Sensor Initialization */ 14642 /* Step 1: Reset the Thermal SBus Receiver */ 14643 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 14644 RESET_SBUS_RECEIVER, 0); 14645 if (ret) { 14646 THERM_FAILURE(dd, ret, "Bus Reset"); 14647 goto done; 14648 } 14649 /* Step 2: Set Reset bit in Thermal block */ 14650 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 14651 WRITE_SBUS_RECEIVER, 0x1); 14652 if (ret) { 14653 THERM_FAILURE(dd, ret, "Therm Block Reset"); 14654 goto done; 14655 } 14656 /* Step 3: Write clock divider value (100MHz -> 2MHz) */ 14657 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1, 14658 WRITE_SBUS_RECEIVER, 0x32); 14659 if (ret) { 14660 THERM_FAILURE(dd, ret, "Write Clock Div"); 14661 goto done; 14662 } 14663 /* Step 4: Select temperature mode */ 14664 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3, 14665 WRITE_SBUS_RECEIVER, 14666 SBUS_THERM_MONITOR_MODE); 14667 if (ret) { 14668 THERM_FAILURE(dd, ret, "Write Mode Sel"); 14669 goto done; 14670 } 14671 /* Step 5: De-assert block reset and start conversion */ 14672 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 14673 WRITE_SBUS_RECEIVER, 0x2); 14674 if (ret) { 14675 THERM_FAILURE(dd, ret, "Write Reset Deassert"); 14676 goto done; 14677 } 14678 /* Step 5.1: Wait for first conversion (21.5ms per spec) */ 14679 msleep(22); 14680 14681 /* Enable polling of thermal readings */ 14682 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1); 14683 14684 /* Set initialized flag */ 14685 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0); 14686 if (ret) 14687 THERM_FAILURE(dd, ret, "Unable to set thermal init flag"); 14688 14689 done: 14690 release_chip_resource(dd, CR_SBUS); 14691 return ret; 14692 } 14693 14694 static void handle_temp_err(struct hfi1_devdata *dd) 14695 { 14696 struct hfi1_pportdata *ppd = &dd->pport[0]; 14697 /* 14698 * Thermal Critical Interrupt 14699 * Put the device into forced freeze mode, take link down to 14700 * offline, and put DC into reset. 14701 */ 14702 dd_dev_emerg(dd, 14703 "Critical temperature reached! Forcing device into freeze mode!\n"); 14704 dd->flags |= HFI1_FORCED_FREEZE; 14705 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT); 14706 /* 14707 * Shut DC down as much and as quickly as possible. 14708 * 14709 * Step 1: Take the link down to OFFLINE. This will cause the 14710 * 8051 to put the Serdes in reset. However, we don't want to 14711 * go through the entire link state machine since we want to 14712 * shutdown ASAP. Furthermore, this is not a graceful shutdown 14713 * but rather an attempt to save the chip. 14714 * Code below is almost the same as quiet_serdes() but avoids 14715 * all the extra work and the sleeps. 14716 */ 14717 ppd->driver_link_ready = 0; 14718 ppd->link_enabled = 0; 14719 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) | 14720 PLS_OFFLINE); 14721 /* 14722 * Step 2: Shutdown LCB and 8051 14723 * After shutdown, do not restore DC_CFG_RESET value. 14724 */ 14725 dc_shutdown(dd); 14726 } 14727