1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause 2 /* 3 * Copyright(c) 2015 - 2020 Intel Corporation. 4 */ 5 6 /* 7 * This file contains all of the code that is specific to the HFI chip 8 */ 9 10 #include <linux/pci.h> 11 #include <linux/delay.h> 12 #include <linux/interrupt.h> 13 #include <linux/module.h> 14 15 #include "hfi.h" 16 #include "trace.h" 17 #include "mad.h" 18 #include "pio.h" 19 #include "sdma.h" 20 #include "eprom.h" 21 #include "efivar.h" 22 #include "platform.h" 23 #include "aspm.h" 24 #include "affinity.h" 25 #include "debugfs.h" 26 #include "fault.h" 27 #include "netdev.h" 28 29 uint num_vls = HFI1_MAX_VLS_SUPPORTED; 30 module_param(num_vls, uint, S_IRUGO); 31 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)"); 32 33 /* 34 * Default time to aggregate two 10K packets from the idle state 35 * (timer not running). The timer starts at the end of the first packet, 36 * so only the time for one 10K packet and header plus a bit extra is needed. 37 * 10 * 1024 + 64 header byte = 10304 byte 38 * 10304 byte / 12.5 GB/s = 824.32ns 39 */ 40 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */ 41 module_param(rcv_intr_timeout, uint, S_IRUGO); 42 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns"); 43 44 uint rcv_intr_count = 16; /* same as qib */ 45 module_param(rcv_intr_count, uint, S_IRUGO); 46 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count"); 47 48 ushort link_crc_mask = SUPPORTED_CRCS; 49 module_param(link_crc_mask, ushort, S_IRUGO); 50 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link"); 51 52 uint loopback; 53 module_param_named(loopback, loopback, uint, S_IRUGO); 54 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable"); 55 56 /* Other driver tunables */ 57 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/ 58 static ushort crc_14b_sideband = 1; 59 static uint use_flr = 1; 60 uint quick_linkup; /* skip LNI */ 61 62 struct flag_table { 63 u64 flag; /* the flag */ 64 char *str; /* description string */ 65 u16 extra; /* extra information */ 66 u16 unused0; 67 u32 unused1; 68 }; 69 70 /* str must be a string constant */ 71 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra} 72 #define FLAG_ENTRY0(str, flag) {flag, str, 0} 73 74 /* Send Error Consequences */ 75 #define SEC_WRITE_DROPPED 0x1 76 #define SEC_PACKET_DROPPED 0x2 77 #define SEC_SC_HALTED 0x4 /* per-context only */ 78 #define SEC_SPC_FREEZE 0x8 /* per-HFI only */ 79 80 #define DEFAULT_KRCVQS 2 81 #define MIN_KERNEL_KCTXTS 2 82 #define FIRST_KERNEL_KCTXT 1 83 84 /* 85 * RSM instance allocation 86 * 0 - User Fecn Handling 87 * 1 - Vnic 88 * 2 - AIP 89 * 3 - Verbs 90 */ 91 #define RSM_INS_FECN 0 92 #define RSM_INS_VNIC 1 93 #define RSM_INS_AIP 2 94 #define RSM_INS_VERBS 3 95 96 /* Bit offset into the GUID which carries HFI id information */ 97 #define GUID_HFI_INDEX_SHIFT 39 98 99 /* extract the emulation revision */ 100 #define emulator_rev(dd) ((dd)->irev >> 8) 101 /* parallel and serial emulation versions are 3 and 4 respectively */ 102 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3) 103 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4) 104 105 /* RSM fields for Verbs */ 106 /* packet type */ 107 #define IB_PACKET_TYPE 2ull 108 #define QW_SHIFT 6ull 109 /* QPN[7..1] */ 110 #define QPN_WIDTH 7ull 111 112 /* LRH.BTH: QW 0, OFFSET 48 - for match */ 113 #define LRH_BTH_QW 0ull 114 #define LRH_BTH_BIT_OFFSET 48ull 115 #define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off)) 116 #define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET) 117 #define LRH_BTH_SELECT 118 #define LRH_BTH_MASK 3ull 119 #define LRH_BTH_VALUE 2ull 120 121 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */ 122 #define LRH_SC_QW 0ull 123 #define LRH_SC_BIT_OFFSET 56ull 124 #define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off)) 125 #define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET) 126 #define LRH_SC_MASK 128ull 127 #define LRH_SC_VALUE 0ull 128 129 /* SC[n..0] QW 0, OFFSET 60 - for select */ 130 #define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull)) 131 132 /* QPN[m+n:1] QW 1, OFFSET 1 */ 133 #define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull)) 134 135 /* RSM fields for AIP */ 136 /* LRH.BTH above is reused for this rule */ 137 138 /* BTH.DESTQP: QW 1, OFFSET 16 for match */ 139 #define BTH_DESTQP_QW 1ull 140 #define BTH_DESTQP_BIT_OFFSET 16ull 141 #define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW << QW_SHIFT) | (off)) 142 #define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET) 143 #define BTH_DESTQP_MASK 0xFFull 144 #define BTH_DESTQP_VALUE 0x81ull 145 146 /* DETH.SQPN: QW 1 Offset 56 for select */ 147 /* We use 8 most significant Soure QPN bits as entropy fpr AIP */ 148 #define DETH_AIP_SQPN_QW 3ull 149 #define DETH_AIP_SQPN_BIT_OFFSET 56ull 150 #define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW << QW_SHIFT) | (off)) 151 #define DETH_AIP_SQPN_SELECT_OFFSET \ 152 DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET) 153 154 /* RSM fields for Vnic */ 155 /* L2_TYPE: QW 0, OFFSET 61 - for match */ 156 #define L2_TYPE_QW 0ull 157 #define L2_TYPE_BIT_OFFSET 61ull 158 #define L2_TYPE_OFFSET(off) ((L2_TYPE_QW << QW_SHIFT) | (off)) 159 #define L2_TYPE_MATCH_OFFSET L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET) 160 #define L2_TYPE_MASK 3ull 161 #define L2_16B_VALUE 2ull 162 163 /* L4_TYPE QW 1, OFFSET 0 - for match */ 164 #define L4_TYPE_QW 1ull 165 #define L4_TYPE_BIT_OFFSET 0ull 166 #define L4_TYPE_OFFSET(off) ((L4_TYPE_QW << QW_SHIFT) | (off)) 167 #define L4_TYPE_MATCH_OFFSET L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET) 168 #define L4_16B_TYPE_MASK 0xFFull 169 #define L4_16B_ETH_VALUE 0x78ull 170 171 /* 16B VESWID - for select */ 172 #define L4_16B_HDR_VESWID_OFFSET ((2 << QW_SHIFT) | (16ull)) 173 /* 16B ENTROPY - for select */ 174 #define L2_16B_ENTROPY_OFFSET ((1 << QW_SHIFT) | (32ull)) 175 176 /* defines to build power on SC2VL table */ 177 #define SC2VL_VAL( \ 178 num, \ 179 sc0, sc0val, \ 180 sc1, sc1val, \ 181 sc2, sc2val, \ 182 sc3, sc3val, \ 183 sc4, sc4val, \ 184 sc5, sc5val, \ 185 sc6, sc6val, \ 186 sc7, sc7val) \ 187 ( \ 188 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \ 189 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \ 190 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \ 191 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \ 192 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \ 193 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \ 194 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \ 195 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \ 196 ) 197 198 #define DC_SC_VL_VAL( \ 199 range, \ 200 e0, e0val, \ 201 e1, e1val, \ 202 e2, e2val, \ 203 e3, e3val, \ 204 e4, e4val, \ 205 e5, e5val, \ 206 e6, e6val, \ 207 e7, e7val, \ 208 e8, e8val, \ 209 e9, e9val, \ 210 e10, e10val, \ 211 e11, e11val, \ 212 e12, e12val, \ 213 e13, e13val, \ 214 e14, e14val, \ 215 e15, e15val) \ 216 ( \ 217 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \ 218 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \ 219 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \ 220 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \ 221 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \ 222 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \ 223 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \ 224 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \ 225 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \ 226 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \ 227 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \ 228 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \ 229 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \ 230 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \ 231 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \ 232 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \ 233 ) 234 235 /* all CceStatus sub-block freeze bits */ 236 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \ 237 | CCE_STATUS_RXE_FROZE_SMASK \ 238 | CCE_STATUS_TXE_FROZE_SMASK \ 239 | CCE_STATUS_TXE_PIO_FROZE_SMASK) 240 /* all CceStatus sub-block TXE pause bits */ 241 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \ 242 | CCE_STATUS_TXE_PAUSED_SMASK \ 243 | CCE_STATUS_SDMA_PAUSED_SMASK) 244 /* all CceStatus sub-block RXE pause bits */ 245 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK 246 247 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL 248 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF 249 250 /* 251 * CCE Error flags. 252 */ 253 static struct flag_table cce_err_status_flags[] = { 254 /* 0*/ FLAG_ENTRY0("CceCsrParityErr", 255 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK), 256 /* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr", 257 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK), 258 /* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr", 259 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK), 260 /* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr", 261 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK), 262 /* 4*/ FLAG_ENTRY0("CceTrgtAccessErr", 263 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK), 264 /* 5*/ FLAG_ENTRY0("CceRspdDataParityErr", 265 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK), 266 /* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr", 267 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK), 268 /* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr", 269 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK), 270 /* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr", 271 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK), 272 /* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 273 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK), 274 /*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 275 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK), 276 /*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError", 277 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK), 278 /*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError", 279 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK), 280 /*13*/ FLAG_ENTRY0("PcicRetryMemCorErr", 281 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK), 282 /*14*/ FLAG_ENTRY0("PcicRetryMemCorErr", 283 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK), 284 /*15*/ FLAG_ENTRY0("PcicPostHdQCorErr", 285 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK), 286 /*16*/ FLAG_ENTRY0("PcicPostHdQCorErr", 287 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK), 288 /*17*/ FLAG_ENTRY0("PcicPostHdQCorErr", 289 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK), 290 /*18*/ FLAG_ENTRY0("PcicCplDatQCorErr", 291 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK), 292 /*19*/ FLAG_ENTRY0("PcicNPostHQParityErr", 293 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK), 294 /*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr", 295 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK), 296 /*21*/ FLAG_ENTRY0("PcicRetryMemUncErr", 297 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK), 298 /*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr", 299 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK), 300 /*23*/ FLAG_ENTRY0("PcicPostHdQUncErr", 301 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK), 302 /*24*/ FLAG_ENTRY0("PcicPostDatQUncErr", 303 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK), 304 /*25*/ FLAG_ENTRY0("PcicCplHdQUncErr", 305 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK), 306 /*26*/ FLAG_ENTRY0("PcicCplDatQUncErr", 307 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK), 308 /*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr", 309 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK), 310 /*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr", 311 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK), 312 /*29*/ FLAG_ENTRY0("PcicReceiveParityErr", 313 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK), 314 /*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr", 315 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK), 316 /*31*/ FLAG_ENTRY0("LATriggered", 317 CCE_ERR_STATUS_LA_TRIGGERED_SMASK), 318 /*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr", 319 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK), 320 /*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr", 321 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK), 322 /*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr", 323 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK), 324 /*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr", 325 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK), 326 /*36*/ FLAG_ENTRY0("CceMsixTableCorErr", 327 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK), 328 /*37*/ FLAG_ENTRY0("CceMsixTableUncErr", 329 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK), 330 /*38*/ FLAG_ENTRY0("CceIntMapCorErr", 331 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK), 332 /*39*/ FLAG_ENTRY0("CceIntMapUncErr", 333 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK), 334 /*40*/ FLAG_ENTRY0("CceMsixCsrParityErr", 335 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK), 336 /*41-63 reserved*/ 337 }; 338 339 /* 340 * Misc Error flags 341 */ 342 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK 343 static struct flag_table misc_err_status_flags[] = { 344 /* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)), 345 /* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)), 346 /* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)), 347 /* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)), 348 /* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)), 349 /* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)), 350 /* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)), 351 /* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)), 352 /* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)), 353 /* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)), 354 /*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)), 355 /*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)), 356 /*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL)) 357 }; 358 359 /* 360 * TXE PIO Error flags and consequences 361 */ 362 static struct flag_table pio_err_status_flags[] = { 363 /* 0*/ FLAG_ENTRY("PioWriteBadCtxt", 364 SEC_WRITE_DROPPED, 365 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK), 366 /* 1*/ FLAG_ENTRY("PioWriteAddrParity", 367 SEC_SPC_FREEZE, 368 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK), 369 /* 2*/ FLAG_ENTRY("PioCsrParity", 370 SEC_SPC_FREEZE, 371 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK), 372 /* 3*/ FLAG_ENTRY("PioSbMemFifo0", 373 SEC_SPC_FREEZE, 374 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK), 375 /* 4*/ FLAG_ENTRY("PioSbMemFifo1", 376 SEC_SPC_FREEZE, 377 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK), 378 /* 5*/ FLAG_ENTRY("PioPccFifoParity", 379 SEC_SPC_FREEZE, 380 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK), 381 /* 6*/ FLAG_ENTRY("PioPecFifoParity", 382 SEC_SPC_FREEZE, 383 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK), 384 /* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity", 385 SEC_SPC_FREEZE, 386 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK), 387 /* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity", 388 SEC_SPC_FREEZE, 389 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK), 390 /* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr", 391 SEC_SPC_FREEZE, 392 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK), 393 /*10*/ FLAG_ENTRY("PioSmPktResetParity", 394 SEC_SPC_FREEZE, 395 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK), 396 /*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc", 397 SEC_SPC_FREEZE, 398 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK), 399 /*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc", 400 SEC_SPC_FREEZE, 401 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK), 402 /*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor", 403 0, 404 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK), 405 /*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor", 406 0, 407 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK), 408 /*15*/ FLAG_ENTRY("PioCreditRetFifoParity", 409 SEC_SPC_FREEZE, 410 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK), 411 /*16*/ FLAG_ENTRY("PioPpmcPblFifo", 412 SEC_SPC_FREEZE, 413 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK), 414 /*17*/ FLAG_ENTRY("PioInitSmIn", 415 0, 416 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK), 417 /*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm", 418 SEC_SPC_FREEZE, 419 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK), 420 /*19*/ FLAG_ENTRY("PioHostAddrMemUnc", 421 SEC_SPC_FREEZE, 422 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK), 423 /*20*/ FLAG_ENTRY("PioHostAddrMemCor", 424 0, 425 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK), 426 /*21*/ FLAG_ENTRY("PioWriteDataParity", 427 SEC_SPC_FREEZE, 428 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK), 429 /*22*/ FLAG_ENTRY("PioStateMachine", 430 SEC_SPC_FREEZE, 431 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK), 432 /*23*/ FLAG_ENTRY("PioWriteQwValidParity", 433 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 434 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK), 435 /*24*/ FLAG_ENTRY("PioBlockQwCountParity", 436 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 437 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK), 438 /*25*/ FLAG_ENTRY("PioVlfVlLenParity", 439 SEC_SPC_FREEZE, 440 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK), 441 /*26*/ FLAG_ENTRY("PioVlfSopParity", 442 SEC_SPC_FREEZE, 443 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK), 444 /*27*/ FLAG_ENTRY("PioVlFifoParity", 445 SEC_SPC_FREEZE, 446 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK), 447 /*28*/ FLAG_ENTRY("PioPpmcBqcMemParity", 448 SEC_SPC_FREEZE, 449 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK), 450 /*29*/ FLAG_ENTRY("PioPpmcSopLen", 451 SEC_SPC_FREEZE, 452 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK), 453 /*30-31 reserved*/ 454 /*32*/ FLAG_ENTRY("PioCurrentFreeCntParity", 455 SEC_SPC_FREEZE, 456 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK), 457 /*33*/ FLAG_ENTRY("PioLastReturnedCntParity", 458 SEC_SPC_FREEZE, 459 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK), 460 /*34*/ FLAG_ENTRY("PioPccSopHeadParity", 461 SEC_SPC_FREEZE, 462 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK), 463 /*35*/ FLAG_ENTRY("PioPecSopHeadParityErr", 464 SEC_SPC_FREEZE, 465 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK), 466 /*36-63 reserved*/ 467 }; 468 469 /* TXE PIO errors that cause an SPC freeze */ 470 #define ALL_PIO_FREEZE_ERR \ 471 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \ 472 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \ 473 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \ 474 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \ 475 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \ 476 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \ 477 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \ 478 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \ 479 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \ 480 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \ 481 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \ 482 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \ 483 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \ 484 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \ 485 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \ 486 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \ 487 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \ 488 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \ 489 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \ 490 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \ 491 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \ 492 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \ 493 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \ 494 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \ 495 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \ 496 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \ 497 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \ 498 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \ 499 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK) 500 501 /* 502 * TXE SDMA Error flags 503 */ 504 static struct flag_table sdma_err_status_flags[] = { 505 /* 0*/ FLAG_ENTRY0("SDmaRpyTagErr", 506 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK), 507 /* 1*/ FLAG_ENTRY0("SDmaCsrParityErr", 508 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK), 509 /* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr", 510 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK), 511 /* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr", 512 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK), 513 /*04-63 reserved*/ 514 }; 515 516 /* TXE SDMA errors that cause an SPC freeze */ 517 #define ALL_SDMA_FREEZE_ERR \ 518 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \ 519 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \ 520 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK) 521 522 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */ 523 #define PORT_DISCARD_EGRESS_ERRS \ 524 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \ 525 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \ 526 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK) 527 528 /* 529 * TXE Egress Error flags 530 */ 531 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK 532 static struct flag_table egress_err_status_flags[] = { 533 /* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)), 534 /* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)), 535 /* 2 reserved */ 536 /* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr", 537 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)), 538 /* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)), 539 /* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)), 540 /* 6 reserved */ 541 /* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr", 542 SEES(TX_PIO_LAUNCH_INTF_PARITY)), 543 /* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr", 544 SEES(TX_SDMA_LAUNCH_INTF_PARITY)), 545 /* 9-10 reserved */ 546 /*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr", 547 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)), 548 /*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)), 549 /*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)), 550 /*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)), 551 /*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)), 552 /*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr", 553 SEES(TX_SDMA0_DISALLOWED_PACKET)), 554 /*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr", 555 SEES(TX_SDMA1_DISALLOWED_PACKET)), 556 /*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr", 557 SEES(TX_SDMA2_DISALLOWED_PACKET)), 558 /*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr", 559 SEES(TX_SDMA3_DISALLOWED_PACKET)), 560 /*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr", 561 SEES(TX_SDMA4_DISALLOWED_PACKET)), 562 /*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr", 563 SEES(TX_SDMA5_DISALLOWED_PACKET)), 564 /*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr", 565 SEES(TX_SDMA6_DISALLOWED_PACKET)), 566 /*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr", 567 SEES(TX_SDMA7_DISALLOWED_PACKET)), 568 /*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr", 569 SEES(TX_SDMA8_DISALLOWED_PACKET)), 570 /*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr", 571 SEES(TX_SDMA9_DISALLOWED_PACKET)), 572 /*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr", 573 SEES(TX_SDMA10_DISALLOWED_PACKET)), 574 /*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr", 575 SEES(TX_SDMA11_DISALLOWED_PACKET)), 576 /*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr", 577 SEES(TX_SDMA12_DISALLOWED_PACKET)), 578 /*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr", 579 SEES(TX_SDMA13_DISALLOWED_PACKET)), 580 /*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr", 581 SEES(TX_SDMA14_DISALLOWED_PACKET)), 582 /*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr", 583 SEES(TX_SDMA15_DISALLOWED_PACKET)), 584 /*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr", 585 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)), 586 /*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr", 587 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)), 588 /*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr", 589 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)), 590 /*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr", 591 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)), 592 /*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr", 593 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)), 594 /*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr", 595 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)), 596 /*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr", 597 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)), 598 /*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr", 599 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)), 600 /*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr", 601 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)), 602 /*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)), 603 /*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)), 604 /*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)), 605 /*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)), 606 /*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)), 607 /*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)), 608 /*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)), 609 /*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)), 610 /*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)), 611 /*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)), 612 /*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)), 613 /*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)), 614 /*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)), 615 /*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)), 616 /*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)), 617 /*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)), 618 /*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)), 619 /*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)), 620 /*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)), 621 /*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)), 622 /*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)), 623 /*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr", 624 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)), 625 /*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr", 626 SEES(TX_READ_PIO_MEMORY_CSR_UNC)), 627 }; 628 629 /* 630 * TXE Egress Error Info flags 631 */ 632 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK 633 static struct flag_table egress_err_info_flags[] = { 634 /* 0*/ FLAG_ENTRY0("Reserved", 0ull), 635 /* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)), 636 /* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 637 /* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 638 /* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)), 639 /* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)), 640 /* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)), 641 /* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)), 642 /* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)), 643 /* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)), 644 /*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)), 645 /*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)), 646 /*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)), 647 /*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)), 648 /*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)), 649 /*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)), 650 /*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)), 651 /*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)), 652 /*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)), 653 /*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)), 654 /*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)), 655 /*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)), 656 }; 657 658 /* TXE Egress errors that cause an SPC freeze */ 659 #define ALL_TXE_EGRESS_FREEZE_ERR \ 660 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \ 661 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \ 662 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \ 663 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \ 664 | SEES(TX_LAUNCH_CSR_PARITY) \ 665 | SEES(TX_SBRD_CTL_CSR_PARITY) \ 666 | SEES(TX_CONFIG_PARITY) \ 667 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \ 668 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \ 669 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \ 670 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \ 671 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \ 672 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \ 673 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \ 674 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \ 675 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \ 676 | SEES(TX_CREDIT_RETURN_PARITY)) 677 678 /* 679 * TXE Send error flags 680 */ 681 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK 682 static struct flag_table send_err_status_flags[] = { 683 /* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)), 684 /* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)), 685 /* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR)) 686 }; 687 688 /* 689 * TXE Send Context Error flags and consequences 690 */ 691 static struct flag_table sc_err_status_flags[] = { 692 /* 0*/ FLAG_ENTRY("InconsistentSop", 693 SEC_PACKET_DROPPED | SEC_SC_HALTED, 694 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK), 695 /* 1*/ FLAG_ENTRY("DisallowedPacket", 696 SEC_PACKET_DROPPED | SEC_SC_HALTED, 697 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK), 698 /* 2*/ FLAG_ENTRY("WriteCrossesBoundary", 699 SEC_WRITE_DROPPED | SEC_SC_HALTED, 700 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK), 701 /* 3*/ FLAG_ENTRY("WriteOverflow", 702 SEC_WRITE_DROPPED | SEC_SC_HALTED, 703 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK), 704 /* 4*/ FLAG_ENTRY("WriteOutOfBounds", 705 SEC_WRITE_DROPPED | SEC_SC_HALTED, 706 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK), 707 /* 5-63 reserved*/ 708 }; 709 710 /* 711 * RXE Receive Error flags 712 */ 713 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK 714 static struct flag_table rxe_err_status_flags[] = { 715 /* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)), 716 /* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)), 717 /* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)), 718 /* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)), 719 /* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)), 720 /* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)), 721 /* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)), 722 /* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)), 723 /* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)), 724 /* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)), 725 /*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)), 726 /*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)), 727 /*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)), 728 /*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)), 729 /*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)), 730 /*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)), 731 /*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr", 732 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)), 733 /*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)), 734 /*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)), 735 /*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr", 736 RXES(RBUF_BLOCK_LIST_READ_UNC)), 737 /*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr", 738 RXES(RBUF_BLOCK_LIST_READ_COR)), 739 /*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr", 740 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)), 741 /*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr", 742 RXES(RBUF_CSR_QENT_CNT_PARITY)), 743 /*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr", 744 RXES(RBUF_CSR_QNEXT_BUF_PARITY)), 745 /*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr", 746 RXES(RBUF_CSR_QVLD_BIT_PARITY)), 747 /*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)), 748 /*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)), 749 /*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr", 750 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)), 751 /*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)), 752 /*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)), 753 /*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)), 754 /*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)), 755 /*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)), 756 /*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)), 757 /*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)), 758 /*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr", 759 RXES(RBUF_FL_INITDONE_PARITY)), 760 /*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr", 761 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)), 762 /*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)), 763 /*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)), 764 /*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)), 765 /*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr", 766 RXES(LOOKUP_DES_PART1_UNC_COR)), 767 /*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr", 768 RXES(LOOKUP_DES_PART2_PARITY)), 769 /*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)), 770 /*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)), 771 /*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)), 772 /*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)), 773 /*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)), 774 /*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)), 775 /*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)), 776 /*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)), 777 /*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)), 778 /*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)), 779 /*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)), 780 /*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)), 781 /*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)), 782 /*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)), 783 /*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)), 784 /*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)), 785 /*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)), 786 /*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)), 787 /*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)), 788 /*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)), 789 /*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)), 790 /*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY)) 791 }; 792 793 /* RXE errors that will trigger an SPC freeze */ 794 #define ALL_RXE_FREEZE_ERR \ 795 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \ 796 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \ 797 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \ 798 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \ 799 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \ 800 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \ 801 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \ 802 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \ 803 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \ 804 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \ 805 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \ 806 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \ 807 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \ 808 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \ 809 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \ 810 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \ 811 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \ 812 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \ 813 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \ 814 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \ 815 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \ 816 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \ 817 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \ 818 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \ 819 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \ 820 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \ 821 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \ 822 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \ 823 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \ 824 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \ 825 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \ 826 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \ 827 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \ 828 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \ 829 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \ 830 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \ 831 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \ 832 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \ 833 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \ 834 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \ 835 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \ 836 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \ 837 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \ 838 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK) 839 840 #define RXE_FREEZE_ABORT_MASK \ 841 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \ 842 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \ 843 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK) 844 845 /* 846 * DCC Error Flags 847 */ 848 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK 849 static struct flag_table dcc_err_flags[] = { 850 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)), 851 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)), 852 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)), 853 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)), 854 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)), 855 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)), 856 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)), 857 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)), 858 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)), 859 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)), 860 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)), 861 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)), 862 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)), 863 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)), 864 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)), 865 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)), 866 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)), 867 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)), 868 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)), 869 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)), 870 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)), 871 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)), 872 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)), 873 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)), 874 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)), 875 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)), 876 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)), 877 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)), 878 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)), 879 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)), 880 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)), 881 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)), 882 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)), 883 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)), 884 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)), 885 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)), 886 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)), 887 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)), 888 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)), 889 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)), 890 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)), 891 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)), 892 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)), 893 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)), 894 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)), 895 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)), 896 }; 897 898 /* 899 * LCB error flags 900 */ 901 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK 902 static struct flag_table lcb_err_flags[] = { 903 /* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)), 904 /* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)), 905 /* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)), 906 /* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST", 907 LCBE(ALL_LNS_FAILED_REINIT_TEST)), 908 /* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)), 909 /* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)), 910 /* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)), 911 /* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)), 912 /* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)), 913 /* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)), 914 /*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)), 915 /*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)), 916 /*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)), 917 /*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER", 918 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)), 919 /*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)), 920 /*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)), 921 /*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)), 922 /*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)), 923 /*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)), 924 /*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE", 925 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)), 926 /*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)), 927 /*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)), 928 /*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)), 929 /*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)), 930 /*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)), 931 /*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)), 932 /*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP", 933 LCBE(RST_FOR_INCOMPLT_RND_TRIP)), 934 /*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)), 935 /*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE", 936 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)), 937 /*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR", 938 LCBE(REDUNDANT_FLIT_PARITY_ERR)) 939 }; 940 941 /* 942 * DC8051 Error Flags 943 */ 944 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK 945 static struct flag_table dc8051_err_flags[] = { 946 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)), 947 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)), 948 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)), 949 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)), 950 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)), 951 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)), 952 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)), 953 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)), 954 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES", 955 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)), 956 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)), 957 }; 958 959 /* 960 * DC8051 Information Error flags 961 * 962 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field. 963 */ 964 static struct flag_table dc8051_info_err_flags[] = { 965 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED), 966 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME), 967 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET), 968 FLAG_ENTRY0("Serdes internal loopback failure", 969 FAILED_SERDES_INTERNAL_LOOPBACK), 970 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT), 971 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING), 972 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE), 973 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM), 974 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ), 975 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1), 976 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2), 977 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT), 978 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT), 979 FLAG_ENTRY0("External Device Request Timeout", 980 EXTERNAL_DEVICE_REQ_TIMEOUT), 981 }; 982 983 /* 984 * DC8051 Information Host Information flags 985 * 986 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field. 987 */ 988 static struct flag_table dc8051_info_host_msg_flags[] = { 989 FLAG_ENTRY0("Host request done", 0x0001), 990 FLAG_ENTRY0("BC PWR_MGM message", 0x0002), 991 FLAG_ENTRY0("BC SMA message", 0x0004), 992 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008), 993 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010), 994 FLAG_ENTRY0("External device config request", 0x0020), 995 FLAG_ENTRY0("VerifyCap all frames received", 0x0040), 996 FLAG_ENTRY0("LinkUp achieved", 0x0080), 997 FLAG_ENTRY0("Link going down", 0x0100), 998 FLAG_ENTRY0("Link width downgraded", 0x0200), 999 }; 1000 1001 static u32 encoded_size(u32 size); 1002 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate); 1003 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state); 1004 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 1005 u8 *continuous); 1006 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 1007 u8 *vcu, u16 *vl15buf, u8 *crc_sizes); 1008 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 1009 u8 *remote_tx_rate, u16 *link_widths); 1010 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits, 1011 u8 *flag_bits, u16 *link_widths); 1012 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 1013 u8 *device_rev); 1014 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx); 1015 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx, 1016 u8 *tx_polarity_inversion, 1017 u8 *rx_polarity_inversion, u8 *max_rate); 1018 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 1019 unsigned int context, u64 err_status); 1020 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg); 1021 static void handle_dcc_err(struct hfi1_devdata *dd, 1022 unsigned int context, u64 err_status); 1023 static void handle_lcb_err(struct hfi1_devdata *dd, 1024 unsigned int context, u64 err_status); 1025 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg); 1026 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1027 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1028 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1029 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1030 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1031 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1032 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1033 static void set_partition_keys(struct hfi1_pportdata *ppd); 1034 static const char *link_state_name(u32 state); 1035 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, 1036 u32 state); 1037 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 1038 u64 *out_data); 1039 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data); 1040 static int thermal_init(struct hfi1_devdata *dd); 1041 1042 static void update_statusp(struct hfi1_pportdata *ppd, u32 state); 1043 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, 1044 int msecs); 1045 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1046 int msecs); 1047 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state); 1048 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state); 1049 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1050 int msecs); 1051 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd, 1052 int msecs); 1053 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc); 1054 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr); 1055 static void handle_temp_err(struct hfi1_devdata *dd); 1056 static void dc_shutdown(struct hfi1_devdata *dd); 1057 static void dc_start(struct hfi1_devdata *dd); 1058 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 1059 unsigned int *np); 1060 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); 1061 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); 1062 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index); 1063 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width); 1064 1065 /* 1066 * Error interrupt table entry. This is used as input to the interrupt 1067 * "clear down" routine used for all second tier error interrupt register. 1068 * Second tier interrupt registers have a single bit representing them 1069 * in the top-level CceIntStatus. 1070 */ 1071 struct err_reg_info { 1072 u32 status; /* status CSR offset */ 1073 u32 clear; /* clear CSR offset */ 1074 u32 mask; /* mask CSR offset */ 1075 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg); 1076 const char *desc; 1077 }; 1078 1079 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START) 1080 #define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START) 1081 #define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START) 1082 1083 /* 1084 * Helpers for building HFI and DC error interrupt table entries. Different 1085 * helpers are needed because of inconsistent register names. 1086 */ 1087 #define EE(reg, handler, desc) \ 1088 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \ 1089 handler, desc } 1090 #define DC_EE1(reg, handler, desc) \ 1091 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc } 1092 #define DC_EE2(reg, handler, desc) \ 1093 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc } 1094 1095 /* 1096 * Table of the "misc" grouping of error interrupts. Each entry refers to 1097 * another register containing more information. 1098 */ 1099 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = { 1100 /* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"), 1101 /* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"), 1102 /* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"), 1103 /* 3*/ { 0, 0, 0, NULL }, /* reserved */ 1104 /* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"), 1105 /* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"), 1106 /* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"), 1107 /* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr") 1108 /* the rest are reserved */ 1109 }; 1110 1111 /* 1112 * Index into the Various section of the interrupt sources 1113 * corresponding to the Critical Temperature interrupt. 1114 */ 1115 #define TCRIT_INT_SOURCE 4 1116 1117 /* 1118 * SDMA error interrupt entry - refers to another register containing more 1119 * information. 1120 */ 1121 static const struct err_reg_info sdma_eng_err = 1122 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr"); 1123 1124 static const struct err_reg_info various_err[NUM_VARIOUS] = { 1125 /* 0*/ { 0, 0, 0, NULL }, /* PbcInt */ 1126 /* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */ 1127 /* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"), 1128 /* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"), 1129 /* 4*/ { 0, 0, 0, NULL }, /* TCritInt */ 1130 /* rest are reserved */ 1131 }; 1132 1133 /* 1134 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG 1135 * register can not be derived from the MTU value because 10K is not 1136 * a power of 2. Therefore, we need a constant. Everything else can 1137 * be calculated. 1138 */ 1139 #define DCC_CFG_PORT_MTU_CAP_10240 7 1140 1141 /* 1142 * Table of the DC grouping of error interrupts. Each entry refers to 1143 * another register containing more information. 1144 */ 1145 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = { 1146 /* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"), 1147 /* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"), 1148 /* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"), 1149 /* 3*/ /* dc_lbm_int - special, see is_dc_int() */ 1150 /* the rest are reserved */ 1151 }; 1152 1153 struct cntr_entry { 1154 /* 1155 * counter name 1156 */ 1157 char *name; 1158 1159 /* 1160 * csr to read for name (if applicable) 1161 */ 1162 u64 csr; 1163 1164 /* 1165 * offset into dd or ppd to store the counter's value 1166 */ 1167 int offset; 1168 1169 /* 1170 * flags 1171 */ 1172 u8 flags; 1173 1174 /* 1175 * accessor for stat element, context either dd or ppd 1176 */ 1177 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl, 1178 int mode, u64 data); 1179 }; 1180 1181 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0 1182 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159 1183 1184 #define CNTR_ELEM(name, csr, offset, flags, accessor) \ 1185 { \ 1186 name, \ 1187 csr, \ 1188 offset, \ 1189 flags, \ 1190 accessor \ 1191 } 1192 1193 /* 32bit RXE */ 1194 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1195 CNTR_ELEM(#name, \ 1196 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1197 0, flags | CNTR_32BIT, \ 1198 port_access_u32_csr) 1199 1200 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \ 1201 CNTR_ELEM(#name, \ 1202 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1203 0, flags | CNTR_32BIT, \ 1204 dev_access_u32_csr) 1205 1206 /* 64bit RXE */ 1207 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1208 CNTR_ELEM(#name, \ 1209 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1210 0, flags, \ 1211 port_access_u64_csr) 1212 1213 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \ 1214 CNTR_ELEM(#name, \ 1215 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1216 0, flags, \ 1217 dev_access_u64_csr) 1218 1219 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx 1220 #define OVR_ELM(ctx) \ 1221 CNTR_ELEM("RcvHdrOvr" #ctx, \ 1222 (RCV_HDR_OVFL_CNT + ctx * 0x100), \ 1223 0, CNTR_NORMAL, port_access_u64_csr) 1224 1225 /* 32bit TXE */ 1226 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1227 CNTR_ELEM(#name, \ 1228 (counter * 8 + SEND_COUNTER_ARRAY32), \ 1229 0, flags | CNTR_32BIT, \ 1230 port_access_u32_csr) 1231 1232 /* 64bit TXE */ 1233 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1234 CNTR_ELEM(#name, \ 1235 (counter * 8 + SEND_COUNTER_ARRAY64), \ 1236 0, flags, \ 1237 port_access_u64_csr) 1238 1239 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \ 1240 CNTR_ELEM(#name,\ 1241 counter * 8 + SEND_COUNTER_ARRAY64, \ 1242 0, \ 1243 flags, \ 1244 dev_access_u64_csr) 1245 1246 /* CCE */ 1247 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \ 1248 CNTR_ELEM(#name, \ 1249 (counter * 8 + CCE_COUNTER_ARRAY32), \ 1250 0, flags | CNTR_32BIT, \ 1251 dev_access_u32_csr) 1252 1253 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \ 1254 CNTR_ELEM(#name, \ 1255 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \ 1256 0, flags | CNTR_32BIT, \ 1257 dev_access_u32_csr) 1258 1259 /* DC */ 1260 #define DC_PERF_CNTR(name, counter, flags) \ 1261 CNTR_ELEM(#name, \ 1262 counter, \ 1263 0, \ 1264 flags, \ 1265 dev_access_u64_csr) 1266 1267 #define DC_PERF_CNTR_LCB(name, counter, flags) \ 1268 CNTR_ELEM(#name, \ 1269 counter, \ 1270 0, \ 1271 flags, \ 1272 dc_access_lcb_cntr) 1273 1274 /* ibp counters */ 1275 #define SW_IBP_CNTR(name, cntr) \ 1276 CNTR_ELEM(#name, \ 1277 0, \ 1278 0, \ 1279 CNTR_SYNTH, \ 1280 access_ibp_##cntr) 1281 1282 /** 1283 * hfi1_addr_from_offset - return addr for readq/writeq 1284 * @dd: the dd device 1285 * @offset: the offset of the CSR within bar0 1286 * 1287 * This routine selects the appropriate base address 1288 * based on the indicated offset. 1289 */ 1290 static inline void __iomem *hfi1_addr_from_offset( 1291 const struct hfi1_devdata *dd, 1292 u32 offset) 1293 { 1294 if (offset >= dd->base2_start) 1295 return dd->kregbase2 + (offset - dd->base2_start); 1296 return dd->kregbase1 + offset; 1297 } 1298 1299 /** 1300 * read_csr - read CSR at the indicated offset 1301 * @dd: the dd device 1302 * @offset: the offset of the CSR within bar0 1303 * 1304 * Return: the value read or all FF's if there 1305 * is no mapping 1306 */ 1307 u64 read_csr(const struct hfi1_devdata *dd, u32 offset) 1308 { 1309 if (dd->flags & HFI1_PRESENT) 1310 return readq(hfi1_addr_from_offset(dd, offset)); 1311 return -1; 1312 } 1313 1314 /** 1315 * write_csr - write CSR at the indicated offset 1316 * @dd: the dd device 1317 * @offset: the offset of the CSR within bar0 1318 * @value: value to write 1319 */ 1320 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value) 1321 { 1322 if (dd->flags & HFI1_PRESENT) { 1323 void __iomem *base = hfi1_addr_from_offset(dd, offset); 1324 1325 /* avoid write to RcvArray */ 1326 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start)) 1327 return; 1328 writeq(value, base); 1329 } 1330 } 1331 1332 /** 1333 * get_csr_addr - return te iomem address for offset 1334 * @dd: the dd device 1335 * @offset: the offset of the CSR within bar0 1336 * 1337 * Return: The iomem address to use in subsequent 1338 * writeq/readq operations. 1339 */ 1340 void __iomem *get_csr_addr( 1341 const struct hfi1_devdata *dd, 1342 u32 offset) 1343 { 1344 if (dd->flags & HFI1_PRESENT) 1345 return hfi1_addr_from_offset(dd, offset); 1346 return NULL; 1347 } 1348 1349 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr, 1350 int mode, u64 value) 1351 { 1352 u64 ret; 1353 1354 if (mode == CNTR_MODE_R) { 1355 ret = read_csr(dd, csr); 1356 } else if (mode == CNTR_MODE_W) { 1357 write_csr(dd, csr, value); 1358 ret = value; 1359 } else { 1360 dd_dev_err(dd, "Invalid cntr register access mode"); 1361 return 0; 1362 } 1363 1364 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode); 1365 return ret; 1366 } 1367 1368 /* Dev Access */ 1369 static u64 dev_access_u32_csr(const struct cntr_entry *entry, 1370 void *context, int vl, int mode, u64 data) 1371 { 1372 struct hfi1_devdata *dd = context; 1373 u64 csr = entry->csr; 1374 1375 if (entry->flags & CNTR_SDMA) { 1376 if (vl == CNTR_INVALID_VL) 1377 return 0; 1378 csr += 0x100 * vl; 1379 } else { 1380 if (vl != CNTR_INVALID_VL) 1381 return 0; 1382 } 1383 return read_write_csr(dd, csr, mode, data); 1384 } 1385 1386 static u64 access_sde_err_cnt(const struct cntr_entry *entry, 1387 void *context, int idx, int mode, u64 data) 1388 { 1389 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1390 1391 if (dd->per_sdma && idx < dd->num_sdma) 1392 return dd->per_sdma[idx].err_cnt; 1393 return 0; 1394 } 1395 1396 static u64 access_sde_int_cnt(const struct cntr_entry *entry, 1397 void *context, int idx, int mode, u64 data) 1398 { 1399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1400 1401 if (dd->per_sdma && idx < dd->num_sdma) 1402 return dd->per_sdma[idx].sdma_int_cnt; 1403 return 0; 1404 } 1405 1406 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry, 1407 void *context, int idx, int mode, u64 data) 1408 { 1409 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1410 1411 if (dd->per_sdma && idx < dd->num_sdma) 1412 return dd->per_sdma[idx].idle_int_cnt; 1413 return 0; 1414 } 1415 1416 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry, 1417 void *context, int idx, int mode, 1418 u64 data) 1419 { 1420 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1421 1422 if (dd->per_sdma && idx < dd->num_sdma) 1423 return dd->per_sdma[idx].progress_int_cnt; 1424 return 0; 1425 } 1426 1427 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context, 1428 int vl, int mode, u64 data) 1429 { 1430 struct hfi1_devdata *dd = context; 1431 1432 u64 val = 0; 1433 u64 csr = entry->csr; 1434 1435 if (entry->flags & CNTR_VL) { 1436 if (vl == CNTR_INVALID_VL) 1437 return 0; 1438 csr += 8 * vl; 1439 } else { 1440 if (vl != CNTR_INVALID_VL) 1441 return 0; 1442 } 1443 1444 val = read_write_csr(dd, csr, mode, data); 1445 return val; 1446 } 1447 1448 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context, 1449 int vl, int mode, u64 data) 1450 { 1451 struct hfi1_devdata *dd = context; 1452 u32 csr = entry->csr; 1453 int ret = 0; 1454 1455 if (vl != CNTR_INVALID_VL) 1456 return 0; 1457 if (mode == CNTR_MODE_R) 1458 ret = read_lcb_csr(dd, csr, &data); 1459 else if (mode == CNTR_MODE_W) 1460 ret = write_lcb_csr(dd, csr, data); 1461 1462 if (ret) { 1463 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr); 1464 return 0; 1465 } 1466 1467 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode); 1468 return data; 1469 } 1470 1471 /* Port Access */ 1472 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context, 1473 int vl, int mode, u64 data) 1474 { 1475 struct hfi1_pportdata *ppd = context; 1476 1477 if (vl != CNTR_INVALID_VL) 1478 return 0; 1479 return read_write_csr(ppd->dd, entry->csr, mode, data); 1480 } 1481 1482 static u64 port_access_u64_csr(const struct cntr_entry *entry, 1483 void *context, int vl, int mode, u64 data) 1484 { 1485 struct hfi1_pportdata *ppd = context; 1486 u64 val; 1487 u64 csr = entry->csr; 1488 1489 if (entry->flags & CNTR_VL) { 1490 if (vl == CNTR_INVALID_VL) 1491 return 0; 1492 csr += 8 * vl; 1493 } else { 1494 if (vl != CNTR_INVALID_VL) 1495 return 0; 1496 } 1497 val = read_write_csr(ppd->dd, csr, mode, data); 1498 return val; 1499 } 1500 1501 /* Software defined */ 1502 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode, 1503 u64 data) 1504 { 1505 u64 ret; 1506 1507 if (mode == CNTR_MODE_R) { 1508 ret = *cntr; 1509 } else if (mode == CNTR_MODE_W) { 1510 *cntr = data; 1511 ret = data; 1512 } else { 1513 dd_dev_err(dd, "Invalid cntr sw access mode"); 1514 return 0; 1515 } 1516 1517 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode); 1518 1519 return ret; 1520 } 1521 1522 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context, 1523 int vl, int mode, u64 data) 1524 { 1525 struct hfi1_pportdata *ppd = context; 1526 1527 if (vl != CNTR_INVALID_VL) 1528 return 0; 1529 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data); 1530 } 1531 1532 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context, 1533 int vl, int mode, u64 data) 1534 { 1535 struct hfi1_pportdata *ppd = context; 1536 1537 if (vl != CNTR_INVALID_VL) 1538 return 0; 1539 return read_write_sw(ppd->dd, &ppd->link_up, mode, data); 1540 } 1541 1542 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry, 1543 void *context, int vl, int mode, 1544 u64 data) 1545 { 1546 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1547 1548 if (vl != CNTR_INVALID_VL) 1549 return 0; 1550 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data); 1551 } 1552 1553 static u64 access_sw_xmit_discards(const struct cntr_entry *entry, 1554 void *context, int vl, int mode, u64 data) 1555 { 1556 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1557 u64 zero = 0; 1558 u64 *counter; 1559 1560 if (vl == CNTR_INVALID_VL) 1561 counter = &ppd->port_xmit_discards; 1562 else if (vl >= 0 && vl < C_VL_COUNT) 1563 counter = &ppd->port_xmit_discards_vl[vl]; 1564 else 1565 counter = &zero; 1566 1567 return read_write_sw(ppd->dd, counter, mode, data); 1568 } 1569 1570 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry, 1571 void *context, int vl, int mode, 1572 u64 data) 1573 { 1574 struct hfi1_pportdata *ppd = context; 1575 1576 if (vl != CNTR_INVALID_VL) 1577 return 0; 1578 1579 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors, 1580 mode, data); 1581 } 1582 1583 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry, 1584 void *context, int vl, int mode, u64 data) 1585 { 1586 struct hfi1_pportdata *ppd = context; 1587 1588 if (vl != CNTR_INVALID_VL) 1589 return 0; 1590 1591 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors, 1592 mode, data); 1593 } 1594 1595 u64 get_all_cpu_total(u64 __percpu *cntr) 1596 { 1597 int cpu; 1598 u64 counter = 0; 1599 1600 for_each_possible_cpu(cpu) 1601 counter += *per_cpu_ptr(cntr, cpu); 1602 return counter; 1603 } 1604 1605 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val, 1606 u64 __percpu *cntr, 1607 int vl, int mode, u64 data) 1608 { 1609 u64 ret = 0; 1610 1611 if (vl != CNTR_INVALID_VL) 1612 return 0; 1613 1614 if (mode == CNTR_MODE_R) { 1615 ret = get_all_cpu_total(cntr) - *z_val; 1616 } else if (mode == CNTR_MODE_W) { 1617 /* A write can only zero the counter */ 1618 if (data == 0) 1619 *z_val = get_all_cpu_total(cntr); 1620 else 1621 dd_dev_err(dd, "Per CPU cntrs can only be zeroed"); 1622 } else { 1623 dd_dev_err(dd, "Invalid cntr sw cpu access mode"); 1624 return 0; 1625 } 1626 1627 return ret; 1628 } 1629 1630 static u64 access_sw_cpu_intr(const struct cntr_entry *entry, 1631 void *context, int vl, int mode, u64 data) 1632 { 1633 struct hfi1_devdata *dd = context; 1634 1635 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl, 1636 mode, data); 1637 } 1638 1639 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry, 1640 void *context, int vl, int mode, u64 data) 1641 { 1642 struct hfi1_devdata *dd = context; 1643 1644 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl, 1645 mode, data); 1646 } 1647 1648 static u64 access_sw_pio_wait(const struct cntr_entry *entry, 1649 void *context, int vl, int mode, u64 data) 1650 { 1651 struct hfi1_devdata *dd = context; 1652 1653 return dd->verbs_dev.n_piowait; 1654 } 1655 1656 static u64 access_sw_pio_drain(const struct cntr_entry *entry, 1657 void *context, int vl, int mode, u64 data) 1658 { 1659 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1660 1661 return dd->verbs_dev.n_piodrain; 1662 } 1663 1664 static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry, 1665 void *context, int vl, int mode, u64 data) 1666 { 1667 struct hfi1_devdata *dd = context; 1668 1669 return dd->ctx0_seq_drop; 1670 } 1671 1672 static u64 access_sw_vtx_wait(const struct cntr_entry *entry, 1673 void *context, int vl, int mode, u64 data) 1674 { 1675 struct hfi1_devdata *dd = context; 1676 1677 return dd->verbs_dev.n_txwait; 1678 } 1679 1680 static u64 access_sw_kmem_wait(const struct cntr_entry *entry, 1681 void *context, int vl, int mode, u64 data) 1682 { 1683 struct hfi1_devdata *dd = context; 1684 1685 return dd->verbs_dev.n_kmem_wait; 1686 } 1687 1688 static u64 access_sw_send_schedule(const struct cntr_entry *entry, 1689 void *context, int vl, int mode, u64 data) 1690 { 1691 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1692 1693 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl, 1694 mode, data); 1695 } 1696 1697 /* Software counters for the error status bits within MISC_ERR_STATUS */ 1698 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry, 1699 void *context, int vl, int mode, 1700 u64 data) 1701 { 1702 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1703 1704 return dd->misc_err_status_cnt[12]; 1705 } 1706 1707 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry, 1708 void *context, int vl, int mode, 1709 u64 data) 1710 { 1711 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1712 1713 return dd->misc_err_status_cnt[11]; 1714 } 1715 1716 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry, 1717 void *context, int vl, int mode, 1718 u64 data) 1719 { 1720 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1721 1722 return dd->misc_err_status_cnt[10]; 1723 } 1724 1725 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry, 1726 void *context, int vl, 1727 int mode, u64 data) 1728 { 1729 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1730 1731 return dd->misc_err_status_cnt[9]; 1732 } 1733 1734 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry, 1735 void *context, int vl, int mode, 1736 u64 data) 1737 { 1738 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1739 1740 return dd->misc_err_status_cnt[8]; 1741 } 1742 1743 static u64 access_misc_efuse_read_bad_addr_err_cnt( 1744 const struct cntr_entry *entry, 1745 void *context, int vl, int mode, u64 data) 1746 { 1747 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1748 1749 return dd->misc_err_status_cnt[7]; 1750 } 1751 1752 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry, 1753 void *context, int vl, 1754 int mode, u64 data) 1755 { 1756 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1757 1758 return dd->misc_err_status_cnt[6]; 1759 } 1760 1761 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry, 1762 void *context, int vl, int mode, 1763 u64 data) 1764 { 1765 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1766 1767 return dd->misc_err_status_cnt[5]; 1768 } 1769 1770 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry, 1771 void *context, int vl, int mode, 1772 u64 data) 1773 { 1774 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1775 1776 return dd->misc_err_status_cnt[4]; 1777 } 1778 1779 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry, 1780 void *context, int vl, 1781 int mode, u64 data) 1782 { 1783 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1784 1785 return dd->misc_err_status_cnt[3]; 1786 } 1787 1788 static u64 access_misc_csr_write_bad_addr_err_cnt( 1789 const struct cntr_entry *entry, 1790 void *context, int vl, int mode, u64 data) 1791 { 1792 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1793 1794 return dd->misc_err_status_cnt[2]; 1795 } 1796 1797 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1798 void *context, int vl, 1799 int mode, u64 data) 1800 { 1801 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1802 1803 return dd->misc_err_status_cnt[1]; 1804 } 1805 1806 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry, 1807 void *context, int vl, int mode, 1808 u64 data) 1809 { 1810 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1811 1812 return dd->misc_err_status_cnt[0]; 1813 } 1814 1815 /* 1816 * Software counter for the aggregate of 1817 * individual CceErrStatus counters 1818 */ 1819 static u64 access_sw_cce_err_status_aggregated_cnt( 1820 const struct cntr_entry *entry, 1821 void *context, int vl, int mode, u64 data) 1822 { 1823 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1824 1825 return dd->sw_cce_err_status_aggregate; 1826 } 1827 1828 /* 1829 * Software counters corresponding to each of the 1830 * error status bits within CceErrStatus 1831 */ 1832 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry, 1833 void *context, int vl, int mode, 1834 u64 data) 1835 { 1836 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1837 1838 return dd->cce_err_status_cnt[40]; 1839 } 1840 1841 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry, 1842 void *context, int vl, int mode, 1843 u64 data) 1844 { 1845 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1846 1847 return dd->cce_err_status_cnt[39]; 1848 } 1849 1850 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry, 1851 void *context, int vl, int mode, 1852 u64 data) 1853 { 1854 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1855 1856 return dd->cce_err_status_cnt[38]; 1857 } 1858 1859 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry, 1860 void *context, int vl, int mode, 1861 u64 data) 1862 { 1863 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1864 1865 return dd->cce_err_status_cnt[37]; 1866 } 1867 1868 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry, 1869 void *context, int vl, int mode, 1870 u64 data) 1871 { 1872 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1873 1874 return dd->cce_err_status_cnt[36]; 1875 } 1876 1877 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt( 1878 const struct cntr_entry *entry, 1879 void *context, int vl, int mode, u64 data) 1880 { 1881 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1882 1883 return dd->cce_err_status_cnt[35]; 1884 } 1885 1886 static u64 access_cce_rcpl_async_fifo_parity_err_cnt( 1887 const struct cntr_entry *entry, 1888 void *context, int vl, int mode, u64 data) 1889 { 1890 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1891 1892 return dd->cce_err_status_cnt[34]; 1893 } 1894 1895 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry, 1896 void *context, int vl, 1897 int mode, u64 data) 1898 { 1899 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1900 1901 return dd->cce_err_status_cnt[33]; 1902 } 1903 1904 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1905 void *context, int vl, int mode, 1906 u64 data) 1907 { 1908 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1909 1910 return dd->cce_err_status_cnt[32]; 1911 } 1912 1913 static u64 access_la_triggered_cnt(const struct cntr_entry *entry, 1914 void *context, int vl, int mode, u64 data) 1915 { 1916 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1917 1918 return dd->cce_err_status_cnt[31]; 1919 } 1920 1921 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry, 1922 void *context, int vl, int mode, 1923 u64 data) 1924 { 1925 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1926 1927 return dd->cce_err_status_cnt[30]; 1928 } 1929 1930 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry, 1931 void *context, int vl, int mode, 1932 u64 data) 1933 { 1934 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1935 1936 return dd->cce_err_status_cnt[29]; 1937 } 1938 1939 static u64 access_pcic_transmit_back_parity_err_cnt( 1940 const struct cntr_entry *entry, 1941 void *context, int vl, int mode, u64 data) 1942 { 1943 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1944 1945 return dd->cce_err_status_cnt[28]; 1946 } 1947 1948 static u64 access_pcic_transmit_front_parity_err_cnt( 1949 const struct cntr_entry *entry, 1950 void *context, int vl, int mode, u64 data) 1951 { 1952 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1953 1954 return dd->cce_err_status_cnt[27]; 1955 } 1956 1957 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1958 void *context, int vl, int mode, 1959 u64 data) 1960 { 1961 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1962 1963 return dd->cce_err_status_cnt[26]; 1964 } 1965 1966 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1967 void *context, int vl, int mode, 1968 u64 data) 1969 { 1970 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1971 1972 return dd->cce_err_status_cnt[25]; 1973 } 1974 1975 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1976 void *context, int vl, int mode, 1977 u64 data) 1978 { 1979 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1980 1981 return dd->cce_err_status_cnt[24]; 1982 } 1983 1984 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1985 void *context, int vl, int mode, 1986 u64 data) 1987 { 1988 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1989 1990 return dd->cce_err_status_cnt[23]; 1991 } 1992 1993 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry, 1994 void *context, int vl, 1995 int mode, u64 data) 1996 { 1997 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1998 1999 return dd->cce_err_status_cnt[22]; 2000 } 2001 2002 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry, 2003 void *context, int vl, int mode, 2004 u64 data) 2005 { 2006 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2007 2008 return dd->cce_err_status_cnt[21]; 2009 } 2010 2011 static u64 access_pcic_n_post_dat_q_parity_err_cnt( 2012 const struct cntr_entry *entry, 2013 void *context, int vl, int mode, u64 data) 2014 { 2015 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2016 2017 return dd->cce_err_status_cnt[20]; 2018 } 2019 2020 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry, 2021 void *context, int vl, 2022 int mode, u64 data) 2023 { 2024 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2025 2026 return dd->cce_err_status_cnt[19]; 2027 } 2028 2029 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry, 2030 void *context, int vl, int mode, 2031 u64 data) 2032 { 2033 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2034 2035 return dd->cce_err_status_cnt[18]; 2036 } 2037 2038 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry, 2039 void *context, int vl, int mode, 2040 u64 data) 2041 { 2042 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2043 2044 return dd->cce_err_status_cnt[17]; 2045 } 2046 2047 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry, 2048 void *context, int vl, int mode, 2049 u64 data) 2050 { 2051 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2052 2053 return dd->cce_err_status_cnt[16]; 2054 } 2055 2056 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry, 2057 void *context, int vl, int mode, 2058 u64 data) 2059 { 2060 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2061 2062 return dd->cce_err_status_cnt[15]; 2063 } 2064 2065 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry, 2066 void *context, int vl, 2067 int mode, u64 data) 2068 { 2069 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2070 2071 return dd->cce_err_status_cnt[14]; 2072 } 2073 2074 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry, 2075 void *context, int vl, int mode, 2076 u64 data) 2077 { 2078 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2079 2080 return dd->cce_err_status_cnt[13]; 2081 } 2082 2083 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt( 2084 const struct cntr_entry *entry, 2085 void *context, int vl, int mode, u64 data) 2086 { 2087 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2088 2089 return dd->cce_err_status_cnt[12]; 2090 } 2091 2092 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt( 2093 const struct cntr_entry *entry, 2094 void *context, int vl, int mode, u64 data) 2095 { 2096 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2097 2098 return dd->cce_err_status_cnt[11]; 2099 } 2100 2101 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt( 2102 const struct cntr_entry *entry, 2103 void *context, int vl, int mode, u64 data) 2104 { 2105 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2106 2107 return dd->cce_err_status_cnt[10]; 2108 } 2109 2110 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt( 2111 const struct cntr_entry *entry, 2112 void *context, int vl, int mode, u64 data) 2113 { 2114 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2115 2116 return dd->cce_err_status_cnt[9]; 2117 } 2118 2119 static u64 access_cce_cli2_async_fifo_parity_err_cnt( 2120 const struct cntr_entry *entry, 2121 void *context, int vl, int mode, u64 data) 2122 { 2123 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2124 2125 return dd->cce_err_status_cnt[8]; 2126 } 2127 2128 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry, 2129 void *context, int vl, 2130 int mode, u64 data) 2131 { 2132 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2133 2134 return dd->cce_err_status_cnt[7]; 2135 } 2136 2137 static u64 access_cce_cli0_async_fifo_parity_err_cnt( 2138 const struct cntr_entry *entry, 2139 void *context, int vl, int mode, u64 data) 2140 { 2141 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2142 2143 return dd->cce_err_status_cnt[6]; 2144 } 2145 2146 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry, 2147 void *context, int vl, int mode, 2148 u64 data) 2149 { 2150 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2151 2152 return dd->cce_err_status_cnt[5]; 2153 } 2154 2155 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry, 2156 void *context, int vl, int mode, 2157 u64 data) 2158 { 2159 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2160 2161 return dd->cce_err_status_cnt[4]; 2162 } 2163 2164 static u64 access_cce_trgt_async_fifo_parity_err_cnt( 2165 const struct cntr_entry *entry, 2166 void *context, int vl, int mode, u64 data) 2167 { 2168 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2169 2170 return dd->cce_err_status_cnt[3]; 2171 } 2172 2173 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2174 void *context, int vl, 2175 int mode, u64 data) 2176 { 2177 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2178 2179 return dd->cce_err_status_cnt[2]; 2180 } 2181 2182 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2183 void *context, int vl, 2184 int mode, u64 data) 2185 { 2186 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2187 2188 return dd->cce_err_status_cnt[1]; 2189 } 2190 2191 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry, 2192 void *context, int vl, int mode, 2193 u64 data) 2194 { 2195 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2196 2197 return dd->cce_err_status_cnt[0]; 2198 } 2199 2200 /* 2201 * Software counters corresponding to each of the 2202 * error status bits within RcvErrStatus 2203 */ 2204 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry, 2205 void *context, int vl, int mode, 2206 u64 data) 2207 { 2208 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2209 2210 return dd->rcv_err_status_cnt[63]; 2211 } 2212 2213 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2214 void *context, int vl, 2215 int mode, u64 data) 2216 { 2217 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2218 2219 return dd->rcv_err_status_cnt[62]; 2220 } 2221 2222 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2223 void *context, int vl, int mode, 2224 u64 data) 2225 { 2226 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2227 2228 return dd->rcv_err_status_cnt[61]; 2229 } 2230 2231 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry, 2232 void *context, int vl, int mode, 2233 u64 data) 2234 { 2235 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2236 2237 return dd->rcv_err_status_cnt[60]; 2238 } 2239 2240 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2241 void *context, int vl, 2242 int mode, u64 data) 2243 { 2244 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2245 2246 return dd->rcv_err_status_cnt[59]; 2247 } 2248 2249 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2250 void *context, int vl, 2251 int mode, u64 data) 2252 { 2253 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2254 2255 return dd->rcv_err_status_cnt[58]; 2256 } 2257 2258 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry, 2259 void *context, int vl, int mode, 2260 u64 data) 2261 { 2262 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2263 2264 return dd->rcv_err_status_cnt[57]; 2265 } 2266 2267 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry, 2268 void *context, int vl, int mode, 2269 u64 data) 2270 { 2271 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2272 2273 return dd->rcv_err_status_cnt[56]; 2274 } 2275 2276 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry, 2277 void *context, int vl, int mode, 2278 u64 data) 2279 { 2280 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2281 2282 return dd->rcv_err_status_cnt[55]; 2283 } 2284 2285 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt( 2286 const struct cntr_entry *entry, 2287 void *context, int vl, int mode, u64 data) 2288 { 2289 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2290 2291 return dd->rcv_err_status_cnt[54]; 2292 } 2293 2294 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt( 2295 const struct cntr_entry *entry, 2296 void *context, int vl, int mode, u64 data) 2297 { 2298 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2299 2300 return dd->rcv_err_status_cnt[53]; 2301 } 2302 2303 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry, 2304 void *context, int vl, 2305 int mode, u64 data) 2306 { 2307 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2308 2309 return dd->rcv_err_status_cnt[52]; 2310 } 2311 2312 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry, 2313 void *context, int vl, 2314 int mode, u64 data) 2315 { 2316 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2317 2318 return dd->rcv_err_status_cnt[51]; 2319 } 2320 2321 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry, 2322 void *context, int vl, 2323 int mode, u64 data) 2324 { 2325 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2326 2327 return dd->rcv_err_status_cnt[50]; 2328 } 2329 2330 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry, 2331 void *context, int vl, 2332 int mode, u64 data) 2333 { 2334 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2335 2336 return dd->rcv_err_status_cnt[49]; 2337 } 2338 2339 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry, 2340 void *context, int vl, 2341 int mode, u64 data) 2342 { 2343 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2344 2345 return dd->rcv_err_status_cnt[48]; 2346 } 2347 2348 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry, 2349 void *context, int vl, 2350 int mode, u64 data) 2351 { 2352 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2353 2354 return dd->rcv_err_status_cnt[47]; 2355 } 2356 2357 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry, 2358 void *context, int vl, int mode, 2359 u64 data) 2360 { 2361 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2362 2363 return dd->rcv_err_status_cnt[46]; 2364 } 2365 2366 static u64 access_rx_hq_intr_csr_parity_err_cnt( 2367 const struct cntr_entry *entry, 2368 void *context, int vl, int mode, u64 data) 2369 { 2370 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2371 2372 return dd->rcv_err_status_cnt[45]; 2373 } 2374 2375 static u64 access_rx_lookup_csr_parity_err_cnt( 2376 const struct cntr_entry *entry, 2377 void *context, int vl, int mode, u64 data) 2378 { 2379 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2380 2381 return dd->rcv_err_status_cnt[44]; 2382 } 2383 2384 static u64 access_rx_lookup_rcv_array_cor_err_cnt( 2385 const struct cntr_entry *entry, 2386 void *context, int vl, int mode, u64 data) 2387 { 2388 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2389 2390 return dd->rcv_err_status_cnt[43]; 2391 } 2392 2393 static u64 access_rx_lookup_rcv_array_unc_err_cnt( 2394 const struct cntr_entry *entry, 2395 void *context, int vl, int mode, u64 data) 2396 { 2397 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2398 2399 return dd->rcv_err_status_cnt[42]; 2400 } 2401 2402 static u64 access_rx_lookup_des_part2_parity_err_cnt( 2403 const struct cntr_entry *entry, 2404 void *context, int vl, int mode, u64 data) 2405 { 2406 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2407 2408 return dd->rcv_err_status_cnt[41]; 2409 } 2410 2411 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt( 2412 const struct cntr_entry *entry, 2413 void *context, int vl, int mode, u64 data) 2414 { 2415 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2416 2417 return dd->rcv_err_status_cnt[40]; 2418 } 2419 2420 static u64 access_rx_lookup_des_part1_unc_err_cnt( 2421 const struct cntr_entry *entry, 2422 void *context, int vl, int mode, u64 data) 2423 { 2424 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2425 2426 return dd->rcv_err_status_cnt[39]; 2427 } 2428 2429 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt( 2430 const struct cntr_entry *entry, 2431 void *context, int vl, int mode, u64 data) 2432 { 2433 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2434 2435 return dd->rcv_err_status_cnt[38]; 2436 } 2437 2438 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt( 2439 const struct cntr_entry *entry, 2440 void *context, int vl, int mode, u64 data) 2441 { 2442 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2443 2444 return dd->rcv_err_status_cnt[37]; 2445 } 2446 2447 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt( 2448 const struct cntr_entry *entry, 2449 void *context, int vl, int mode, u64 data) 2450 { 2451 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2452 2453 return dd->rcv_err_status_cnt[36]; 2454 } 2455 2456 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt( 2457 const struct cntr_entry *entry, 2458 void *context, int vl, int mode, u64 data) 2459 { 2460 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2461 2462 return dd->rcv_err_status_cnt[35]; 2463 } 2464 2465 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt( 2466 const struct cntr_entry *entry, 2467 void *context, int vl, int mode, u64 data) 2468 { 2469 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2470 2471 return dd->rcv_err_status_cnt[34]; 2472 } 2473 2474 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt( 2475 const struct cntr_entry *entry, 2476 void *context, int vl, int mode, u64 data) 2477 { 2478 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2479 2480 return dd->rcv_err_status_cnt[33]; 2481 } 2482 2483 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry, 2484 void *context, int vl, int mode, 2485 u64 data) 2486 { 2487 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2488 2489 return dd->rcv_err_status_cnt[32]; 2490 } 2491 2492 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry, 2493 void *context, int vl, int mode, 2494 u64 data) 2495 { 2496 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2497 2498 return dd->rcv_err_status_cnt[31]; 2499 } 2500 2501 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry, 2502 void *context, int vl, int mode, 2503 u64 data) 2504 { 2505 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2506 2507 return dd->rcv_err_status_cnt[30]; 2508 } 2509 2510 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry, 2511 void *context, int vl, int mode, 2512 u64 data) 2513 { 2514 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2515 2516 return dd->rcv_err_status_cnt[29]; 2517 } 2518 2519 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry, 2520 void *context, int vl, 2521 int mode, u64 data) 2522 { 2523 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2524 2525 return dd->rcv_err_status_cnt[28]; 2526 } 2527 2528 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt( 2529 const struct cntr_entry *entry, 2530 void *context, int vl, int mode, u64 data) 2531 { 2532 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2533 2534 return dd->rcv_err_status_cnt[27]; 2535 } 2536 2537 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt( 2538 const struct cntr_entry *entry, 2539 void *context, int vl, int mode, u64 data) 2540 { 2541 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2542 2543 return dd->rcv_err_status_cnt[26]; 2544 } 2545 2546 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt( 2547 const struct cntr_entry *entry, 2548 void *context, int vl, int mode, u64 data) 2549 { 2550 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2551 2552 return dd->rcv_err_status_cnt[25]; 2553 } 2554 2555 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt( 2556 const struct cntr_entry *entry, 2557 void *context, int vl, int mode, u64 data) 2558 { 2559 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2560 2561 return dd->rcv_err_status_cnt[24]; 2562 } 2563 2564 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt( 2565 const struct cntr_entry *entry, 2566 void *context, int vl, int mode, u64 data) 2567 { 2568 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2569 2570 return dd->rcv_err_status_cnt[23]; 2571 } 2572 2573 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt( 2574 const struct cntr_entry *entry, 2575 void *context, int vl, int mode, u64 data) 2576 { 2577 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2578 2579 return dd->rcv_err_status_cnt[22]; 2580 } 2581 2582 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt( 2583 const struct cntr_entry *entry, 2584 void *context, int vl, int mode, u64 data) 2585 { 2586 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2587 2588 return dd->rcv_err_status_cnt[21]; 2589 } 2590 2591 static u64 access_rx_rbuf_block_list_read_cor_err_cnt( 2592 const struct cntr_entry *entry, 2593 void *context, int vl, int mode, u64 data) 2594 { 2595 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2596 2597 return dd->rcv_err_status_cnt[20]; 2598 } 2599 2600 static u64 access_rx_rbuf_block_list_read_unc_err_cnt( 2601 const struct cntr_entry *entry, 2602 void *context, int vl, int mode, u64 data) 2603 { 2604 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2605 2606 return dd->rcv_err_status_cnt[19]; 2607 } 2608 2609 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry, 2610 void *context, int vl, 2611 int mode, u64 data) 2612 { 2613 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2614 2615 return dd->rcv_err_status_cnt[18]; 2616 } 2617 2618 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry, 2619 void *context, int vl, 2620 int mode, u64 data) 2621 { 2622 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2623 2624 return dd->rcv_err_status_cnt[17]; 2625 } 2626 2627 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt( 2628 const struct cntr_entry *entry, 2629 void *context, int vl, int mode, u64 data) 2630 { 2631 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2632 2633 return dd->rcv_err_status_cnt[16]; 2634 } 2635 2636 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt( 2637 const struct cntr_entry *entry, 2638 void *context, int vl, int mode, u64 data) 2639 { 2640 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2641 2642 return dd->rcv_err_status_cnt[15]; 2643 } 2644 2645 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry, 2646 void *context, int vl, 2647 int mode, u64 data) 2648 { 2649 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2650 2651 return dd->rcv_err_status_cnt[14]; 2652 } 2653 2654 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry, 2655 void *context, int vl, 2656 int mode, u64 data) 2657 { 2658 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2659 2660 return dd->rcv_err_status_cnt[13]; 2661 } 2662 2663 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2664 void *context, int vl, int mode, 2665 u64 data) 2666 { 2667 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2668 2669 return dd->rcv_err_status_cnt[12]; 2670 } 2671 2672 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry, 2673 void *context, int vl, int mode, 2674 u64 data) 2675 { 2676 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2677 2678 return dd->rcv_err_status_cnt[11]; 2679 } 2680 2681 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry, 2682 void *context, int vl, int mode, 2683 u64 data) 2684 { 2685 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2686 2687 return dd->rcv_err_status_cnt[10]; 2688 } 2689 2690 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry, 2691 void *context, int vl, int mode, 2692 u64 data) 2693 { 2694 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2695 2696 return dd->rcv_err_status_cnt[9]; 2697 } 2698 2699 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry, 2700 void *context, int vl, int mode, 2701 u64 data) 2702 { 2703 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2704 2705 return dd->rcv_err_status_cnt[8]; 2706 } 2707 2708 static u64 access_rx_rcv_qp_map_table_cor_err_cnt( 2709 const struct cntr_entry *entry, 2710 void *context, int vl, int mode, u64 data) 2711 { 2712 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2713 2714 return dd->rcv_err_status_cnt[7]; 2715 } 2716 2717 static u64 access_rx_rcv_qp_map_table_unc_err_cnt( 2718 const struct cntr_entry *entry, 2719 void *context, int vl, int mode, u64 data) 2720 { 2721 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2722 2723 return dd->rcv_err_status_cnt[6]; 2724 } 2725 2726 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry, 2727 void *context, int vl, int mode, 2728 u64 data) 2729 { 2730 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2731 2732 return dd->rcv_err_status_cnt[5]; 2733 } 2734 2735 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry, 2736 void *context, int vl, int mode, 2737 u64 data) 2738 { 2739 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2740 2741 return dd->rcv_err_status_cnt[4]; 2742 } 2743 2744 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry, 2745 void *context, int vl, int mode, 2746 u64 data) 2747 { 2748 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2749 2750 return dd->rcv_err_status_cnt[3]; 2751 } 2752 2753 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry, 2754 void *context, int vl, int mode, 2755 u64 data) 2756 { 2757 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2758 2759 return dd->rcv_err_status_cnt[2]; 2760 } 2761 2762 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry, 2763 void *context, int vl, int mode, 2764 u64 data) 2765 { 2766 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2767 2768 return dd->rcv_err_status_cnt[1]; 2769 } 2770 2771 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry, 2772 void *context, int vl, int mode, 2773 u64 data) 2774 { 2775 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2776 2777 return dd->rcv_err_status_cnt[0]; 2778 } 2779 2780 /* 2781 * Software counters corresponding to each of the 2782 * error status bits within SendPioErrStatus 2783 */ 2784 static u64 access_pio_pec_sop_head_parity_err_cnt( 2785 const struct cntr_entry *entry, 2786 void *context, int vl, int mode, u64 data) 2787 { 2788 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2789 2790 return dd->send_pio_err_status_cnt[35]; 2791 } 2792 2793 static u64 access_pio_pcc_sop_head_parity_err_cnt( 2794 const struct cntr_entry *entry, 2795 void *context, int vl, int mode, u64 data) 2796 { 2797 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2798 2799 return dd->send_pio_err_status_cnt[34]; 2800 } 2801 2802 static u64 access_pio_last_returned_cnt_parity_err_cnt( 2803 const struct cntr_entry *entry, 2804 void *context, int vl, int mode, u64 data) 2805 { 2806 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2807 2808 return dd->send_pio_err_status_cnt[33]; 2809 } 2810 2811 static u64 access_pio_current_free_cnt_parity_err_cnt( 2812 const struct cntr_entry *entry, 2813 void *context, int vl, int mode, u64 data) 2814 { 2815 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2816 2817 return dd->send_pio_err_status_cnt[32]; 2818 } 2819 2820 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry, 2821 void *context, int vl, int mode, 2822 u64 data) 2823 { 2824 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2825 2826 return dd->send_pio_err_status_cnt[31]; 2827 } 2828 2829 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry, 2830 void *context, int vl, int mode, 2831 u64 data) 2832 { 2833 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2834 2835 return dd->send_pio_err_status_cnt[30]; 2836 } 2837 2838 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry, 2839 void *context, int vl, int mode, 2840 u64 data) 2841 { 2842 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2843 2844 return dd->send_pio_err_status_cnt[29]; 2845 } 2846 2847 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt( 2848 const struct cntr_entry *entry, 2849 void *context, int vl, int mode, u64 data) 2850 { 2851 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2852 2853 return dd->send_pio_err_status_cnt[28]; 2854 } 2855 2856 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry, 2857 void *context, int vl, int mode, 2858 u64 data) 2859 { 2860 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2861 2862 return dd->send_pio_err_status_cnt[27]; 2863 } 2864 2865 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry, 2866 void *context, int vl, int mode, 2867 u64 data) 2868 { 2869 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2870 2871 return dd->send_pio_err_status_cnt[26]; 2872 } 2873 2874 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry, 2875 void *context, int vl, 2876 int mode, u64 data) 2877 { 2878 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2879 2880 return dd->send_pio_err_status_cnt[25]; 2881 } 2882 2883 static u64 access_pio_block_qw_count_parity_err_cnt( 2884 const struct cntr_entry *entry, 2885 void *context, int vl, int mode, u64 data) 2886 { 2887 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2888 2889 return dd->send_pio_err_status_cnt[24]; 2890 } 2891 2892 static u64 access_pio_write_qw_valid_parity_err_cnt( 2893 const struct cntr_entry *entry, 2894 void *context, int vl, int mode, u64 data) 2895 { 2896 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2897 2898 return dd->send_pio_err_status_cnt[23]; 2899 } 2900 2901 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry, 2902 void *context, int vl, int mode, 2903 u64 data) 2904 { 2905 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2906 2907 return dd->send_pio_err_status_cnt[22]; 2908 } 2909 2910 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry, 2911 void *context, int vl, 2912 int mode, u64 data) 2913 { 2914 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2915 2916 return dd->send_pio_err_status_cnt[21]; 2917 } 2918 2919 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry, 2920 void *context, int vl, 2921 int mode, u64 data) 2922 { 2923 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2924 2925 return dd->send_pio_err_status_cnt[20]; 2926 } 2927 2928 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry, 2929 void *context, int vl, 2930 int mode, u64 data) 2931 { 2932 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2933 2934 return dd->send_pio_err_status_cnt[19]; 2935 } 2936 2937 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt( 2938 const struct cntr_entry *entry, 2939 void *context, int vl, int mode, u64 data) 2940 { 2941 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2942 2943 return dd->send_pio_err_status_cnt[18]; 2944 } 2945 2946 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry, 2947 void *context, int vl, int mode, 2948 u64 data) 2949 { 2950 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2951 2952 return dd->send_pio_err_status_cnt[17]; 2953 } 2954 2955 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry, 2956 void *context, int vl, int mode, 2957 u64 data) 2958 { 2959 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2960 2961 return dd->send_pio_err_status_cnt[16]; 2962 } 2963 2964 static u64 access_pio_credit_ret_fifo_parity_err_cnt( 2965 const struct cntr_entry *entry, 2966 void *context, int vl, int mode, u64 data) 2967 { 2968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2969 2970 return dd->send_pio_err_status_cnt[15]; 2971 } 2972 2973 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt( 2974 const struct cntr_entry *entry, 2975 void *context, int vl, int mode, u64 data) 2976 { 2977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2978 2979 return dd->send_pio_err_status_cnt[14]; 2980 } 2981 2982 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt( 2983 const struct cntr_entry *entry, 2984 void *context, int vl, int mode, u64 data) 2985 { 2986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2987 2988 return dd->send_pio_err_status_cnt[13]; 2989 } 2990 2991 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt( 2992 const struct cntr_entry *entry, 2993 void *context, int vl, int mode, u64 data) 2994 { 2995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2996 2997 return dd->send_pio_err_status_cnt[12]; 2998 } 2999 3000 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt( 3001 const struct cntr_entry *entry, 3002 void *context, int vl, int mode, u64 data) 3003 { 3004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3005 3006 return dd->send_pio_err_status_cnt[11]; 3007 } 3008 3009 static u64 access_pio_sm_pkt_reset_parity_err_cnt( 3010 const struct cntr_entry *entry, 3011 void *context, int vl, int mode, u64 data) 3012 { 3013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3014 3015 return dd->send_pio_err_status_cnt[10]; 3016 } 3017 3018 static u64 access_pio_pkt_evict_fifo_parity_err_cnt( 3019 const struct cntr_entry *entry, 3020 void *context, int vl, int mode, u64 data) 3021 { 3022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3023 3024 return dd->send_pio_err_status_cnt[9]; 3025 } 3026 3027 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt( 3028 const struct cntr_entry *entry, 3029 void *context, int vl, int mode, u64 data) 3030 { 3031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3032 3033 return dd->send_pio_err_status_cnt[8]; 3034 } 3035 3036 static u64 access_pio_sbrdctl_crrel_parity_err_cnt( 3037 const struct cntr_entry *entry, 3038 void *context, int vl, int mode, u64 data) 3039 { 3040 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3041 3042 return dd->send_pio_err_status_cnt[7]; 3043 } 3044 3045 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry, 3046 void *context, int vl, int mode, 3047 u64 data) 3048 { 3049 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3050 3051 return dd->send_pio_err_status_cnt[6]; 3052 } 3053 3054 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry, 3055 void *context, int vl, int mode, 3056 u64 data) 3057 { 3058 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3059 3060 return dd->send_pio_err_status_cnt[5]; 3061 } 3062 3063 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry, 3064 void *context, int vl, int mode, 3065 u64 data) 3066 { 3067 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3068 3069 return dd->send_pio_err_status_cnt[4]; 3070 } 3071 3072 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry, 3073 void *context, int vl, int mode, 3074 u64 data) 3075 { 3076 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3077 3078 return dd->send_pio_err_status_cnt[3]; 3079 } 3080 3081 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry, 3082 void *context, int vl, int mode, 3083 u64 data) 3084 { 3085 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3086 3087 return dd->send_pio_err_status_cnt[2]; 3088 } 3089 3090 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry, 3091 void *context, int vl, 3092 int mode, u64 data) 3093 { 3094 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3095 3096 return dd->send_pio_err_status_cnt[1]; 3097 } 3098 3099 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry, 3100 void *context, int vl, int mode, 3101 u64 data) 3102 { 3103 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3104 3105 return dd->send_pio_err_status_cnt[0]; 3106 } 3107 3108 /* 3109 * Software counters corresponding to each of the 3110 * error status bits within SendDmaErrStatus 3111 */ 3112 static u64 access_sdma_pcie_req_tracking_cor_err_cnt( 3113 const struct cntr_entry *entry, 3114 void *context, int vl, int mode, u64 data) 3115 { 3116 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3117 3118 return dd->send_dma_err_status_cnt[3]; 3119 } 3120 3121 static u64 access_sdma_pcie_req_tracking_unc_err_cnt( 3122 const struct cntr_entry *entry, 3123 void *context, int vl, int mode, u64 data) 3124 { 3125 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3126 3127 return dd->send_dma_err_status_cnt[2]; 3128 } 3129 3130 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry, 3131 void *context, int vl, int mode, 3132 u64 data) 3133 { 3134 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3135 3136 return dd->send_dma_err_status_cnt[1]; 3137 } 3138 3139 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry, 3140 void *context, int vl, int mode, 3141 u64 data) 3142 { 3143 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3144 3145 return dd->send_dma_err_status_cnt[0]; 3146 } 3147 3148 /* 3149 * Software counters corresponding to each of the 3150 * error status bits within SendEgressErrStatus 3151 */ 3152 static u64 access_tx_read_pio_memory_csr_unc_err_cnt( 3153 const struct cntr_entry *entry, 3154 void *context, int vl, int mode, u64 data) 3155 { 3156 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3157 3158 return dd->send_egress_err_status_cnt[63]; 3159 } 3160 3161 static u64 access_tx_read_sdma_memory_csr_err_cnt( 3162 const struct cntr_entry *entry, 3163 void *context, int vl, int mode, u64 data) 3164 { 3165 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3166 3167 return dd->send_egress_err_status_cnt[62]; 3168 } 3169 3170 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry, 3171 void *context, int vl, int mode, 3172 u64 data) 3173 { 3174 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3175 3176 return dd->send_egress_err_status_cnt[61]; 3177 } 3178 3179 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry, 3180 void *context, int vl, 3181 int mode, u64 data) 3182 { 3183 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3184 3185 return dd->send_egress_err_status_cnt[60]; 3186 } 3187 3188 static u64 access_tx_read_sdma_memory_cor_err_cnt( 3189 const struct cntr_entry *entry, 3190 void *context, int vl, int mode, u64 data) 3191 { 3192 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3193 3194 return dd->send_egress_err_status_cnt[59]; 3195 } 3196 3197 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry, 3198 void *context, int vl, int mode, 3199 u64 data) 3200 { 3201 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3202 3203 return dd->send_egress_err_status_cnt[58]; 3204 } 3205 3206 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry, 3207 void *context, int vl, int mode, 3208 u64 data) 3209 { 3210 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3211 3212 return dd->send_egress_err_status_cnt[57]; 3213 } 3214 3215 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry, 3216 void *context, int vl, int mode, 3217 u64 data) 3218 { 3219 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3220 3221 return dd->send_egress_err_status_cnt[56]; 3222 } 3223 3224 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry, 3225 void *context, int vl, int mode, 3226 u64 data) 3227 { 3228 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3229 3230 return dd->send_egress_err_status_cnt[55]; 3231 } 3232 3233 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry, 3234 void *context, int vl, int mode, 3235 u64 data) 3236 { 3237 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3238 3239 return dd->send_egress_err_status_cnt[54]; 3240 } 3241 3242 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry, 3243 void *context, int vl, int mode, 3244 u64 data) 3245 { 3246 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3247 3248 return dd->send_egress_err_status_cnt[53]; 3249 } 3250 3251 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry, 3252 void *context, int vl, int mode, 3253 u64 data) 3254 { 3255 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3256 3257 return dd->send_egress_err_status_cnt[52]; 3258 } 3259 3260 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry, 3261 void *context, int vl, int mode, 3262 u64 data) 3263 { 3264 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3265 3266 return dd->send_egress_err_status_cnt[51]; 3267 } 3268 3269 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry, 3270 void *context, int vl, int mode, 3271 u64 data) 3272 { 3273 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3274 3275 return dd->send_egress_err_status_cnt[50]; 3276 } 3277 3278 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry, 3279 void *context, int vl, int mode, 3280 u64 data) 3281 { 3282 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3283 3284 return dd->send_egress_err_status_cnt[49]; 3285 } 3286 3287 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry, 3288 void *context, int vl, int mode, 3289 u64 data) 3290 { 3291 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3292 3293 return dd->send_egress_err_status_cnt[48]; 3294 } 3295 3296 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry, 3297 void *context, int vl, int mode, 3298 u64 data) 3299 { 3300 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3301 3302 return dd->send_egress_err_status_cnt[47]; 3303 } 3304 3305 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry, 3306 void *context, int vl, int mode, 3307 u64 data) 3308 { 3309 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3310 3311 return dd->send_egress_err_status_cnt[46]; 3312 } 3313 3314 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry, 3315 void *context, int vl, int mode, 3316 u64 data) 3317 { 3318 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3319 3320 return dd->send_egress_err_status_cnt[45]; 3321 } 3322 3323 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry, 3324 void *context, int vl, 3325 int mode, u64 data) 3326 { 3327 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3328 3329 return dd->send_egress_err_status_cnt[44]; 3330 } 3331 3332 static u64 access_tx_read_sdma_memory_unc_err_cnt( 3333 const struct cntr_entry *entry, 3334 void *context, int vl, int mode, u64 data) 3335 { 3336 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3337 3338 return dd->send_egress_err_status_cnt[43]; 3339 } 3340 3341 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry, 3342 void *context, int vl, int mode, 3343 u64 data) 3344 { 3345 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3346 3347 return dd->send_egress_err_status_cnt[42]; 3348 } 3349 3350 static u64 access_tx_credit_return_partiy_err_cnt( 3351 const struct cntr_entry *entry, 3352 void *context, int vl, int mode, u64 data) 3353 { 3354 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3355 3356 return dd->send_egress_err_status_cnt[41]; 3357 } 3358 3359 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt( 3360 const struct cntr_entry *entry, 3361 void *context, int vl, int mode, u64 data) 3362 { 3363 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3364 3365 return dd->send_egress_err_status_cnt[40]; 3366 } 3367 3368 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt( 3369 const struct cntr_entry *entry, 3370 void *context, int vl, int mode, u64 data) 3371 { 3372 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3373 3374 return dd->send_egress_err_status_cnt[39]; 3375 } 3376 3377 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt( 3378 const struct cntr_entry *entry, 3379 void *context, int vl, int mode, u64 data) 3380 { 3381 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3382 3383 return dd->send_egress_err_status_cnt[38]; 3384 } 3385 3386 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt( 3387 const struct cntr_entry *entry, 3388 void *context, int vl, int mode, u64 data) 3389 { 3390 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3391 3392 return dd->send_egress_err_status_cnt[37]; 3393 } 3394 3395 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt( 3396 const struct cntr_entry *entry, 3397 void *context, int vl, int mode, u64 data) 3398 { 3399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3400 3401 return dd->send_egress_err_status_cnt[36]; 3402 } 3403 3404 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt( 3405 const struct cntr_entry *entry, 3406 void *context, int vl, int mode, u64 data) 3407 { 3408 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3409 3410 return dd->send_egress_err_status_cnt[35]; 3411 } 3412 3413 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt( 3414 const struct cntr_entry *entry, 3415 void *context, int vl, int mode, u64 data) 3416 { 3417 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3418 3419 return dd->send_egress_err_status_cnt[34]; 3420 } 3421 3422 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt( 3423 const struct cntr_entry *entry, 3424 void *context, int vl, int mode, u64 data) 3425 { 3426 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3427 3428 return dd->send_egress_err_status_cnt[33]; 3429 } 3430 3431 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt( 3432 const struct cntr_entry *entry, 3433 void *context, int vl, int mode, u64 data) 3434 { 3435 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3436 3437 return dd->send_egress_err_status_cnt[32]; 3438 } 3439 3440 static u64 access_tx_sdma15_disallowed_packet_err_cnt( 3441 const struct cntr_entry *entry, 3442 void *context, int vl, int mode, u64 data) 3443 { 3444 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3445 3446 return dd->send_egress_err_status_cnt[31]; 3447 } 3448 3449 static u64 access_tx_sdma14_disallowed_packet_err_cnt( 3450 const struct cntr_entry *entry, 3451 void *context, int vl, int mode, u64 data) 3452 { 3453 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3454 3455 return dd->send_egress_err_status_cnt[30]; 3456 } 3457 3458 static u64 access_tx_sdma13_disallowed_packet_err_cnt( 3459 const struct cntr_entry *entry, 3460 void *context, int vl, int mode, u64 data) 3461 { 3462 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3463 3464 return dd->send_egress_err_status_cnt[29]; 3465 } 3466 3467 static u64 access_tx_sdma12_disallowed_packet_err_cnt( 3468 const struct cntr_entry *entry, 3469 void *context, int vl, int mode, u64 data) 3470 { 3471 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3472 3473 return dd->send_egress_err_status_cnt[28]; 3474 } 3475 3476 static u64 access_tx_sdma11_disallowed_packet_err_cnt( 3477 const struct cntr_entry *entry, 3478 void *context, int vl, int mode, u64 data) 3479 { 3480 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3481 3482 return dd->send_egress_err_status_cnt[27]; 3483 } 3484 3485 static u64 access_tx_sdma10_disallowed_packet_err_cnt( 3486 const struct cntr_entry *entry, 3487 void *context, int vl, int mode, u64 data) 3488 { 3489 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3490 3491 return dd->send_egress_err_status_cnt[26]; 3492 } 3493 3494 static u64 access_tx_sdma9_disallowed_packet_err_cnt( 3495 const struct cntr_entry *entry, 3496 void *context, int vl, int mode, u64 data) 3497 { 3498 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3499 3500 return dd->send_egress_err_status_cnt[25]; 3501 } 3502 3503 static u64 access_tx_sdma8_disallowed_packet_err_cnt( 3504 const struct cntr_entry *entry, 3505 void *context, int vl, int mode, u64 data) 3506 { 3507 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3508 3509 return dd->send_egress_err_status_cnt[24]; 3510 } 3511 3512 static u64 access_tx_sdma7_disallowed_packet_err_cnt( 3513 const struct cntr_entry *entry, 3514 void *context, int vl, int mode, u64 data) 3515 { 3516 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3517 3518 return dd->send_egress_err_status_cnt[23]; 3519 } 3520 3521 static u64 access_tx_sdma6_disallowed_packet_err_cnt( 3522 const struct cntr_entry *entry, 3523 void *context, int vl, int mode, u64 data) 3524 { 3525 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3526 3527 return dd->send_egress_err_status_cnt[22]; 3528 } 3529 3530 static u64 access_tx_sdma5_disallowed_packet_err_cnt( 3531 const struct cntr_entry *entry, 3532 void *context, int vl, int mode, u64 data) 3533 { 3534 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3535 3536 return dd->send_egress_err_status_cnt[21]; 3537 } 3538 3539 static u64 access_tx_sdma4_disallowed_packet_err_cnt( 3540 const struct cntr_entry *entry, 3541 void *context, int vl, int mode, u64 data) 3542 { 3543 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3544 3545 return dd->send_egress_err_status_cnt[20]; 3546 } 3547 3548 static u64 access_tx_sdma3_disallowed_packet_err_cnt( 3549 const struct cntr_entry *entry, 3550 void *context, int vl, int mode, u64 data) 3551 { 3552 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3553 3554 return dd->send_egress_err_status_cnt[19]; 3555 } 3556 3557 static u64 access_tx_sdma2_disallowed_packet_err_cnt( 3558 const struct cntr_entry *entry, 3559 void *context, int vl, int mode, u64 data) 3560 { 3561 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3562 3563 return dd->send_egress_err_status_cnt[18]; 3564 } 3565 3566 static u64 access_tx_sdma1_disallowed_packet_err_cnt( 3567 const struct cntr_entry *entry, 3568 void *context, int vl, int mode, u64 data) 3569 { 3570 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3571 3572 return dd->send_egress_err_status_cnt[17]; 3573 } 3574 3575 static u64 access_tx_sdma0_disallowed_packet_err_cnt( 3576 const struct cntr_entry *entry, 3577 void *context, int vl, int mode, u64 data) 3578 { 3579 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3580 3581 return dd->send_egress_err_status_cnt[16]; 3582 } 3583 3584 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry, 3585 void *context, int vl, int mode, 3586 u64 data) 3587 { 3588 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3589 3590 return dd->send_egress_err_status_cnt[15]; 3591 } 3592 3593 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry, 3594 void *context, int vl, 3595 int mode, u64 data) 3596 { 3597 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3598 3599 return dd->send_egress_err_status_cnt[14]; 3600 } 3601 3602 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry, 3603 void *context, int vl, int mode, 3604 u64 data) 3605 { 3606 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3607 3608 return dd->send_egress_err_status_cnt[13]; 3609 } 3610 3611 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry, 3612 void *context, int vl, int mode, 3613 u64 data) 3614 { 3615 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3616 3617 return dd->send_egress_err_status_cnt[12]; 3618 } 3619 3620 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt( 3621 const struct cntr_entry *entry, 3622 void *context, int vl, int mode, u64 data) 3623 { 3624 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3625 3626 return dd->send_egress_err_status_cnt[11]; 3627 } 3628 3629 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry, 3630 void *context, int vl, int mode, 3631 u64 data) 3632 { 3633 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3634 3635 return dd->send_egress_err_status_cnt[10]; 3636 } 3637 3638 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry, 3639 void *context, int vl, int mode, 3640 u64 data) 3641 { 3642 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3643 3644 return dd->send_egress_err_status_cnt[9]; 3645 } 3646 3647 static u64 access_tx_sdma_launch_intf_parity_err_cnt( 3648 const struct cntr_entry *entry, 3649 void *context, int vl, int mode, u64 data) 3650 { 3651 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3652 3653 return dd->send_egress_err_status_cnt[8]; 3654 } 3655 3656 static u64 access_tx_pio_launch_intf_parity_err_cnt( 3657 const struct cntr_entry *entry, 3658 void *context, int vl, int mode, u64 data) 3659 { 3660 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3661 3662 return dd->send_egress_err_status_cnt[7]; 3663 } 3664 3665 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry, 3666 void *context, int vl, int mode, 3667 u64 data) 3668 { 3669 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3670 3671 return dd->send_egress_err_status_cnt[6]; 3672 } 3673 3674 static u64 access_tx_incorrect_link_state_err_cnt( 3675 const struct cntr_entry *entry, 3676 void *context, int vl, int mode, u64 data) 3677 { 3678 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3679 3680 return dd->send_egress_err_status_cnt[5]; 3681 } 3682 3683 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry, 3684 void *context, int vl, int mode, 3685 u64 data) 3686 { 3687 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3688 3689 return dd->send_egress_err_status_cnt[4]; 3690 } 3691 3692 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt( 3693 const struct cntr_entry *entry, 3694 void *context, int vl, int mode, u64 data) 3695 { 3696 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3697 3698 return dd->send_egress_err_status_cnt[3]; 3699 } 3700 3701 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry, 3702 void *context, int vl, int mode, 3703 u64 data) 3704 { 3705 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3706 3707 return dd->send_egress_err_status_cnt[2]; 3708 } 3709 3710 static u64 access_tx_pkt_integrity_mem_unc_err_cnt( 3711 const struct cntr_entry *entry, 3712 void *context, int vl, int mode, u64 data) 3713 { 3714 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3715 3716 return dd->send_egress_err_status_cnt[1]; 3717 } 3718 3719 static u64 access_tx_pkt_integrity_mem_cor_err_cnt( 3720 const struct cntr_entry *entry, 3721 void *context, int vl, int mode, u64 data) 3722 { 3723 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3724 3725 return dd->send_egress_err_status_cnt[0]; 3726 } 3727 3728 /* 3729 * Software counters corresponding to each of the 3730 * error status bits within SendErrStatus 3731 */ 3732 static u64 access_send_csr_write_bad_addr_err_cnt( 3733 const struct cntr_entry *entry, 3734 void *context, int vl, int mode, u64 data) 3735 { 3736 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3737 3738 return dd->send_err_status_cnt[2]; 3739 } 3740 3741 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 3742 void *context, int vl, 3743 int mode, u64 data) 3744 { 3745 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3746 3747 return dd->send_err_status_cnt[1]; 3748 } 3749 3750 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry, 3751 void *context, int vl, int mode, 3752 u64 data) 3753 { 3754 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3755 3756 return dd->send_err_status_cnt[0]; 3757 } 3758 3759 /* 3760 * Software counters corresponding to each of the 3761 * error status bits within SendCtxtErrStatus 3762 */ 3763 static u64 access_pio_write_out_of_bounds_err_cnt( 3764 const struct cntr_entry *entry, 3765 void *context, int vl, int mode, u64 data) 3766 { 3767 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3768 3769 return dd->sw_ctxt_err_status_cnt[4]; 3770 } 3771 3772 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry, 3773 void *context, int vl, int mode, 3774 u64 data) 3775 { 3776 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3777 3778 return dd->sw_ctxt_err_status_cnt[3]; 3779 } 3780 3781 static u64 access_pio_write_crosses_boundary_err_cnt( 3782 const struct cntr_entry *entry, 3783 void *context, int vl, int mode, u64 data) 3784 { 3785 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3786 3787 return dd->sw_ctxt_err_status_cnt[2]; 3788 } 3789 3790 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry, 3791 void *context, int vl, 3792 int mode, u64 data) 3793 { 3794 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3795 3796 return dd->sw_ctxt_err_status_cnt[1]; 3797 } 3798 3799 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry, 3800 void *context, int vl, int mode, 3801 u64 data) 3802 { 3803 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3804 3805 return dd->sw_ctxt_err_status_cnt[0]; 3806 } 3807 3808 /* 3809 * Software counters corresponding to each of the 3810 * error status bits within SendDmaEngErrStatus 3811 */ 3812 static u64 access_sdma_header_request_fifo_cor_err_cnt( 3813 const struct cntr_entry *entry, 3814 void *context, int vl, int mode, u64 data) 3815 { 3816 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3817 3818 return dd->sw_send_dma_eng_err_status_cnt[23]; 3819 } 3820 3821 static u64 access_sdma_header_storage_cor_err_cnt( 3822 const struct cntr_entry *entry, 3823 void *context, int vl, int mode, u64 data) 3824 { 3825 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3826 3827 return dd->sw_send_dma_eng_err_status_cnt[22]; 3828 } 3829 3830 static u64 access_sdma_packet_tracking_cor_err_cnt( 3831 const struct cntr_entry *entry, 3832 void *context, int vl, int mode, u64 data) 3833 { 3834 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3835 3836 return dd->sw_send_dma_eng_err_status_cnt[21]; 3837 } 3838 3839 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry, 3840 void *context, int vl, int mode, 3841 u64 data) 3842 { 3843 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3844 3845 return dd->sw_send_dma_eng_err_status_cnt[20]; 3846 } 3847 3848 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry, 3849 void *context, int vl, int mode, 3850 u64 data) 3851 { 3852 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3853 3854 return dd->sw_send_dma_eng_err_status_cnt[19]; 3855 } 3856 3857 static u64 access_sdma_header_request_fifo_unc_err_cnt( 3858 const struct cntr_entry *entry, 3859 void *context, int vl, int mode, u64 data) 3860 { 3861 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3862 3863 return dd->sw_send_dma_eng_err_status_cnt[18]; 3864 } 3865 3866 static u64 access_sdma_header_storage_unc_err_cnt( 3867 const struct cntr_entry *entry, 3868 void *context, int vl, int mode, u64 data) 3869 { 3870 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3871 3872 return dd->sw_send_dma_eng_err_status_cnt[17]; 3873 } 3874 3875 static u64 access_sdma_packet_tracking_unc_err_cnt( 3876 const struct cntr_entry *entry, 3877 void *context, int vl, int mode, u64 data) 3878 { 3879 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3880 3881 return dd->sw_send_dma_eng_err_status_cnt[16]; 3882 } 3883 3884 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry, 3885 void *context, int vl, int mode, 3886 u64 data) 3887 { 3888 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3889 3890 return dd->sw_send_dma_eng_err_status_cnt[15]; 3891 } 3892 3893 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry, 3894 void *context, int vl, int mode, 3895 u64 data) 3896 { 3897 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3898 3899 return dd->sw_send_dma_eng_err_status_cnt[14]; 3900 } 3901 3902 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry, 3903 void *context, int vl, int mode, 3904 u64 data) 3905 { 3906 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3907 3908 return dd->sw_send_dma_eng_err_status_cnt[13]; 3909 } 3910 3911 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry, 3912 void *context, int vl, int mode, 3913 u64 data) 3914 { 3915 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3916 3917 return dd->sw_send_dma_eng_err_status_cnt[12]; 3918 } 3919 3920 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry, 3921 void *context, int vl, int mode, 3922 u64 data) 3923 { 3924 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3925 3926 return dd->sw_send_dma_eng_err_status_cnt[11]; 3927 } 3928 3929 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry, 3930 void *context, int vl, int mode, 3931 u64 data) 3932 { 3933 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3934 3935 return dd->sw_send_dma_eng_err_status_cnt[10]; 3936 } 3937 3938 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry, 3939 void *context, int vl, int mode, 3940 u64 data) 3941 { 3942 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3943 3944 return dd->sw_send_dma_eng_err_status_cnt[9]; 3945 } 3946 3947 static u64 access_sdma_packet_desc_overflow_err_cnt( 3948 const struct cntr_entry *entry, 3949 void *context, int vl, int mode, u64 data) 3950 { 3951 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3952 3953 return dd->sw_send_dma_eng_err_status_cnt[8]; 3954 } 3955 3956 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry, 3957 void *context, int vl, 3958 int mode, u64 data) 3959 { 3960 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3961 3962 return dd->sw_send_dma_eng_err_status_cnt[7]; 3963 } 3964 3965 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry, 3966 void *context, int vl, int mode, u64 data) 3967 { 3968 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3969 3970 return dd->sw_send_dma_eng_err_status_cnt[6]; 3971 } 3972 3973 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry, 3974 void *context, int vl, int mode, 3975 u64 data) 3976 { 3977 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3978 3979 return dd->sw_send_dma_eng_err_status_cnt[5]; 3980 } 3981 3982 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry, 3983 void *context, int vl, int mode, 3984 u64 data) 3985 { 3986 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3987 3988 return dd->sw_send_dma_eng_err_status_cnt[4]; 3989 } 3990 3991 static u64 access_sdma_tail_out_of_bounds_err_cnt( 3992 const struct cntr_entry *entry, 3993 void *context, int vl, int mode, u64 data) 3994 { 3995 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3996 3997 return dd->sw_send_dma_eng_err_status_cnt[3]; 3998 } 3999 4000 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry, 4001 void *context, int vl, int mode, 4002 u64 data) 4003 { 4004 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4005 4006 return dd->sw_send_dma_eng_err_status_cnt[2]; 4007 } 4008 4009 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry, 4010 void *context, int vl, int mode, 4011 u64 data) 4012 { 4013 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4014 4015 return dd->sw_send_dma_eng_err_status_cnt[1]; 4016 } 4017 4018 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry, 4019 void *context, int vl, int mode, 4020 u64 data) 4021 { 4022 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4023 4024 return dd->sw_send_dma_eng_err_status_cnt[0]; 4025 } 4026 4027 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry, 4028 void *context, int vl, int mode, 4029 u64 data) 4030 { 4031 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4032 4033 u64 val = 0; 4034 u64 csr = entry->csr; 4035 4036 val = read_write_csr(dd, csr, mode, data); 4037 if (mode == CNTR_MODE_R) { 4038 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ? 4039 CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors; 4040 } else if (mode == CNTR_MODE_W) { 4041 dd->sw_rcv_bypass_packet_errors = 0; 4042 } else { 4043 dd_dev_err(dd, "Invalid cntr register access mode"); 4044 return 0; 4045 } 4046 return val; 4047 } 4048 4049 #define def_access_sw_cpu(cntr) \ 4050 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \ 4051 void *context, int vl, int mode, u64 data) \ 4052 { \ 4053 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 4054 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \ 4055 ppd->ibport_data.rvp.cntr, vl, \ 4056 mode, data); \ 4057 } 4058 4059 def_access_sw_cpu(rc_acks); 4060 def_access_sw_cpu(rc_qacks); 4061 def_access_sw_cpu(rc_delayed_comp); 4062 4063 #define def_access_ibp_counter(cntr) \ 4064 static u64 access_ibp_##cntr(const struct cntr_entry *entry, \ 4065 void *context, int vl, int mode, u64 data) \ 4066 { \ 4067 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 4068 \ 4069 if (vl != CNTR_INVALID_VL) \ 4070 return 0; \ 4071 \ 4072 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \ 4073 mode, data); \ 4074 } 4075 4076 def_access_ibp_counter(loop_pkts); 4077 def_access_ibp_counter(rc_resends); 4078 def_access_ibp_counter(rnr_naks); 4079 def_access_ibp_counter(other_naks); 4080 def_access_ibp_counter(rc_timeouts); 4081 def_access_ibp_counter(pkt_drops); 4082 def_access_ibp_counter(dmawait); 4083 def_access_ibp_counter(rc_seqnak); 4084 def_access_ibp_counter(rc_dupreq); 4085 def_access_ibp_counter(rdma_seq); 4086 def_access_ibp_counter(unaligned); 4087 def_access_ibp_counter(seq_naks); 4088 def_access_ibp_counter(rc_crwaits); 4089 4090 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { 4091 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH), 4092 [C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH), 4093 [C_RX_SHORT_ERR] = RXE32_DEV_CNTR_ELEM(RxShrErr, RCV_SHORT_ERR_CNT, CNTR_SYNTH), 4094 [C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH), 4095 [C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH), 4096 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT, 4097 CNTR_NORMAL), 4098 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT, 4099 CNTR_NORMAL), 4100 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs, 4101 RCV_TID_FLOW_GEN_MISMATCH_CNT, 4102 CNTR_NORMAL), 4103 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL, 4104 CNTR_NORMAL), 4105 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs, 4106 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL), 4107 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt, 4108 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL), 4109 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT, 4110 CNTR_NORMAL), 4111 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT, 4112 CNTR_NORMAL), 4113 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT, 4114 CNTR_NORMAL), 4115 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT, 4116 CNTR_NORMAL), 4117 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT, 4118 CNTR_NORMAL), 4119 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT, 4120 CNTR_NORMAL), 4121 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt, 4122 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL), 4123 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt, 4124 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL), 4125 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT, 4126 CNTR_SYNTH), 4127 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH, 4128 access_dc_rcv_err_cnt), 4129 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT, 4130 CNTR_SYNTH), 4131 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT, 4132 CNTR_SYNTH), 4133 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT, 4134 CNTR_SYNTH), 4135 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts, 4136 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH), 4137 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts, 4138 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT, 4139 CNTR_SYNTH), 4140 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr, 4141 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH), 4142 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT, 4143 CNTR_SYNTH), 4144 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT, 4145 CNTR_SYNTH), 4146 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT, 4147 CNTR_SYNTH), 4148 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT, 4149 CNTR_SYNTH), 4150 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT, 4151 CNTR_SYNTH), 4152 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT, 4153 CNTR_SYNTH), 4154 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT, 4155 CNTR_SYNTH), 4156 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT, 4157 CNTR_SYNTH | CNTR_VL), 4158 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT, 4159 CNTR_SYNTH | CNTR_VL), 4160 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH), 4161 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT, 4162 CNTR_SYNTH | CNTR_VL), 4163 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH), 4164 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT, 4165 CNTR_SYNTH | CNTR_VL), 4166 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT, 4167 CNTR_SYNTH), 4168 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT, 4169 CNTR_SYNTH | CNTR_VL), 4170 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT, 4171 CNTR_SYNTH), 4172 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT, 4173 CNTR_SYNTH | CNTR_VL), 4174 [C_DC_TOTAL_CRC] = 4175 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR, 4176 CNTR_SYNTH), 4177 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0, 4178 CNTR_SYNTH), 4179 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1, 4180 CNTR_SYNTH), 4181 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2, 4182 CNTR_SYNTH), 4183 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3, 4184 CNTR_SYNTH), 4185 [C_DC_CRC_MULT_LN] = 4186 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN, 4187 CNTR_SYNTH), 4188 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT, 4189 CNTR_SYNTH), 4190 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT, 4191 CNTR_SYNTH), 4192 [C_DC_SEQ_CRC_CNT] = 4193 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT, 4194 CNTR_SYNTH), 4195 [C_DC_ESC0_ONLY_CNT] = 4196 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT, 4197 CNTR_SYNTH), 4198 [C_DC_ESC0_PLUS1_CNT] = 4199 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT, 4200 CNTR_SYNTH), 4201 [C_DC_ESC0_PLUS2_CNT] = 4202 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT, 4203 CNTR_SYNTH), 4204 [C_DC_REINIT_FROM_PEER_CNT] = 4205 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 4206 CNTR_SYNTH), 4207 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT, 4208 CNTR_SYNTH), 4209 [C_DC_MISC_FLG_CNT] = 4210 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT, 4211 CNTR_SYNTH), 4212 [C_DC_PRF_GOOD_LTP_CNT] = 4213 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH), 4214 [C_DC_PRF_ACCEPTED_LTP_CNT] = 4215 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT, 4216 CNTR_SYNTH), 4217 [C_DC_PRF_RX_FLIT_CNT] = 4218 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH), 4219 [C_DC_PRF_TX_FLIT_CNT] = 4220 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH), 4221 [C_DC_PRF_CLK_CNTR] = 4222 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH), 4223 [C_DC_PG_DBG_FLIT_CRDTS_CNT] = 4224 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH), 4225 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] = 4226 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT, 4227 CNTR_SYNTH), 4228 [C_DC_PG_STS_TX_SBE_CNT] = 4229 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH), 4230 [C_DC_PG_STS_TX_MBE_CNT] = 4231 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT, 4232 CNTR_SYNTH), 4233 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL, 4234 access_sw_cpu_intr), 4235 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL, 4236 access_sw_cpu_rcv_limit), 4237 [C_SW_CTX0_SEQ_DROP] = CNTR_ELEM("SeqDrop0", 0, 0, CNTR_NORMAL, 4238 access_sw_ctx0_seq_drop), 4239 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL, 4240 access_sw_vtx_wait), 4241 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL, 4242 access_sw_pio_wait), 4243 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL, 4244 access_sw_pio_drain), 4245 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL, 4246 access_sw_kmem_wait), 4247 [C_SW_TID_WAIT] = CNTR_ELEM("TidWait", 0, 0, CNTR_NORMAL, 4248 hfi1_access_sw_tid_wait), 4249 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL, 4250 access_sw_send_schedule), 4251 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn", 4252 SEND_DMA_DESC_FETCHED_CNT, 0, 4253 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4254 dev_access_u32_csr), 4255 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0, 4256 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4257 access_sde_int_cnt), 4258 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0, 4259 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4260 access_sde_err_cnt), 4261 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0, 4262 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4263 access_sde_idle_int_cnt), 4264 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0, 4265 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4266 access_sde_progress_int_cnt), 4267 /* MISC_ERR_STATUS */ 4268 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0, 4269 CNTR_NORMAL, 4270 access_misc_pll_lock_fail_err_cnt), 4271 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0, 4272 CNTR_NORMAL, 4273 access_misc_mbist_fail_err_cnt), 4274 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0, 4275 CNTR_NORMAL, 4276 access_misc_invalid_eep_cmd_err_cnt), 4277 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0, 4278 CNTR_NORMAL, 4279 access_misc_efuse_done_parity_err_cnt), 4280 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0, 4281 CNTR_NORMAL, 4282 access_misc_efuse_write_err_cnt), 4283 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0, 4284 0, CNTR_NORMAL, 4285 access_misc_efuse_read_bad_addr_err_cnt), 4286 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0, 4287 CNTR_NORMAL, 4288 access_misc_efuse_csr_parity_err_cnt), 4289 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0, 4290 CNTR_NORMAL, 4291 access_misc_fw_auth_failed_err_cnt), 4292 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0, 4293 CNTR_NORMAL, 4294 access_misc_key_mismatch_err_cnt), 4295 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0, 4296 CNTR_NORMAL, 4297 access_misc_sbus_write_failed_err_cnt), 4298 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0, 4299 CNTR_NORMAL, 4300 access_misc_csr_write_bad_addr_err_cnt), 4301 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0, 4302 CNTR_NORMAL, 4303 access_misc_csr_read_bad_addr_err_cnt), 4304 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0, 4305 CNTR_NORMAL, 4306 access_misc_csr_parity_err_cnt), 4307 /* CceErrStatus */ 4308 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0, 4309 CNTR_NORMAL, 4310 access_sw_cce_err_status_aggregated_cnt), 4311 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0, 4312 CNTR_NORMAL, 4313 access_cce_msix_csr_parity_err_cnt), 4314 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0, 4315 CNTR_NORMAL, 4316 access_cce_int_map_unc_err_cnt), 4317 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0, 4318 CNTR_NORMAL, 4319 access_cce_int_map_cor_err_cnt), 4320 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0, 4321 CNTR_NORMAL, 4322 access_cce_msix_table_unc_err_cnt), 4323 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0, 4324 CNTR_NORMAL, 4325 access_cce_msix_table_cor_err_cnt), 4326 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0, 4327 0, CNTR_NORMAL, 4328 access_cce_rxdma_conv_fifo_parity_err_cnt), 4329 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0, 4330 0, CNTR_NORMAL, 4331 access_cce_rcpl_async_fifo_parity_err_cnt), 4332 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0, 4333 CNTR_NORMAL, 4334 access_cce_seg_write_bad_addr_err_cnt), 4335 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0, 4336 CNTR_NORMAL, 4337 access_cce_seg_read_bad_addr_err_cnt), 4338 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0, 4339 CNTR_NORMAL, 4340 access_la_triggered_cnt), 4341 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0, 4342 CNTR_NORMAL, 4343 access_cce_trgt_cpl_timeout_err_cnt), 4344 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0, 4345 CNTR_NORMAL, 4346 access_pcic_receive_parity_err_cnt), 4347 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0, 4348 CNTR_NORMAL, 4349 access_pcic_transmit_back_parity_err_cnt), 4350 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0, 4351 0, CNTR_NORMAL, 4352 access_pcic_transmit_front_parity_err_cnt), 4353 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0, 4354 CNTR_NORMAL, 4355 access_pcic_cpl_dat_q_unc_err_cnt), 4356 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0, 4357 CNTR_NORMAL, 4358 access_pcic_cpl_hd_q_unc_err_cnt), 4359 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0, 4360 CNTR_NORMAL, 4361 access_pcic_post_dat_q_unc_err_cnt), 4362 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0, 4363 CNTR_NORMAL, 4364 access_pcic_post_hd_q_unc_err_cnt), 4365 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0, 4366 CNTR_NORMAL, 4367 access_pcic_retry_sot_mem_unc_err_cnt), 4368 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0, 4369 CNTR_NORMAL, 4370 access_pcic_retry_mem_unc_err), 4371 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0, 4372 CNTR_NORMAL, 4373 access_pcic_n_post_dat_q_parity_err_cnt), 4374 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0, 4375 CNTR_NORMAL, 4376 access_pcic_n_post_h_q_parity_err_cnt), 4377 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0, 4378 CNTR_NORMAL, 4379 access_pcic_cpl_dat_q_cor_err_cnt), 4380 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0, 4381 CNTR_NORMAL, 4382 access_pcic_cpl_hd_q_cor_err_cnt), 4383 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0, 4384 CNTR_NORMAL, 4385 access_pcic_post_dat_q_cor_err_cnt), 4386 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0, 4387 CNTR_NORMAL, 4388 access_pcic_post_hd_q_cor_err_cnt), 4389 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0, 4390 CNTR_NORMAL, 4391 access_pcic_retry_sot_mem_cor_err_cnt), 4392 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0, 4393 CNTR_NORMAL, 4394 access_pcic_retry_mem_cor_err_cnt), 4395 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM( 4396 "CceCli1AsyncFifoDbgParityError", 0, 0, 4397 CNTR_NORMAL, 4398 access_cce_cli1_async_fifo_dbg_parity_err_cnt), 4399 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM( 4400 "CceCli1AsyncFifoRxdmaParityError", 0, 0, 4401 CNTR_NORMAL, 4402 access_cce_cli1_async_fifo_rxdma_parity_err_cnt 4403 ), 4404 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM( 4405 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0, 4406 CNTR_NORMAL, 4407 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt), 4408 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM( 4409 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0, 4410 CNTR_NORMAL, 4411 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt), 4412 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0, 4413 0, CNTR_NORMAL, 4414 access_cce_cli2_async_fifo_parity_err_cnt), 4415 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0, 4416 CNTR_NORMAL, 4417 access_cce_csr_cfg_bus_parity_err_cnt), 4418 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0, 4419 0, CNTR_NORMAL, 4420 access_cce_cli0_async_fifo_parity_err_cnt), 4421 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0, 4422 CNTR_NORMAL, 4423 access_cce_rspd_data_parity_err_cnt), 4424 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0, 4425 CNTR_NORMAL, 4426 access_cce_trgt_access_err_cnt), 4427 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0, 4428 0, CNTR_NORMAL, 4429 access_cce_trgt_async_fifo_parity_err_cnt), 4430 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0, 4431 CNTR_NORMAL, 4432 access_cce_csr_write_bad_addr_err_cnt), 4433 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0, 4434 CNTR_NORMAL, 4435 access_cce_csr_read_bad_addr_err_cnt), 4436 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0, 4437 CNTR_NORMAL, 4438 access_ccs_csr_parity_err_cnt), 4439 4440 /* RcvErrStatus */ 4441 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0, 4442 CNTR_NORMAL, 4443 access_rx_csr_parity_err_cnt), 4444 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0, 4445 CNTR_NORMAL, 4446 access_rx_csr_write_bad_addr_err_cnt), 4447 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0, 4448 CNTR_NORMAL, 4449 access_rx_csr_read_bad_addr_err_cnt), 4450 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0, 4451 CNTR_NORMAL, 4452 access_rx_dma_csr_unc_err_cnt), 4453 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0, 4454 CNTR_NORMAL, 4455 access_rx_dma_dq_fsm_encoding_err_cnt), 4456 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0, 4457 CNTR_NORMAL, 4458 access_rx_dma_eq_fsm_encoding_err_cnt), 4459 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0, 4460 CNTR_NORMAL, 4461 access_rx_dma_csr_parity_err_cnt), 4462 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0, 4463 CNTR_NORMAL, 4464 access_rx_rbuf_data_cor_err_cnt), 4465 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0, 4466 CNTR_NORMAL, 4467 access_rx_rbuf_data_unc_err_cnt), 4468 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0, 4469 CNTR_NORMAL, 4470 access_rx_dma_data_fifo_rd_cor_err_cnt), 4471 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0, 4472 CNTR_NORMAL, 4473 access_rx_dma_data_fifo_rd_unc_err_cnt), 4474 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0, 4475 CNTR_NORMAL, 4476 access_rx_dma_hdr_fifo_rd_cor_err_cnt), 4477 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0, 4478 CNTR_NORMAL, 4479 access_rx_dma_hdr_fifo_rd_unc_err_cnt), 4480 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0, 4481 CNTR_NORMAL, 4482 access_rx_rbuf_desc_part2_cor_err_cnt), 4483 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0, 4484 CNTR_NORMAL, 4485 access_rx_rbuf_desc_part2_unc_err_cnt), 4486 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0, 4487 CNTR_NORMAL, 4488 access_rx_rbuf_desc_part1_cor_err_cnt), 4489 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0, 4490 CNTR_NORMAL, 4491 access_rx_rbuf_desc_part1_unc_err_cnt), 4492 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0, 4493 CNTR_NORMAL, 4494 access_rx_hq_intr_fsm_err_cnt), 4495 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0, 4496 CNTR_NORMAL, 4497 access_rx_hq_intr_csr_parity_err_cnt), 4498 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0, 4499 CNTR_NORMAL, 4500 access_rx_lookup_csr_parity_err_cnt), 4501 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0, 4502 CNTR_NORMAL, 4503 access_rx_lookup_rcv_array_cor_err_cnt), 4504 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0, 4505 CNTR_NORMAL, 4506 access_rx_lookup_rcv_array_unc_err_cnt), 4507 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0, 4508 0, CNTR_NORMAL, 4509 access_rx_lookup_des_part2_parity_err_cnt), 4510 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0, 4511 0, CNTR_NORMAL, 4512 access_rx_lookup_des_part1_unc_cor_err_cnt), 4513 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0, 4514 CNTR_NORMAL, 4515 access_rx_lookup_des_part1_unc_err_cnt), 4516 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0, 4517 CNTR_NORMAL, 4518 access_rx_rbuf_next_free_buf_cor_err_cnt), 4519 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0, 4520 CNTR_NORMAL, 4521 access_rx_rbuf_next_free_buf_unc_err_cnt), 4522 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM( 4523 "RxRbufFlInitWrAddrParityErr", 0, 0, 4524 CNTR_NORMAL, 4525 access_rbuf_fl_init_wr_addr_parity_err_cnt), 4526 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0, 4527 0, CNTR_NORMAL, 4528 access_rx_rbuf_fl_initdone_parity_err_cnt), 4529 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0, 4530 0, CNTR_NORMAL, 4531 access_rx_rbuf_fl_write_addr_parity_err_cnt), 4532 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0, 4533 CNTR_NORMAL, 4534 access_rx_rbuf_fl_rd_addr_parity_err_cnt), 4535 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0, 4536 CNTR_NORMAL, 4537 access_rx_rbuf_empty_err_cnt), 4538 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0, 4539 CNTR_NORMAL, 4540 access_rx_rbuf_full_err_cnt), 4541 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0, 4542 CNTR_NORMAL, 4543 access_rbuf_bad_lookup_err_cnt), 4544 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0, 4545 CNTR_NORMAL, 4546 access_rbuf_ctx_id_parity_err_cnt), 4547 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0, 4548 CNTR_NORMAL, 4549 access_rbuf_csr_qeopdw_parity_err_cnt), 4550 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM( 4551 "RxRbufCsrQNumOfPktParityErr", 0, 0, 4552 CNTR_NORMAL, 4553 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt), 4554 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM( 4555 "RxRbufCsrQTlPtrParityErr", 0, 0, 4556 CNTR_NORMAL, 4557 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt), 4558 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0, 4559 0, CNTR_NORMAL, 4560 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt), 4561 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0, 4562 0, CNTR_NORMAL, 4563 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt), 4564 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr", 4565 0, 0, CNTR_NORMAL, 4566 access_rx_rbuf_csr_q_next_buf_parity_err_cnt), 4567 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0, 4568 0, CNTR_NORMAL, 4569 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt), 4570 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM( 4571 "RxRbufCsrQHeadBufNumParityErr", 0, 0, 4572 CNTR_NORMAL, 4573 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt), 4574 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0, 4575 0, CNTR_NORMAL, 4576 access_rx_rbuf_block_list_read_cor_err_cnt), 4577 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0, 4578 0, CNTR_NORMAL, 4579 access_rx_rbuf_block_list_read_unc_err_cnt), 4580 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0, 4581 CNTR_NORMAL, 4582 access_rx_rbuf_lookup_des_cor_err_cnt), 4583 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0, 4584 CNTR_NORMAL, 4585 access_rx_rbuf_lookup_des_unc_err_cnt), 4586 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM( 4587 "RxRbufLookupDesRegUncCorErr", 0, 0, 4588 CNTR_NORMAL, 4589 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt), 4590 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0, 4591 CNTR_NORMAL, 4592 access_rx_rbuf_lookup_des_reg_unc_err_cnt), 4593 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0, 4594 CNTR_NORMAL, 4595 access_rx_rbuf_free_list_cor_err_cnt), 4596 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0, 4597 CNTR_NORMAL, 4598 access_rx_rbuf_free_list_unc_err_cnt), 4599 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0, 4600 CNTR_NORMAL, 4601 access_rx_rcv_fsm_encoding_err_cnt), 4602 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0, 4603 CNTR_NORMAL, 4604 access_rx_dma_flag_cor_err_cnt), 4605 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0, 4606 CNTR_NORMAL, 4607 access_rx_dma_flag_unc_err_cnt), 4608 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0, 4609 CNTR_NORMAL, 4610 access_rx_dc_sop_eop_parity_err_cnt), 4611 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0, 4612 CNTR_NORMAL, 4613 access_rx_rcv_csr_parity_err_cnt), 4614 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0, 4615 CNTR_NORMAL, 4616 access_rx_rcv_qp_map_table_cor_err_cnt), 4617 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0, 4618 CNTR_NORMAL, 4619 access_rx_rcv_qp_map_table_unc_err_cnt), 4620 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0, 4621 CNTR_NORMAL, 4622 access_rx_rcv_data_cor_err_cnt), 4623 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0, 4624 CNTR_NORMAL, 4625 access_rx_rcv_data_unc_err_cnt), 4626 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0, 4627 CNTR_NORMAL, 4628 access_rx_rcv_hdr_cor_err_cnt), 4629 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0, 4630 CNTR_NORMAL, 4631 access_rx_rcv_hdr_unc_err_cnt), 4632 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0, 4633 CNTR_NORMAL, 4634 access_rx_dc_intf_parity_err_cnt), 4635 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0, 4636 CNTR_NORMAL, 4637 access_rx_dma_csr_cor_err_cnt), 4638 /* SendPioErrStatus */ 4639 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0, 4640 CNTR_NORMAL, 4641 access_pio_pec_sop_head_parity_err_cnt), 4642 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0, 4643 CNTR_NORMAL, 4644 access_pio_pcc_sop_head_parity_err_cnt), 4645 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr", 4646 0, 0, CNTR_NORMAL, 4647 access_pio_last_returned_cnt_parity_err_cnt), 4648 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0, 4649 0, CNTR_NORMAL, 4650 access_pio_current_free_cnt_parity_err_cnt), 4651 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0, 4652 CNTR_NORMAL, 4653 access_pio_reserved_31_err_cnt), 4654 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0, 4655 CNTR_NORMAL, 4656 access_pio_reserved_30_err_cnt), 4657 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0, 4658 CNTR_NORMAL, 4659 access_pio_ppmc_sop_len_err_cnt), 4660 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0, 4661 CNTR_NORMAL, 4662 access_pio_ppmc_bqc_mem_parity_err_cnt), 4663 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0, 4664 CNTR_NORMAL, 4665 access_pio_vl_fifo_parity_err_cnt), 4666 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0, 4667 CNTR_NORMAL, 4668 access_pio_vlf_sop_parity_err_cnt), 4669 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0, 4670 CNTR_NORMAL, 4671 access_pio_vlf_v1_len_parity_err_cnt), 4672 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0, 4673 CNTR_NORMAL, 4674 access_pio_block_qw_count_parity_err_cnt), 4675 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0, 4676 CNTR_NORMAL, 4677 access_pio_write_qw_valid_parity_err_cnt), 4678 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0, 4679 CNTR_NORMAL, 4680 access_pio_state_machine_err_cnt), 4681 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0, 4682 CNTR_NORMAL, 4683 access_pio_write_data_parity_err_cnt), 4684 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0, 4685 CNTR_NORMAL, 4686 access_pio_host_addr_mem_cor_err_cnt), 4687 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0, 4688 CNTR_NORMAL, 4689 access_pio_host_addr_mem_unc_err_cnt), 4690 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0, 4691 CNTR_NORMAL, 4692 access_pio_pkt_evict_sm_or_arb_sm_err_cnt), 4693 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0, 4694 CNTR_NORMAL, 4695 access_pio_init_sm_in_err_cnt), 4696 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0, 4697 CNTR_NORMAL, 4698 access_pio_ppmc_pbl_fifo_err_cnt), 4699 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0, 4700 0, CNTR_NORMAL, 4701 access_pio_credit_ret_fifo_parity_err_cnt), 4702 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0, 4703 CNTR_NORMAL, 4704 access_pio_v1_len_mem_bank1_cor_err_cnt), 4705 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0, 4706 CNTR_NORMAL, 4707 access_pio_v1_len_mem_bank0_cor_err_cnt), 4708 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0, 4709 CNTR_NORMAL, 4710 access_pio_v1_len_mem_bank1_unc_err_cnt), 4711 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0, 4712 CNTR_NORMAL, 4713 access_pio_v1_len_mem_bank0_unc_err_cnt), 4714 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0, 4715 CNTR_NORMAL, 4716 access_pio_sm_pkt_reset_parity_err_cnt), 4717 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0, 4718 CNTR_NORMAL, 4719 access_pio_pkt_evict_fifo_parity_err_cnt), 4720 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM( 4721 "PioSbrdctrlCrrelFifoParityErr", 0, 0, 4722 CNTR_NORMAL, 4723 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt), 4724 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0, 4725 CNTR_NORMAL, 4726 access_pio_sbrdctl_crrel_parity_err_cnt), 4727 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0, 4728 CNTR_NORMAL, 4729 access_pio_pec_fifo_parity_err_cnt), 4730 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0, 4731 CNTR_NORMAL, 4732 access_pio_pcc_fifo_parity_err_cnt), 4733 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0, 4734 CNTR_NORMAL, 4735 access_pio_sb_mem_fifo1_err_cnt), 4736 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0, 4737 CNTR_NORMAL, 4738 access_pio_sb_mem_fifo0_err_cnt), 4739 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0, 4740 CNTR_NORMAL, 4741 access_pio_csr_parity_err_cnt), 4742 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0, 4743 CNTR_NORMAL, 4744 access_pio_write_addr_parity_err_cnt), 4745 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0, 4746 CNTR_NORMAL, 4747 access_pio_write_bad_ctxt_err_cnt), 4748 /* SendDmaErrStatus */ 4749 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0, 4750 0, CNTR_NORMAL, 4751 access_sdma_pcie_req_tracking_cor_err_cnt), 4752 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0, 4753 0, CNTR_NORMAL, 4754 access_sdma_pcie_req_tracking_unc_err_cnt), 4755 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0, 4756 CNTR_NORMAL, 4757 access_sdma_csr_parity_err_cnt), 4758 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0, 4759 CNTR_NORMAL, 4760 access_sdma_rpy_tag_err_cnt), 4761 /* SendEgressErrStatus */ 4762 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0, 4763 CNTR_NORMAL, 4764 access_tx_read_pio_memory_csr_unc_err_cnt), 4765 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0, 4766 0, CNTR_NORMAL, 4767 access_tx_read_sdma_memory_csr_err_cnt), 4768 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0, 4769 CNTR_NORMAL, 4770 access_tx_egress_fifo_cor_err_cnt), 4771 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0, 4772 CNTR_NORMAL, 4773 access_tx_read_pio_memory_cor_err_cnt), 4774 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0, 4775 CNTR_NORMAL, 4776 access_tx_read_sdma_memory_cor_err_cnt), 4777 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0, 4778 CNTR_NORMAL, 4779 access_tx_sb_hdr_cor_err_cnt), 4780 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0, 4781 CNTR_NORMAL, 4782 access_tx_credit_overrun_err_cnt), 4783 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0, 4784 CNTR_NORMAL, 4785 access_tx_launch_fifo8_cor_err_cnt), 4786 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0, 4787 CNTR_NORMAL, 4788 access_tx_launch_fifo7_cor_err_cnt), 4789 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0, 4790 CNTR_NORMAL, 4791 access_tx_launch_fifo6_cor_err_cnt), 4792 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0, 4793 CNTR_NORMAL, 4794 access_tx_launch_fifo5_cor_err_cnt), 4795 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0, 4796 CNTR_NORMAL, 4797 access_tx_launch_fifo4_cor_err_cnt), 4798 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0, 4799 CNTR_NORMAL, 4800 access_tx_launch_fifo3_cor_err_cnt), 4801 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0, 4802 CNTR_NORMAL, 4803 access_tx_launch_fifo2_cor_err_cnt), 4804 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0, 4805 CNTR_NORMAL, 4806 access_tx_launch_fifo1_cor_err_cnt), 4807 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0, 4808 CNTR_NORMAL, 4809 access_tx_launch_fifo0_cor_err_cnt), 4810 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0, 4811 CNTR_NORMAL, 4812 access_tx_credit_return_vl_err_cnt), 4813 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0, 4814 CNTR_NORMAL, 4815 access_tx_hcrc_insertion_err_cnt), 4816 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0, 4817 CNTR_NORMAL, 4818 access_tx_egress_fifo_unc_err_cnt), 4819 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0, 4820 CNTR_NORMAL, 4821 access_tx_read_pio_memory_unc_err_cnt), 4822 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0, 4823 CNTR_NORMAL, 4824 access_tx_read_sdma_memory_unc_err_cnt), 4825 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0, 4826 CNTR_NORMAL, 4827 access_tx_sb_hdr_unc_err_cnt), 4828 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0, 4829 CNTR_NORMAL, 4830 access_tx_credit_return_partiy_err_cnt), 4831 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr", 4832 0, 0, CNTR_NORMAL, 4833 access_tx_launch_fifo8_unc_or_parity_err_cnt), 4834 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr", 4835 0, 0, CNTR_NORMAL, 4836 access_tx_launch_fifo7_unc_or_parity_err_cnt), 4837 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr", 4838 0, 0, CNTR_NORMAL, 4839 access_tx_launch_fifo6_unc_or_parity_err_cnt), 4840 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr", 4841 0, 0, CNTR_NORMAL, 4842 access_tx_launch_fifo5_unc_or_parity_err_cnt), 4843 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr", 4844 0, 0, CNTR_NORMAL, 4845 access_tx_launch_fifo4_unc_or_parity_err_cnt), 4846 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr", 4847 0, 0, CNTR_NORMAL, 4848 access_tx_launch_fifo3_unc_or_parity_err_cnt), 4849 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr", 4850 0, 0, CNTR_NORMAL, 4851 access_tx_launch_fifo2_unc_or_parity_err_cnt), 4852 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr", 4853 0, 0, CNTR_NORMAL, 4854 access_tx_launch_fifo1_unc_or_parity_err_cnt), 4855 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr", 4856 0, 0, CNTR_NORMAL, 4857 access_tx_launch_fifo0_unc_or_parity_err_cnt), 4858 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr", 4859 0, 0, CNTR_NORMAL, 4860 access_tx_sdma15_disallowed_packet_err_cnt), 4861 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr", 4862 0, 0, CNTR_NORMAL, 4863 access_tx_sdma14_disallowed_packet_err_cnt), 4864 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr", 4865 0, 0, CNTR_NORMAL, 4866 access_tx_sdma13_disallowed_packet_err_cnt), 4867 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr", 4868 0, 0, CNTR_NORMAL, 4869 access_tx_sdma12_disallowed_packet_err_cnt), 4870 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr", 4871 0, 0, CNTR_NORMAL, 4872 access_tx_sdma11_disallowed_packet_err_cnt), 4873 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr", 4874 0, 0, CNTR_NORMAL, 4875 access_tx_sdma10_disallowed_packet_err_cnt), 4876 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr", 4877 0, 0, CNTR_NORMAL, 4878 access_tx_sdma9_disallowed_packet_err_cnt), 4879 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr", 4880 0, 0, CNTR_NORMAL, 4881 access_tx_sdma8_disallowed_packet_err_cnt), 4882 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr", 4883 0, 0, CNTR_NORMAL, 4884 access_tx_sdma7_disallowed_packet_err_cnt), 4885 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr", 4886 0, 0, CNTR_NORMAL, 4887 access_tx_sdma6_disallowed_packet_err_cnt), 4888 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr", 4889 0, 0, CNTR_NORMAL, 4890 access_tx_sdma5_disallowed_packet_err_cnt), 4891 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr", 4892 0, 0, CNTR_NORMAL, 4893 access_tx_sdma4_disallowed_packet_err_cnt), 4894 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr", 4895 0, 0, CNTR_NORMAL, 4896 access_tx_sdma3_disallowed_packet_err_cnt), 4897 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr", 4898 0, 0, CNTR_NORMAL, 4899 access_tx_sdma2_disallowed_packet_err_cnt), 4900 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr", 4901 0, 0, CNTR_NORMAL, 4902 access_tx_sdma1_disallowed_packet_err_cnt), 4903 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr", 4904 0, 0, CNTR_NORMAL, 4905 access_tx_sdma0_disallowed_packet_err_cnt), 4906 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0, 4907 CNTR_NORMAL, 4908 access_tx_config_parity_err_cnt), 4909 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0, 4910 CNTR_NORMAL, 4911 access_tx_sbrd_ctl_csr_parity_err_cnt), 4912 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0, 4913 CNTR_NORMAL, 4914 access_tx_launch_csr_parity_err_cnt), 4915 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0, 4916 CNTR_NORMAL, 4917 access_tx_illegal_vl_err_cnt), 4918 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM( 4919 "TxSbrdCtlStateMachineParityErr", 0, 0, 4920 CNTR_NORMAL, 4921 access_tx_sbrd_ctl_state_machine_parity_err_cnt), 4922 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0, 4923 CNTR_NORMAL, 4924 access_egress_reserved_10_err_cnt), 4925 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0, 4926 CNTR_NORMAL, 4927 access_egress_reserved_9_err_cnt), 4928 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr", 4929 0, 0, CNTR_NORMAL, 4930 access_tx_sdma_launch_intf_parity_err_cnt), 4931 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0, 4932 CNTR_NORMAL, 4933 access_tx_pio_launch_intf_parity_err_cnt), 4934 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0, 4935 CNTR_NORMAL, 4936 access_egress_reserved_6_err_cnt), 4937 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0, 4938 CNTR_NORMAL, 4939 access_tx_incorrect_link_state_err_cnt), 4940 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0, 4941 CNTR_NORMAL, 4942 access_tx_linkdown_err_cnt), 4943 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM( 4944 "EgressFifoUnderrunOrParityErr", 0, 0, 4945 CNTR_NORMAL, 4946 access_tx_egress_fifi_underrun_or_parity_err_cnt), 4947 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0, 4948 CNTR_NORMAL, 4949 access_egress_reserved_2_err_cnt), 4950 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0, 4951 CNTR_NORMAL, 4952 access_tx_pkt_integrity_mem_unc_err_cnt), 4953 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0, 4954 CNTR_NORMAL, 4955 access_tx_pkt_integrity_mem_cor_err_cnt), 4956 /* SendErrStatus */ 4957 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0, 4958 CNTR_NORMAL, 4959 access_send_csr_write_bad_addr_err_cnt), 4960 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0, 4961 CNTR_NORMAL, 4962 access_send_csr_read_bad_addr_err_cnt), 4963 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0, 4964 CNTR_NORMAL, 4965 access_send_csr_parity_cnt), 4966 /* SendCtxtErrStatus */ 4967 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0, 4968 CNTR_NORMAL, 4969 access_pio_write_out_of_bounds_err_cnt), 4970 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0, 4971 CNTR_NORMAL, 4972 access_pio_write_overflow_err_cnt), 4973 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr", 4974 0, 0, CNTR_NORMAL, 4975 access_pio_write_crosses_boundary_err_cnt), 4976 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0, 4977 CNTR_NORMAL, 4978 access_pio_disallowed_packet_err_cnt), 4979 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0, 4980 CNTR_NORMAL, 4981 access_pio_inconsistent_sop_err_cnt), 4982 /* SendDmaEngErrStatus */ 4983 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr", 4984 0, 0, CNTR_NORMAL, 4985 access_sdma_header_request_fifo_cor_err_cnt), 4986 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0, 4987 CNTR_NORMAL, 4988 access_sdma_header_storage_cor_err_cnt), 4989 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0, 4990 CNTR_NORMAL, 4991 access_sdma_packet_tracking_cor_err_cnt), 4992 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0, 4993 CNTR_NORMAL, 4994 access_sdma_assembly_cor_err_cnt), 4995 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0, 4996 CNTR_NORMAL, 4997 access_sdma_desc_table_cor_err_cnt), 4998 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr", 4999 0, 0, CNTR_NORMAL, 5000 access_sdma_header_request_fifo_unc_err_cnt), 5001 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0, 5002 CNTR_NORMAL, 5003 access_sdma_header_storage_unc_err_cnt), 5004 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0, 5005 CNTR_NORMAL, 5006 access_sdma_packet_tracking_unc_err_cnt), 5007 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0, 5008 CNTR_NORMAL, 5009 access_sdma_assembly_unc_err_cnt), 5010 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0, 5011 CNTR_NORMAL, 5012 access_sdma_desc_table_unc_err_cnt), 5013 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0, 5014 CNTR_NORMAL, 5015 access_sdma_timeout_err_cnt), 5016 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0, 5017 CNTR_NORMAL, 5018 access_sdma_header_length_err_cnt), 5019 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0, 5020 CNTR_NORMAL, 5021 access_sdma_header_address_err_cnt), 5022 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0, 5023 CNTR_NORMAL, 5024 access_sdma_header_select_err_cnt), 5025 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0, 5026 CNTR_NORMAL, 5027 access_sdma_reserved_9_err_cnt), 5028 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0, 5029 CNTR_NORMAL, 5030 access_sdma_packet_desc_overflow_err_cnt), 5031 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0, 5032 CNTR_NORMAL, 5033 access_sdma_length_mismatch_err_cnt), 5034 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0, 5035 CNTR_NORMAL, 5036 access_sdma_halt_err_cnt), 5037 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0, 5038 CNTR_NORMAL, 5039 access_sdma_mem_read_err_cnt), 5040 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0, 5041 CNTR_NORMAL, 5042 access_sdma_first_desc_err_cnt), 5043 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0, 5044 CNTR_NORMAL, 5045 access_sdma_tail_out_of_bounds_err_cnt), 5046 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0, 5047 CNTR_NORMAL, 5048 access_sdma_too_long_err_cnt), 5049 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0, 5050 CNTR_NORMAL, 5051 access_sdma_gen_mismatch_err_cnt), 5052 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0, 5053 CNTR_NORMAL, 5054 access_sdma_wrong_dw_err_cnt), 5055 }; 5056 5057 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = { 5058 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT, 5059 CNTR_NORMAL), 5060 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT, 5061 CNTR_NORMAL), 5062 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT, 5063 CNTR_NORMAL), 5064 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT, 5065 CNTR_NORMAL), 5066 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT, 5067 CNTR_NORMAL), 5068 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT, 5069 CNTR_NORMAL), 5070 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT, 5071 CNTR_NORMAL), 5072 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL), 5073 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL), 5074 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH), 5075 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT, 5076 CNTR_SYNTH | CNTR_VL), 5077 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT, 5078 CNTR_SYNTH | CNTR_VL), 5079 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT, 5080 CNTR_SYNTH | CNTR_VL), 5081 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL), 5082 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL), 5083 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5084 access_sw_link_dn_cnt), 5085 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5086 access_sw_link_up_cnt), 5087 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL, 5088 access_sw_unknown_frame_cnt), 5089 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5090 access_sw_xmit_discards), 5091 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0, 5092 CNTR_SYNTH | CNTR_32BIT | CNTR_VL, 5093 access_sw_xmit_discards), 5094 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH, 5095 access_xmit_constraint_errs), 5096 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH, 5097 access_rcv_constraint_errs), 5098 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts), 5099 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends), 5100 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks), 5101 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks), 5102 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts), 5103 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops), 5104 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait), 5105 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak), 5106 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq), 5107 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq), 5108 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned), 5109 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks), 5110 [C_SW_IBP_RC_CRWAITS] = SW_IBP_CNTR(RcCrWait, rc_crwaits), 5111 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL, 5112 access_sw_cpu_rc_acks), 5113 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL, 5114 access_sw_cpu_rc_qacks), 5115 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL, 5116 access_sw_cpu_rc_delayed_comp), 5117 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1), 5118 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3), 5119 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5), 5120 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7), 5121 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9), 5122 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11), 5123 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13), 5124 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15), 5125 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17), 5126 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19), 5127 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21), 5128 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23), 5129 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25), 5130 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27), 5131 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29), 5132 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31), 5133 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33), 5134 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35), 5135 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37), 5136 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39), 5137 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41), 5138 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43), 5139 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45), 5140 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47), 5141 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49), 5142 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51), 5143 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53), 5144 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55), 5145 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57), 5146 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59), 5147 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61), 5148 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63), 5149 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65), 5150 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67), 5151 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69), 5152 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71), 5153 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73), 5154 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75), 5155 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77), 5156 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79), 5157 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81), 5158 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83), 5159 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85), 5160 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87), 5161 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89), 5162 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91), 5163 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93), 5164 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95), 5165 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97), 5166 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99), 5167 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101), 5168 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103), 5169 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105), 5170 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107), 5171 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109), 5172 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111), 5173 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113), 5174 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115), 5175 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117), 5176 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119), 5177 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121), 5178 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123), 5179 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125), 5180 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127), 5181 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129), 5182 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131), 5183 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133), 5184 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135), 5185 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137), 5186 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139), 5187 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141), 5188 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143), 5189 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145), 5190 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147), 5191 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149), 5192 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151), 5193 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153), 5194 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155), 5195 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157), 5196 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159), 5197 }; 5198 5199 /* ======================================================================== */ 5200 5201 /* return true if this is chip revision revision a */ 5202 int is_ax(struct hfi1_devdata *dd) 5203 { 5204 u8 chip_rev_minor = 5205 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5206 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5207 return (chip_rev_minor & 0xf0) == 0; 5208 } 5209 5210 /* return true if this is chip revision revision b */ 5211 int is_bx(struct hfi1_devdata *dd) 5212 { 5213 u8 chip_rev_minor = 5214 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5215 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5216 return (chip_rev_minor & 0xF0) == 0x10; 5217 } 5218 5219 /* return true is kernel urg disabled for rcd */ 5220 bool is_urg_masked(struct hfi1_ctxtdata *rcd) 5221 { 5222 u64 mask; 5223 u32 is = IS_RCVURGENT_START + rcd->ctxt; 5224 u8 bit = is % 64; 5225 5226 mask = read_csr(rcd->dd, CCE_INT_MASK + (8 * (is / 64))); 5227 return !(mask & BIT_ULL(bit)); 5228 } 5229 5230 /* 5231 * Append string s to buffer buf. Arguments curp and len are the current 5232 * position and remaining length, respectively. 5233 * 5234 * return 0 on success, 1 on out of room 5235 */ 5236 static int append_str(char *buf, char **curp, int *lenp, const char *s) 5237 { 5238 char *p = *curp; 5239 int len = *lenp; 5240 int result = 0; /* success */ 5241 char c; 5242 5243 /* add a comma, if first in the buffer */ 5244 if (p != buf) { 5245 if (len == 0) { 5246 result = 1; /* out of room */ 5247 goto done; 5248 } 5249 *p++ = ','; 5250 len--; 5251 } 5252 5253 /* copy the string */ 5254 while ((c = *s++) != 0) { 5255 if (len == 0) { 5256 result = 1; /* out of room */ 5257 goto done; 5258 } 5259 *p++ = c; 5260 len--; 5261 } 5262 5263 done: 5264 /* write return values */ 5265 *curp = p; 5266 *lenp = len; 5267 5268 return result; 5269 } 5270 5271 /* 5272 * Using the given flag table, print a comma separated string into 5273 * the buffer. End in '*' if the buffer is too short. 5274 */ 5275 static char *flag_string(char *buf, int buf_len, u64 flags, 5276 struct flag_table *table, int table_size) 5277 { 5278 char extra[32]; 5279 char *p = buf; 5280 int len = buf_len; 5281 int no_room = 0; 5282 int i; 5283 5284 /* make sure there is at least 2 so we can form "*" */ 5285 if (len < 2) 5286 return ""; 5287 5288 len--; /* leave room for a nul */ 5289 for (i = 0; i < table_size; i++) { 5290 if (flags & table[i].flag) { 5291 no_room = append_str(buf, &p, &len, table[i].str); 5292 if (no_room) 5293 break; 5294 flags &= ~table[i].flag; 5295 } 5296 } 5297 5298 /* any undocumented bits left? */ 5299 if (!no_room && flags) { 5300 snprintf(extra, sizeof(extra), "bits 0x%llx", flags); 5301 no_room = append_str(buf, &p, &len, extra); 5302 } 5303 5304 /* add * if ran out of room */ 5305 if (no_room) { 5306 /* may need to back up to add space for a '*' */ 5307 if (len == 0) 5308 --p; 5309 *p++ = '*'; 5310 } 5311 5312 /* add final nul - space already allocated above */ 5313 *p = 0; 5314 return buf; 5315 } 5316 5317 /* first 8 CCE error interrupt source names */ 5318 static const char * const cce_misc_names[] = { 5319 "CceErrInt", /* 0 */ 5320 "RxeErrInt", /* 1 */ 5321 "MiscErrInt", /* 2 */ 5322 "Reserved3", /* 3 */ 5323 "PioErrInt", /* 4 */ 5324 "SDmaErrInt", /* 5 */ 5325 "EgressErrInt", /* 6 */ 5326 "TxeErrInt" /* 7 */ 5327 }; 5328 5329 /* 5330 * Return the miscellaneous error interrupt name. 5331 */ 5332 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source) 5333 { 5334 if (source < ARRAY_SIZE(cce_misc_names)) 5335 strncpy(buf, cce_misc_names[source], bsize); 5336 else 5337 snprintf(buf, bsize, "Reserved%u", 5338 source + IS_GENERAL_ERR_START); 5339 5340 return buf; 5341 } 5342 5343 /* 5344 * Return the SDMA engine error interrupt name. 5345 */ 5346 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source) 5347 { 5348 snprintf(buf, bsize, "SDmaEngErrInt%u", source); 5349 return buf; 5350 } 5351 5352 /* 5353 * Return the send context error interrupt name. 5354 */ 5355 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source) 5356 { 5357 snprintf(buf, bsize, "SendCtxtErrInt%u", source); 5358 return buf; 5359 } 5360 5361 static const char * const various_names[] = { 5362 "PbcInt", 5363 "GpioAssertInt", 5364 "Qsfp1Int", 5365 "Qsfp2Int", 5366 "TCritInt" 5367 }; 5368 5369 /* 5370 * Return the various interrupt name. 5371 */ 5372 static char *is_various_name(char *buf, size_t bsize, unsigned int source) 5373 { 5374 if (source < ARRAY_SIZE(various_names)) 5375 strncpy(buf, various_names[source], bsize); 5376 else 5377 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START); 5378 return buf; 5379 } 5380 5381 /* 5382 * Return the DC interrupt name. 5383 */ 5384 static char *is_dc_name(char *buf, size_t bsize, unsigned int source) 5385 { 5386 static const char * const dc_int_names[] = { 5387 "common", 5388 "lcb", 5389 "8051", 5390 "lbm" /* local block merge */ 5391 }; 5392 5393 if (source < ARRAY_SIZE(dc_int_names)) 5394 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]); 5395 else 5396 snprintf(buf, bsize, "DCInt%u", source); 5397 return buf; 5398 } 5399 5400 static const char * const sdma_int_names[] = { 5401 "SDmaInt", 5402 "SdmaIdleInt", 5403 "SdmaProgressInt", 5404 }; 5405 5406 /* 5407 * Return the SDMA engine interrupt name. 5408 */ 5409 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source) 5410 { 5411 /* what interrupt */ 5412 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 5413 /* which engine */ 5414 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 5415 5416 if (likely(what < 3)) 5417 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which); 5418 else 5419 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source); 5420 return buf; 5421 } 5422 5423 /* 5424 * Return the receive available interrupt name. 5425 */ 5426 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source) 5427 { 5428 snprintf(buf, bsize, "RcvAvailInt%u", source); 5429 return buf; 5430 } 5431 5432 /* 5433 * Return the receive urgent interrupt name. 5434 */ 5435 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source) 5436 { 5437 snprintf(buf, bsize, "RcvUrgentInt%u", source); 5438 return buf; 5439 } 5440 5441 /* 5442 * Return the send credit interrupt name. 5443 */ 5444 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source) 5445 { 5446 snprintf(buf, bsize, "SendCreditInt%u", source); 5447 return buf; 5448 } 5449 5450 /* 5451 * Return the reserved interrupt name. 5452 */ 5453 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source) 5454 { 5455 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START); 5456 return buf; 5457 } 5458 5459 static char *cce_err_status_string(char *buf, int buf_len, u64 flags) 5460 { 5461 return flag_string(buf, buf_len, flags, 5462 cce_err_status_flags, 5463 ARRAY_SIZE(cce_err_status_flags)); 5464 } 5465 5466 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags) 5467 { 5468 return flag_string(buf, buf_len, flags, 5469 rxe_err_status_flags, 5470 ARRAY_SIZE(rxe_err_status_flags)); 5471 } 5472 5473 static char *misc_err_status_string(char *buf, int buf_len, u64 flags) 5474 { 5475 return flag_string(buf, buf_len, flags, misc_err_status_flags, 5476 ARRAY_SIZE(misc_err_status_flags)); 5477 } 5478 5479 static char *pio_err_status_string(char *buf, int buf_len, u64 flags) 5480 { 5481 return flag_string(buf, buf_len, flags, 5482 pio_err_status_flags, 5483 ARRAY_SIZE(pio_err_status_flags)); 5484 } 5485 5486 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags) 5487 { 5488 return flag_string(buf, buf_len, flags, 5489 sdma_err_status_flags, 5490 ARRAY_SIZE(sdma_err_status_flags)); 5491 } 5492 5493 static char *egress_err_status_string(char *buf, int buf_len, u64 flags) 5494 { 5495 return flag_string(buf, buf_len, flags, 5496 egress_err_status_flags, 5497 ARRAY_SIZE(egress_err_status_flags)); 5498 } 5499 5500 static char *egress_err_info_string(char *buf, int buf_len, u64 flags) 5501 { 5502 return flag_string(buf, buf_len, flags, 5503 egress_err_info_flags, 5504 ARRAY_SIZE(egress_err_info_flags)); 5505 } 5506 5507 static char *send_err_status_string(char *buf, int buf_len, u64 flags) 5508 { 5509 return flag_string(buf, buf_len, flags, 5510 send_err_status_flags, 5511 ARRAY_SIZE(send_err_status_flags)); 5512 } 5513 5514 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5515 { 5516 char buf[96]; 5517 int i = 0; 5518 5519 /* 5520 * For most these errors, there is nothing that can be done except 5521 * report or record it. 5522 */ 5523 dd_dev_info(dd, "CCE Error: %s\n", 5524 cce_err_status_string(buf, sizeof(buf), reg)); 5525 5526 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) && 5527 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) { 5528 /* this error requires a manual drop into SPC freeze mode */ 5529 /* then a fix up */ 5530 start_freeze_handling(dd->pport, FREEZE_SELF); 5531 } 5532 5533 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) { 5534 if (reg & (1ull << i)) { 5535 incr_cntr64(&dd->cce_err_status_cnt[i]); 5536 /* maintain a counter over all cce_err_status errors */ 5537 incr_cntr64(&dd->sw_cce_err_status_aggregate); 5538 } 5539 } 5540 } 5541 5542 /* 5543 * Check counters for receive errors that do not have an interrupt 5544 * associated with them. 5545 */ 5546 #define RCVERR_CHECK_TIME 10 5547 static void update_rcverr_timer(struct timer_list *t) 5548 { 5549 struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer); 5550 struct hfi1_pportdata *ppd = dd->pport; 5551 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL); 5552 5553 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt && 5554 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) { 5555 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__); 5556 set_link_down_reason( 5557 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0, 5558 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN); 5559 queue_work(ppd->link_wq, &ppd->link_bounce_work); 5560 } 5561 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt; 5562 5563 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5564 } 5565 5566 static int init_rcverr(struct hfi1_devdata *dd) 5567 { 5568 timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0); 5569 /* Assume the hardware counter has been reset */ 5570 dd->rcv_ovfl_cnt = 0; 5571 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5572 } 5573 5574 static void free_rcverr(struct hfi1_devdata *dd) 5575 { 5576 if (dd->rcverr_timer.function) 5577 del_timer_sync(&dd->rcverr_timer); 5578 } 5579 5580 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5581 { 5582 char buf[96]; 5583 int i = 0; 5584 5585 dd_dev_info(dd, "Receive Error: %s\n", 5586 rxe_err_status_string(buf, sizeof(buf), reg)); 5587 5588 if (reg & ALL_RXE_FREEZE_ERR) { 5589 int flags = 0; 5590 5591 /* 5592 * Freeze mode recovery is disabled for the errors 5593 * in RXE_FREEZE_ABORT_MASK 5594 */ 5595 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK)) 5596 flags = FREEZE_ABORT; 5597 5598 start_freeze_handling(dd->pport, flags); 5599 } 5600 5601 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) { 5602 if (reg & (1ull << i)) 5603 incr_cntr64(&dd->rcv_err_status_cnt[i]); 5604 } 5605 } 5606 5607 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5608 { 5609 char buf[96]; 5610 int i = 0; 5611 5612 dd_dev_info(dd, "Misc Error: %s", 5613 misc_err_status_string(buf, sizeof(buf), reg)); 5614 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) { 5615 if (reg & (1ull << i)) 5616 incr_cntr64(&dd->misc_err_status_cnt[i]); 5617 } 5618 } 5619 5620 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5621 { 5622 char buf[96]; 5623 int i = 0; 5624 5625 dd_dev_info(dd, "PIO Error: %s\n", 5626 pio_err_status_string(buf, sizeof(buf), reg)); 5627 5628 if (reg & ALL_PIO_FREEZE_ERR) 5629 start_freeze_handling(dd->pport, 0); 5630 5631 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) { 5632 if (reg & (1ull << i)) 5633 incr_cntr64(&dd->send_pio_err_status_cnt[i]); 5634 } 5635 } 5636 5637 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5638 { 5639 char buf[96]; 5640 int i = 0; 5641 5642 dd_dev_info(dd, "SDMA Error: %s\n", 5643 sdma_err_status_string(buf, sizeof(buf), reg)); 5644 5645 if (reg & ALL_SDMA_FREEZE_ERR) 5646 start_freeze_handling(dd->pport, 0); 5647 5648 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) { 5649 if (reg & (1ull << i)) 5650 incr_cntr64(&dd->send_dma_err_status_cnt[i]); 5651 } 5652 } 5653 5654 static inline void __count_port_discards(struct hfi1_pportdata *ppd) 5655 { 5656 incr_cntr64(&ppd->port_xmit_discards); 5657 } 5658 5659 static void count_port_inactive(struct hfi1_devdata *dd) 5660 { 5661 __count_port_discards(dd->pport); 5662 } 5663 5664 /* 5665 * We have had a "disallowed packet" error during egress. Determine the 5666 * integrity check which failed, and update relevant error counter, etc. 5667 * 5668 * Note that the SEND_EGRESS_ERR_INFO register has only a single 5669 * bit of state per integrity check, and so we can miss the reason for an 5670 * egress error if more than one packet fails the same integrity check 5671 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO. 5672 */ 5673 static void handle_send_egress_err_info(struct hfi1_devdata *dd, 5674 int vl) 5675 { 5676 struct hfi1_pportdata *ppd = dd->pport; 5677 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */ 5678 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO); 5679 char buf[96]; 5680 5681 /* clear down all observed info as quickly as possible after read */ 5682 write_csr(dd, SEND_EGRESS_ERR_INFO, info); 5683 5684 dd_dev_info(dd, 5685 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n", 5686 info, egress_err_info_string(buf, sizeof(buf), info), src); 5687 5688 /* Eventually add other counters for each bit */ 5689 if (info & PORT_DISCARD_EGRESS_ERRS) { 5690 int weight, i; 5691 5692 /* 5693 * Count all applicable bits as individual errors and 5694 * attribute them to the packet that triggered this handler. 5695 * This may not be completely accurate due to limitations 5696 * on the available hardware error information. There is 5697 * a single information register and any number of error 5698 * packets may have occurred and contributed to it before 5699 * this routine is called. This means that: 5700 * a) If multiple packets with the same error occur before 5701 * this routine is called, earlier packets are missed. 5702 * There is only a single bit for each error type. 5703 * b) Errors may not be attributed to the correct VL. 5704 * The driver is attributing all bits in the info register 5705 * to the packet that triggered this call, but bits 5706 * could be an accumulation of different packets with 5707 * different VLs. 5708 * c) A single error packet may have multiple counts attached 5709 * to it. There is no way for the driver to know if 5710 * multiple bits set in the info register are due to a 5711 * single packet or multiple packets. The driver assumes 5712 * multiple packets. 5713 */ 5714 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS); 5715 for (i = 0; i < weight; i++) { 5716 __count_port_discards(ppd); 5717 if (vl >= 0 && vl < TXE_NUM_DATA_VL) 5718 incr_cntr64(&ppd->port_xmit_discards_vl[vl]); 5719 else if (vl == 15) 5720 incr_cntr64(&ppd->port_xmit_discards_vl 5721 [C_VL_15]); 5722 } 5723 } 5724 } 5725 5726 /* 5727 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5728 * register. Does it represent a 'port inactive' error? 5729 */ 5730 static inline int port_inactive_err(u64 posn) 5731 { 5732 return (posn >= SEES(TX_LINKDOWN) && 5733 posn <= SEES(TX_INCORRECT_LINK_STATE)); 5734 } 5735 5736 /* 5737 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5738 * register. Does it represent a 'disallowed packet' error? 5739 */ 5740 static inline int disallowed_pkt_err(int posn) 5741 { 5742 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) && 5743 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET)); 5744 } 5745 5746 /* 5747 * Input value is a bit position of one of the SDMA engine disallowed 5748 * packet errors. Return which engine. Use of this must be guarded by 5749 * disallowed_pkt_err(). 5750 */ 5751 static inline int disallowed_pkt_engine(int posn) 5752 { 5753 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET); 5754 } 5755 5756 /* 5757 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot 5758 * be done. 5759 */ 5760 static int engine_to_vl(struct hfi1_devdata *dd, int engine) 5761 { 5762 struct sdma_vl_map *m; 5763 int vl; 5764 5765 /* range check */ 5766 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES) 5767 return -1; 5768 5769 rcu_read_lock(); 5770 m = rcu_dereference(dd->sdma_map); 5771 vl = m->engine_to_vl[engine]; 5772 rcu_read_unlock(); 5773 5774 return vl; 5775 } 5776 5777 /* 5778 * Translate the send context (sofware index) into a VL. Return -1 if the 5779 * translation cannot be done. 5780 */ 5781 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index) 5782 { 5783 struct send_context_info *sci; 5784 struct send_context *sc; 5785 int i; 5786 5787 sci = &dd->send_contexts[sw_index]; 5788 5789 /* there is no information for user (PSM) and ack contexts */ 5790 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15)) 5791 return -1; 5792 5793 sc = sci->sc; 5794 if (!sc) 5795 return -1; 5796 if (dd->vld[15].sc == sc) 5797 return 15; 5798 for (i = 0; i < num_vls; i++) 5799 if (dd->vld[i].sc == sc) 5800 return i; 5801 5802 return -1; 5803 } 5804 5805 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5806 { 5807 u64 reg_copy = reg, handled = 0; 5808 char buf[96]; 5809 int i = 0; 5810 5811 if (reg & ALL_TXE_EGRESS_FREEZE_ERR) 5812 start_freeze_handling(dd->pport, 0); 5813 else if (is_ax(dd) && 5814 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) && 5815 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) 5816 start_freeze_handling(dd->pport, 0); 5817 5818 while (reg_copy) { 5819 int posn = fls64(reg_copy); 5820 /* fls64() returns a 1-based offset, we want it zero based */ 5821 int shift = posn - 1; 5822 u64 mask = 1ULL << shift; 5823 5824 if (port_inactive_err(shift)) { 5825 count_port_inactive(dd); 5826 handled |= mask; 5827 } else if (disallowed_pkt_err(shift)) { 5828 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift)); 5829 5830 handle_send_egress_err_info(dd, vl); 5831 handled |= mask; 5832 } 5833 reg_copy &= ~mask; 5834 } 5835 5836 reg &= ~handled; 5837 5838 if (reg) 5839 dd_dev_info(dd, "Egress Error: %s\n", 5840 egress_err_status_string(buf, sizeof(buf), reg)); 5841 5842 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) { 5843 if (reg & (1ull << i)) 5844 incr_cntr64(&dd->send_egress_err_status_cnt[i]); 5845 } 5846 } 5847 5848 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5849 { 5850 char buf[96]; 5851 int i = 0; 5852 5853 dd_dev_info(dd, "Send Error: %s\n", 5854 send_err_status_string(buf, sizeof(buf), reg)); 5855 5856 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) { 5857 if (reg & (1ull << i)) 5858 incr_cntr64(&dd->send_err_status_cnt[i]); 5859 } 5860 } 5861 5862 /* 5863 * The maximum number of times the error clear down will loop before 5864 * blocking a repeating error. This value is arbitrary. 5865 */ 5866 #define MAX_CLEAR_COUNT 20 5867 5868 /* 5869 * Clear and handle an error register. All error interrupts are funneled 5870 * through here to have a central location to correctly handle single- 5871 * or multi-shot errors. 5872 * 5873 * For non per-context registers, call this routine with a context value 5874 * of 0 so the per-context offset is zero. 5875 * 5876 * If the handler loops too many times, assume that something is wrong 5877 * and can't be fixed, so mask the error bits. 5878 */ 5879 static void interrupt_clear_down(struct hfi1_devdata *dd, 5880 u32 context, 5881 const struct err_reg_info *eri) 5882 { 5883 u64 reg; 5884 u32 count; 5885 5886 /* read in a loop until no more errors are seen */ 5887 count = 0; 5888 while (1) { 5889 reg = read_kctxt_csr(dd, context, eri->status); 5890 if (reg == 0) 5891 break; 5892 write_kctxt_csr(dd, context, eri->clear, reg); 5893 if (likely(eri->handler)) 5894 eri->handler(dd, context, reg); 5895 count++; 5896 if (count > MAX_CLEAR_COUNT) { 5897 u64 mask; 5898 5899 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n", 5900 eri->desc, reg); 5901 /* 5902 * Read-modify-write so any other masked bits 5903 * remain masked. 5904 */ 5905 mask = read_kctxt_csr(dd, context, eri->mask); 5906 mask &= ~reg; 5907 write_kctxt_csr(dd, context, eri->mask, mask); 5908 break; 5909 } 5910 } 5911 } 5912 5913 /* 5914 * CCE block "misc" interrupt. Source is < 16. 5915 */ 5916 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source) 5917 { 5918 const struct err_reg_info *eri = &misc_errs[source]; 5919 5920 if (eri->handler) { 5921 interrupt_clear_down(dd, 0, eri); 5922 } else { 5923 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n", 5924 source); 5925 } 5926 } 5927 5928 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags) 5929 { 5930 return flag_string(buf, buf_len, flags, 5931 sc_err_status_flags, 5932 ARRAY_SIZE(sc_err_status_flags)); 5933 } 5934 5935 /* 5936 * Send context error interrupt. Source (hw_context) is < 160. 5937 * 5938 * All send context errors cause the send context to halt. The normal 5939 * clear-down mechanism cannot be used because we cannot clear the 5940 * error bits until several other long-running items are done first. 5941 * This is OK because with the context halted, nothing else is going 5942 * to happen on it anyway. 5943 */ 5944 static void is_sendctxt_err_int(struct hfi1_devdata *dd, 5945 unsigned int hw_context) 5946 { 5947 struct send_context_info *sci; 5948 struct send_context *sc; 5949 char flags[96]; 5950 u64 status; 5951 u32 sw_index; 5952 int i = 0; 5953 unsigned long irq_flags; 5954 5955 sw_index = dd->hw_to_sw[hw_context]; 5956 if (sw_index >= dd->num_send_contexts) { 5957 dd_dev_err(dd, 5958 "out of range sw index %u for send context %u\n", 5959 sw_index, hw_context); 5960 return; 5961 } 5962 sci = &dd->send_contexts[sw_index]; 5963 spin_lock_irqsave(&dd->sc_lock, irq_flags); 5964 sc = sci->sc; 5965 if (!sc) { 5966 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__, 5967 sw_index, hw_context); 5968 spin_unlock_irqrestore(&dd->sc_lock, irq_flags); 5969 return; 5970 } 5971 5972 /* tell the software that a halt has begun */ 5973 sc_stop(sc, SCF_HALTED); 5974 5975 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS); 5976 5977 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context, 5978 send_context_err_status_string(flags, sizeof(flags), 5979 status)); 5980 5981 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK) 5982 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index)); 5983 5984 /* 5985 * Automatically restart halted kernel contexts out of interrupt 5986 * context. User contexts must ask the driver to restart the context. 5987 */ 5988 if (sc->type != SC_USER) 5989 queue_work(dd->pport->hfi1_wq, &sc->halt_work); 5990 spin_unlock_irqrestore(&dd->sc_lock, irq_flags); 5991 5992 /* 5993 * Update the counters for the corresponding status bits. 5994 * Note that these particular counters are aggregated over all 5995 * 160 contexts. 5996 */ 5997 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) { 5998 if (status & (1ull << i)) 5999 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]); 6000 } 6001 } 6002 6003 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 6004 unsigned int source, u64 status) 6005 { 6006 struct sdma_engine *sde; 6007 int i = 0; 6008 6009 sde = &dd->per_sdma[source]; 6010 #ifdef CONFIG_SDMA_VERBOSITY 6011 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 6012 slashstrip(__FILE__), __LINE__, __func__); 6013 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n", 6014 sde->this_idx, source, (unsigned long long)status); 6015 #endif 6016 sde->err_cnt++; 6017 sdma_engine_error(sde, status); 6018 6019 /* 6020 * Update the counters for the corresponding status bits. 6021 * Note that these particular counters are aggregated over 6022 * all 16 DMA engines. 6023 */ 6024 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) { 6025 if (status & (1ull << i)) 6026 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]); 6027 } 6028 } 6029 6030 /* 6031 * CCE block SDMA error interrupt. Source is < 16. 6032 */ 6033 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source) 6034 { 6035 #ifdef CONFIG_SDMA_VERBOSITY 6036 struct sdma_engine *sde = &dd->per_sdma[source]; 6037 6038 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 6039 slashstrip(__FILE__), __LINE__, __func__); 6040 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx, 6041 source); 6042 sdma_dumpstate(sde); 6043 #endif 6044 interrupt_clear_down(dd, source, &sdma_eng_err); 6045 } 6046 6047 /* 6048 * CCE block "various" interrupt. Source is < 8. 6049 */ 6050 static void is_various_int(struct hfi1_devdata *dd, unsigned int source) 6051 { 6052 const struct err_reg_info *eri = &various_err[source]; 6053 6054 /* 6055 * TCritInt cannot go through interrupt_clear_down() 6056 * because it is not a second tier interrupt. The handler 6057 * should be called directly. 6058 */ 6059 if (source == TCRIT_INT_SOURCE) 6060 handle_temp_err(dd); 6061 else if (eri->handler) 6062 interrupt_clear_down(dd, 0, eri); 6063 else 6064 dd_dev_info(dd, 6065 "%s: Unimplemented/reserved interrupt %d\n", 6066 __func__, source); 6067 } 6068 6069 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg) 6070 { 6071 /* src_ctx is always zero */ 6072 struct hfi1_pportdata *ppd = dd->pport; 6073 unsigned long flags; 6074 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 6075 6076 if (reg & QSFP_HFI0_MODPRST_N) { 6077 if (!qsfp_mod_present(ppd)) { 6078 dd_dev_info(dd, "%s: QSFP module removed\n", 6079 __func__); 6080 6081 ppd->driver_link_ready = 0; 6082 /* 6083 * Cable removed, reset all our information about the 6084 * cache and cable capabilities 6085 */ 6086 6087 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6088 /* 6089 * We don't set cache_refresh_required here as we expect 6090 * an interrupt when a cable is inserted 6091 */ 6092 ppd->qsfp_info.cache_valid = 0; 6093 ppd->qsfp_info.reset_needed = 0; 6094 ppd->qsfp_info.limiting_active = 0; 6095 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6096 flags); 6097 /* Invert the ModPresent pin now to detect plug-in */ 6098 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6099 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6100 6101 if ((ppd->offline_disabled_reason > 6102 HFI1_ODR_MASK( 6103 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) || 6104 (ppd->offline_disabled_reason == 6105 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))) 6106 ppd->offline_disabled_reason = 6107 HFI1_ODR_MASK( 6108 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED); 6109 6110 if (ppd->host_link_state == HLS_DN_POLL) { 6111 /* 6112 * The link is still in POLL. This means 6113 * that the normal link down processing 6114 * will not happen. We have to do it here 6115 * before turning the DC off. 6116 */ 6117 queue_work(ppd->link_wq, &ppd->link_down_work); 6118 } 6119 } else { 6120 dd_dev_info(dd, "%s: QSFP module inserted\n", 6121 __func__); 6122 6123 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6124 ppd->qsfp_info.cache_valid = 0; 6125 ppd->qsfp_info.cache_refresh_required = 1; 6126 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6127 flags); 6128 6129 /* 6130 * Stop inversion of ModPresent pin to detect 6131 * removal of the cable 6132 */ 6133 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N; 6134 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6135 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6136 6137 ppd->offline_disabled_reason = 6138 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 6139 } 6140 } 6141 6142 if (reg & QSFP_HFI0_INT_N) { 6143 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n", 6144 __func__); 6145 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6146 ppd->qsfp_info.check_interrupt_flags = 1; 6147 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags); 6148 } 6149 6150 /* Schedule the QSFP work only if there is a cable attached. */ 6151 if (qsfp_mod_present(ppd)) 6152 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work); 6153 } 6154 6155 static int request_host_lcb_access(struct hfi1_devdata *dd) 6156 { 6157 int ret; 6158 6159 ret = do_8051_command(dd, HCMD_MISC, 6160 (u64)HCMD_MISC_REQUEST_LCB_ACCESS << 6161 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6162 if (ret != HCMD_SUCCESS) { 6163 dd_dev_err(dd, "%s: command failed with error %d\n", 6164 __func__, ret); 6165 } 6166 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6167 } 6168 6169 static int request_8051_lcb_access(struct hfi1_devdata *dd) 6170 { 6171 int ret; 6172 6173 ret = do_8051_command(dd, HCMD_MISC, 6174 (u64)HCMD_MISC_GRANT_LCB_ACCESS << 6175 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6176 if (ret != HCMD_SUCCESS) { 6177 dd_dev_err(dd, "%s: command failed with error %d\n", 6178 __func__, ret); 6179 } 6180 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6181 } 6182 6183 /* 6184 * Set the LCB selector - allow host access. The DCC selector always 6185 * points to the host. 6186 */ 6187 static inline void set_host_lcb_access(struct hfi1_devdata *dd) 6188 { 6189 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6190 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK | 6191 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK); 6192 } 6193 6194 /* 6195 * Clear the LCB selector - allow 8051 access. The DCC selector always 6196 * points to the host. 6197 */ 6198 static inline void set_8051_lcb_access(struct hfi1_devdata *dd) 6199 { 6200 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6201 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK); 6202 } 6203 6204 /* 6205 * Acquire LCB access from the 8051. If the host already has access, 6206 * just increment a counter. Otherwise, inform the 8051 that the 6207 * host is taking access. 6208 * 6209 * Returns: 6210 * 0 on success 6211 * -EBUSY if the 8051 has control and cannot be disturbed 6212 * -errno if unable to acquire access from the 8051 6213 */ 6214 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6215 { 6216 struct hfi1_pportdata *ppd = dd->pport; 6217 int ret = 0; 6218 6219 /* 6220 * Use the host link state lock so the operation of this routine 6221 * { link state check, selector change, count increment } can occur 6222 * as a unit against a link state change. Otherwise there is a 6223 * race between the state change and the count increment. 6224 */ 6225 if (sleep_ok) { 6226 mutex_lock(&ppd->hls_lock); 6227 } else { 6228 while (!mutex_trylock(&ppd->hls_lock)) 6229 udelay(1); 6230 } 6231 6232 /* this access is valid only when the link is up */ 6233 if (ppd->host_link_state & HLS_DOWN) { 6234 dd_dev_info(dd, "%s: link state %s not up\n", 6235 __func__, link_state_name(ppd->host_link_state)); 6236 ret = -EBUSY; 6237 goto done; 6238 } 6239 6240 if (dd->lcb_access_count == 0) { 6241 ret = request_host_lcb_access(dd); 6242 if (ret) { 6243 dd_dev_err(dd, 6244 "%s: unable to acquire LCB access, err %d\n", 6245 __func__, ret); 6246 goto done; 6247 } 6248 set_host_lcb_access(dd); 6249 } 6250 dd->lcb_access_count++; 6251 done: 6252 mutex_unlock(&ppd->hls_lock); 6253 return ret; 6254 } 6255 6256 /* 6257 * Release LCB access by decrementing the use count. If the count is moving 6258 * from 1 to 0, inform 8051 that it has control back. 6259 * 6260 * Returns: 6261 * 0 on success 6262 * -errno if unable to release access to the 8051 6263 */ 6264 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6265 { 6266 int ret = 0; 6267 6268 /* 6269 * Use the host link state lock because the acquire needed it. 6270 * Here, we only need to keep { selector change, count decrement } 6271 * as a unit. 6272 */ 6273 if (sleep_ok) { 6274 mutex_lock(&dd->pport->hls_lock); 6275 } else { 6276 while (!mutex_trylock(&dd->pport->hls_lock)) 6277 udelay(1); 6278 } 6279 6280 if (dd->lcb_access_count == 0) { 6281 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n", 6282 __func__); 6283 goto done; 6284 } 6285 6286 if (dd->lcb_access_count == 1) { 6287 set_8051_lcb_access(dd); 6288 ret = request_8051_lcb_access(dd); 6289 if (ret) { 6290 dd_dev_err(dd, 6291 "%s: unable to release LCB access, err %d\n", 6292 __func__, ret); 6293 /* restore host access if the grant didn't work */ 6294 set_host_lcb_access(dd); 6295 goto done; 6296 } 6297 } 6298 dd->lcb_access_count--; 6299 done: 6300 mutex_unlock(&dd->pport->hls_lock); 6301 return ret; 6302 } 6303 6304 /* 6305 * Initialize LCB access variables and state. Called during driver load, 6306 * after most of the initialization is finished. 6307 * 6308 * The DC default is LCB access on for the host. The driver defaults to 6309 * leaving access to the 8051. Assign access now - this constrains the call 6310 * to this routine to be after all LCB set-up is done. In particular, after 6311 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts() 6312 */ 6313 static void init_lcb_access(struct hfi1_devdata *dd) 6314 { 6315 dd->lcb_access_count = 0; 6316 } 6317 6318 /* 6319 * Write a response back to a 8051 request. 6320 */ 6321 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data) 6322 { 6323 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 6324 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK | 6325 (u64)return_code << 6326 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT | 6327 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 6328 } 6329 6330 /* 6331 * Handle host requests from the 8051. 6332 */ 6333 static void handle_8051_request(struct hfi1_pportdata *ppd) 6334 { 6335 struct hfi1_devdata *dd = ppd->dd; 6336 u64 reg; 6337 u16 data = 0; 6338 u8 type; 6339 6340 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1); 6341 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0) 6342 return; /* no request */ 6343 6344 /* zero out COMPLETED so the response is seen */ 6345 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0); 6346 6347 /* extract request details */ 6348 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT) 6349 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK; 6350 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT) 6351 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK; 6352 6353 switch (type) { 6354 case HREQ_LOAD_CONFIG: 6355 case HREQ_SAVE_CONFIG: 6356 case HREQ_READ_CONFIG: 6357 case HREQ_SET_TX_EQ_ABS: 6358 case HREQ_SET_TX_EQ_REL: 6359 case HREQ_ENABLE: 6360 dd_dev_info(dd, "8051 request: request 0x%x not supported\n", 6361 type); 6362 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6363 break; 6364 case HREQ_LCB_RESET: 6365 /* Put the LCB, RX FPE and TX FPE into reset */ 6366 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET); 6367 /* Make sure the write completed */ 6368 (void)read_csr(dd, DCC_CFG_RESET); 6369 /* Hold the reset long enough to take effect */ 6370 udelay(1); 6371 /* Take the LCB, RX FPE and TX FPE out of reset */ 6372 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET); 6373 hreq_response(dd, HREQ_SUCCESS, 0); 6374 6375 break; 6376 case HREQ_CONFIG_DONE: 6377 hreq_response(dd, HREQ_SUCCESS, 0); 6378 break; 6379 6380 case HREQ_INTERFACE_TEST: 6381 hreq_response(dd, HREQ_SUCCESS, data); 6382 break; 6383 default: 6384 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type); 6385 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6386 break; 6387 } 6388 } 6389 6390 /* 6391 * Set up allocation unit vaulue. 6392 */ 6393 void set_up_vau(struct hfi1_devdata *dd, u8 vau) 6394 { 6395 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 6396 6397 /* do not modify other values in the register */ 6398 reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK; 6399 reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT; 6400 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 6401 } 6402 6403 /* 6404 * Set up initial VL15 credits of the remote. Assumes the rest of 6405 * the CM credit registers are zero from a previous global or credit reset. 6406 * Shared limit for VL15 will always be 0. 6407 */ 6408 void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf) 6409 { 6410 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 6411 6412 /* set initial values for total and shared credit limit */ 6413 reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK | 6414 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK); 6415 6416 /* 6417 * Set total limit to be equal to VL15 credits. 6418 * Leave shared limit at 0. 6419 */ 6420 reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 6421 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 6422 6423 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf 6424 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT); 6425 } 6426 6427 /* 6428 * Zero all credit details from the previous connection and 6429 * reset the CM manager's internal counters. 6430 */ 6431 void reset_link_credits(struct hfi1_devdata *dd) 6432 { 6433 int i; 6434 6435 /* remove all previous VL credit limits */ 6436 for (i = 0; i < TXE_NUM_DATA_VL; i++) 6437 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 6438 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 6439 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0); 6440 /* reset the CM block */ 6441 pio_send_control(dd, PSC_CM_RESET); 6442 /* reset cached value */ 6443 dd->vl15buf_cached = 0; 6444 } 6445 6446 /* convert a vCU to a CU */ 6447 static u32 vcu_to_cu(u8 vcu) 6448 { 6449 return 1 << vcu; 6450 } 6451 6452 /* convert a CU to a vCU */ 6453 static u8 cu_to_vcu(u32 cu) 6454 { 6455 return ilog2(cu); 6456 } 6457 6458 /* convert a vAU to an AU */ 6459 static u32 vau_to_au(u8 vau) 6460 { 6461 return 8 * (1 << vau); 6462 } 6463 6464 static void set_linkup_defaults(struct hfi1_pportdata *ppd) 6465 { 6466 ppd->sm_trap_qp = 0x0; 6467 ppd->sa_qp = 0x1; 6468 } 6469 6470 /* 6471 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset. 6472 */ 6473 static void lcb_shutdown(struct hfi1_devdata *dd, int abort) 6474 { 6475 u64 reg; 6476 6477 /* clear lcb run: LCB_CFG_RUN.EN = 0 */ 6478 write_csr(dd, DC_LCB_CFG_RUN, 0); 6479 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */ 6480 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 6481 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT); 6482 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */ 6483 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN); 6484 reg = read_csr(dd, DCC_CFG_RESET); 6485 write_csr(dd, DCC_CFG_RESET, reg | 6486 DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE); 6487 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */ 6488 if (!abort) { 6489 udelay(1); /* must hold for the longer of 16cclks or 20ns */ 6490 write_csr(dd, DCC_CFG_RESET, reg); 6491 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6492 } 6493 } 6494 6495 /* 6496 * This routine should be called after the link has been transitioned to 6497 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into 6498 * reset). 6499 * 6500 * The expectation is that the caller of this routine would have taken 6501 * care of properly transitioning the link into the correct state. 6502 * NOTE: the caller needs to acquire the dd->dc8051_lock lock 6503 * before calling this function. 6504 */ 6505 static void _dc_shutdown(struct hfi1_devdata *dd) 6506 { 6507 lockdep_assert_held(&dd->dc8051_lock); 6508 6509 if (dd->dc_shutdown) 6510 return; 6511 6512 dd->dc_shutdown = 1; 6513 /* Shutdown the LCB */ 6514 lcb_shutdown(dd, 1); 6515 /* 6516 * Going to OFFLINE would have causes the 8051 to put the 6517 * SerDes into reset already. Just need to shut down the 8051, 6518 * itself. 6519 */ 6520 write_csr(dd, DC_DC8051_CFG_RST, 0x1); 6521 } 6522 6523 static void dc_shutdown(struct hfi1_devdata *dd) 6524 { 6525 mutex_lock(&dd->dc8051_lock); 6526 _dc_shutdown(dd); 6527 mutex_unlock(&dd->dc8051_lock); 6528 } 6529 6530 /* 6531 * Calling this after the DC has been brought out of reset should not 6532 * do any damage. 6533 * NOTE: the caller needs to acquire the dd->dc8051_lock lock 6534 * before calling this function. 6535 */ 6536 static void _dc_start(struct hfi1_devdata *dd) 6537 { 6538 lockdep_assert_held(&dd->dc8051_lock); 6539 6540 if (!dd->dc_shutdown) 6541 return; 6542 6543 /* Take the 8051 out of reset */ 6544 write_csr(dd, DC_DC8051_CFG_RST, 0ull); 6545 /* Wait until 8051 is ready */ 6546 if (wait_fm_ready(dd, TIMEOUT_8051_START)) 6547 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n", 6548 __func__); 6549 6550 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */ 6551 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET); 6552 /* lcb_shutdown() with abort=1 does not restore these */ 6553 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6554 dd->dc_shutdown = 0; 6555 } 6556 6557 static void dc_start(struct hfi1_devdata *dd) 6558 { 6559 mutex_lock(&dd->dc8051_lock); 6560 _dc_start(dd); 6561 mutex_unlock(&dd->dc8051_lock); 6562 } 6563 6564 /* 6565 * These LCB adjustments are for the Aurora SerDes core in the FPGA. 6566 */ 6567 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd) 6568 { 6569 u64 rx_radr, tx_radr; 6570 u32 version; 6571 6572 if (dd->icode != ICODE_FPGA_EMULATION) 6573 return; 6574 6575 /* 6576 * These LCB defaults on emulator _s are good, nothing to do here: 6577 * LCB_CFG_TX_FIFOS_RADR 6578 * LCB_CFG_RX_FIFOS_RADR 6579 * LCB_CFG_LN_DCLK 6580 * LCB_CFG_IGNORE_LOST_RCLK 6581 */ 6582 if (is_emulator_s(dd)) 6583 return; 6584 /* else this is _p */ 6585 6586 version = emulator_rev(dd); 6587 if (!is_ax(dd)) 6588 version = 0x2d; /* all B0 use 0x2d or higher settings */ 6589 6590 if (version <= 0x12) { 6591 /* release 0x12 and below */ 6592 6593 /* 6594 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9 6595 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9 6596 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa 6597 */ 6598 rx_radr = 6599 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6600 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6601 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6602 /* 6603 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default) 6604 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6 6605 */ 6606 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6607 } else if (version <= 0x18) { 6608 /* release 0x13 up to 0x18 */ 6609 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6610 rx_radr = 6611 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6612 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6613 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6614 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6615 } else if (version == 0x19) { 6616 /* release 0x19 */ 6617 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */ 6618 rx_radr = 6619 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6620 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6621 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6622 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6623 } else if (version == 0x1a) { 6624 /* release 0x1a */ 6625 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6626 rx_radr = 6627 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6628 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6629 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6630 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6631 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull); 6632 } else { 6633 /* release 0x1b and higher */ 6634 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */ 6635 rx_radr = 6636 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6637 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6638 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6639 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6640 } 6641 6642 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr); 6643 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */ 6644 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 6645 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 6646 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr); 6647 } 6648 6649 /* 6650 * Handle a SMA idle message 6651 * 6652 * This is a work-queue function outside of the interrupt. 6653 */ 6654 void handle_sma_message(struct work_struct *work) 6655 { 6656 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6657 sma_message_work); 6658 struct hfi1_devdata *dd = ppd->dd; 6659 u64 msg; 6660 int ret; 6661 6662 /* 6663 * msg is bytes 1-4 of the 40-bit idle message - the command code 6664 * is stripped off 6665 */ 6666 ret = read_idle_sma(dd, &msg); 6667 if (ret) 6668 return; 6669 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg); 6670 /* 6671 * React to the SMA message. Byte[1] (0 for us) is the command. 6672 */ 6673 switch (msg & 0xff) { 6674 case SMA_IDLE_ARM: 6675 /* 6676 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6677 * State Transitions 6678 * 6679 * Only expected in INIT or ARMED, discard otherwise. 6680 */ 6681 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED)) 6682 ppd->neighbor_normal = 1; 6683 break; 6684 case SMA_IDLE_ACTIVE: 6685 /* 6686 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6687 * State Transitions 6688 * 6689 * Can activate the node. Discard otherwise. 6690 */ 6691 if (ppd->host_link_state == HLS_UP_ARMED && 6692 ppd->is_active_optimize_enabled) { 6693 ppd->neighbor_normal = 1; 6694 ret = set_link_state(ppd, HLS_UP_ACTIVE); 6695 if (ret) 6696 dd_dev_err( 6697 dd, 6698 "%s: received Active SMA idle message, couldn't set link to Active\n", 6699 __func__); 6700 } 6701 break; 6702 default: 6703 dd_dev_err(dd, 6704 "%s: received unexpected SMA idle message 0x%llx\n", 6705 __func__, msg); 6706 break; 6707 } 6708 } 6709 6710 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear) 6711 { 6712 u64 rcvctrl; 6713 unsigned long flags; 6714 6715 spin_lock_irqsave(&dd->rcvctrl_lock, flags); 6716 rcvctrl = read_csr(dd, RCV_CTRL); 6717 rcvctrl |= add; 6718 rcvctrl &= ~clear; 6719 write_csr(dd, RCV_CTRL, rcvctrl); 6720 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags); 6721 } 6722 6723 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add) 6724 { 6725 adjust_rcvctrl(dd, add, 0); 6726 } 6727 6728 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear) 6729 { 6730 adjust_rcvctrl(dd, 0, clear); 6731 } 6732 6733 /* 6734 * Called from all interrupt handlers to start handling an SPC freeze. 6735 */ 6736 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags) 6737 { 6738 struct hfi1_devdata *dd = ppd->dd; 6739 struct send_context *sc; 6740 int i; 6741 int sc_flags; 6742 6743 if (flags & FREEZE_SELF) 6744 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6745 6746 /* enter frozen mode */ 6747 dd->flags |= HFI1_FROZEN; 6748 6749 /* notify all SDMA engines that they are going into a freeze */ 6750 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN)); 6751 6752 sc_flags = SCF_FROZEN | SCF_HALTED | (flags & FREEZE_LINK_DOWN ? 6753 SCF_LINK_DOWN : 0); 6754 /* do halt pre-handling on all enabled send contexts */ 6755 for (i = 0; i < dd->num_send_contexts; i++) { 6756 sc = dd->send_contexts[i].sc; 6757 if (sc && (sc->flags & SCF_ENABLED)) 6758 sc_stop(sc, sc_flags); 6759 } 6760 6761 /* Send context are frozen. Notify user space */ 6762 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT); 6763 6764 if (flags & FREEZE_ABORT) { 6765 dd_dev_err(dd, 6766 "Aborted freeze recovery. Please REBOOT system\n"); 6767 return; 6768 } 6769 /* queue non-interrupt handler */ 6770 queue_work(ppd->hfi1_wq, &ppd->freeze_work); 6771 } 6772 6773 /* 6774 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen, 6775 * depending on the "freeze" parameter. 6776 * 6777 * No need to return an error if it times out, our only option 6778 * is to proceed anyway. 6779 */ 6780 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze) 6781 { 6782 unsigned long timeout; 6783 u64 reg; 6784 6785 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT); 6786 while (1) { 6787 reg = read_csr(dd, CCE_STATUS); 6788 if (freeze) { 6789 /* waiting until all indicators are set */ 6790 if ((reg & ALL_FROZE) == ALL_FROZE) 6791 return; /* all done */ 6792 } else { 6793 /* waiting until all indicators are clear */ 6794 if ((reg & ALL_FROZE) == 0) 6795 return; /* all done */ 6796 } 6797 6798 if (time_after(jiffies, timeout)) { 6799 dd_dev_err(dd, 6800 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing", 6801 freeze ? "" : "un", reg & ALL_FROZE, 6802 freeze ? ALL_FROZE : 0ull); 6803 return; 6804 } 6805 usleep_range(80, 120); 6806 } 6807 } 6808 6809 /* 6810 * Do all freeze handling for the RXE block. 6811 */ 6812 static void rxe_freeze(struct hfi1_devdata *dd) 6813 { 6814 int i; 6815 struct hfi1_ctxtdata *rcd; 6816 6817 /* disable port */ 6818 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6819 6820 /* disable all receive contexts */ 6821 for (i = 0; i < dd->num_rcv_contexts; i++) { 6822 rcd = hfi1_rcd_get_by_index(dd, i); 6823 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd); 6824 hfi1_rcd_put(rcd); 6825 } 6826 } 6827 6828 /* 6829 * Unfreeze handling for the RXE block - kernel contexts only. 6830 * This will also enable the port. User contexts will do unfreeze 6831 * handling on a per-context basis as they call into the driver. 6832 * 6833 */ 6834 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd) 6835 { 6836 u32 rcvmask; 6837 u16 i; 6838 struct hfi1_ctxtdata *rcd; 6839 6840 /* enable all kernel contexts */ 6841 for (i = 0; i < dd->num_rcv_contexts; i++) { 6842 rcd = hfi1_rcd_get_by_index(dd, i); 6843 6844 /* Ensure all non-user contexts(including vnic) are enabled */ 6845 if (!rcd || 6846 (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) { 6847 hfi1_rcd_put(rcd); 6848 continue; 6849 } 6850 rcvmask = HFI1_RCVCTRL_CTXT_ENB; 6851 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */ 6852 rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ? 6853 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS; 6854 hfi1_rcvctrl(dd, rcvmask, rcd); 6855 hfi1_rcd_put(rcd); 6856 } 6857 6858 /* enable port */ 6859 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6860 } 6861 6862 /* 6863 * Non-interrupt SPC freeze handling. 6864 * 6865 * This is a work-queue function outside of the triggering interrupt. 6866 */ 6867 void handle_freeze(struct work_struct *work) 6868 { 6869 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6870 freeze_work); 6871 struct hfi1_devdata *dd = ppd->dd; 6872 6873 /* wait for freeze indicators on all affected blocks */ 6874 wait_for_freeze_status(dd, 1); 6875 6876 /* SPC is now frozen */ 6877 6878 /* do send PIO freeze steps */ 6879 pio_freeze(dd); 6880 6881 /* do send DMA freeze steps */ 6882 sdma_freeze(dd); 6883 6884 /* do send egress freeze steps - nothing to do */ 6885 6886 /* do receive freeze steps */ 6887 rxe_freeze(dd); 6888 6889 /* 6890 * Unfreeze the hardware - clear the freeze, wait for each 6891 * block's frozen bit to clear, then clear the frozen flag. 6892 */ 6893 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6894 wait_for_freeze_status(dd, 0); 6895 6896 if (is_ax(dd)) { 6897 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6898 wait_for_freeze_status(dd, 1); 6899 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6900 wait_for_freeze_status(dd, 0); 6901 } 6902 6903 /* do send PIO unfreeze steps for kernel contexts */ 6904 pio_kernel_unfreeze(dd); 6905 6906 /* do send DMA unfreeze steps */ 6907 sdma_unfreeze(dd); 6908 6909 /* do send egress unfreeze steps - nothing to do */ 6910 6911 /* do receive unfreeze steps for kernel contexts */ 6912 rxe_kernel_unfreeze(dd); 6913 6914 /* 6915 * The unfreeze procedure touches global device registers when 6916 * it disables and re-enables RXE. Mark the device unfrozen 6917 * after all that is done so other parts of the driver waiting 6918 * for the device to unfreeze don't do things out of order. 6919 * 6920 * The above implies that the meaning of HFI1_FROZEN flag is 6921 * "Device has gone into freeze mode and freeze mode handling 6922 * is still in progress." 6923 * 6924 * The flag will be removed when freeze mode processing has 6925 * completed. 6926 */ 6927 dd->flags &= ~HFI1_FROZEN; 6928 wake_up(&dd->event_queue); 6929 6930 /* no longer frozen */ 6931 } 6932 6933 /** 6934 * update_xmit_counters - update PortXmitWait/PortVlXmitWait 6935 * counters. 6936 * @ppd: info of physical Hfi port 6937 * @link_width: new link width after link up or downgrade 6938 * 6939 * Update the PortXmitWait and PortVlXmitWait counters after 6940 * a link up or downgrade event to reflect a link width change. 6941 */ 6942 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width) 6943 { 6944 int i; 6945 u16 tx_width; 6946 u16 link_speed; 6947 6948 tx_width = tx_link_width(link_width); 6949 link_speed = get_link_speed(ppd->link_speed_active); 6950 6951 /* 6952 * There are C_VL_COUNT number of PortVLXmitWait counters. 6953 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter. 6954 */ 6955 for (i = 0; i < C_VL_COUNT + 1; i++) 6956 get_xmit_wait_counters(ppd, tx_width, link_speed, i); 6957 } 6958 6959 /* 6960 * Handle a link up interrupt from the 8051. 6961 * 6962 * This is a work-queue function outside of the interrupt. 6963 */ 6964 void handle_link_up(struct work_struct *work) 6965 { 6966 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6967 link_up_work); 6968 struct hfi1_devdata *dd = ppd->dd; 6969 6970 set_link_state(ppd, HLS_UP_INIT); 6971 6972 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */ 6973 read_ltp_rtt(dd); 6974 /* 6975 * OPA specifies that certain counters are cleared on a transition 6976 * to link up, so do that. 6977 */ 6978 clear_linkup_counters(dd); 6979 /* 6980 * And (re)set link up default values. 6981 */ 6982 set_linkup_defaults(ppd); 6983 6984 /* 6985 * Set VL15 credits. Use cached value from verify cap interrupt. 6986 * In case of quick linkup or simulator, vl15 value will be set by 6987 * handle_linkup_change. VerifyCap interrupt handler will not be 6988 * called in those scenarios. 6989 */ 6990 if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) 6991 set_up_vl15(dd, dd->vl15buf_cached); 6992 6993 /* enforce link speed enabled */ 6994 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) { 6995 /* oops - current speed is not enabled, bounce */ 6996 dd_dev_err(dd, 6997 "Link speed active 0x%x is outside enabled 0x%x, downing link\n", 6998 ppd->link_speed_active, ppd->link_speed_enabled); 6999 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0, 7000 OPA_LINKDOWN_REASON_SPEED_POLICY); 7001 set_link_state(ppd, HLS_DN_OFFLINE); 7002 start_link(ppd); 7003 } 7004 } 7005 7006 /* 7007 * Several pieces of LNI information were cached for SMA in ppd. 7008 * Reset these on link down 7009 */ 7010 static void reset_neighbor_info(struct hfi1_pportdata *ppd) 7011 { 7012 ppd->neighbor_guid = 0; 7013 ppd->neighbor_port_number = 0; 7014 ppd->neighbor_type = 0; 7015 ppd->neighbor_fm_security = 0; 7016 } 7017 7018 static const char * const link_down_reason_strs[] = { 7019 [OPA_LINKDOWN_REASON_NONE] = "None", 7020 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0", 7021 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length", 7022 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long", 7023 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short", 7024 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID", 7025 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID", 7026 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2", 7027 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC", 7028 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8", 7029 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail", 7030 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10", 7031 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error", 7032 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15", 7033 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker", 7034 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14", 7035 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15", 7036 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance", 7037 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance", 7038 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance", 7039 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack", 7040 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker", 7041 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt", 7042 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit", 7043 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit", 7044 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24", 7045 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25", 7046 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26", 7047 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27", 7048 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28", 7049 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29", 7050 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30", 7051 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] = 7052 "Excessive buffer overrun", 7053 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown", 7054 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot", 7055 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown", 7056 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce", 7057 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy", 7058 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy", 7059 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected", 7060 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] = 7061 "Local media not installed", 7062 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed", 7063 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config", 7064 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] = 7065 "End to end not installed", 7066 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy", 7067 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy", 7068 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy", 7069 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management", 7070 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled", 7071 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient" 7072 }; 7073 7074 /* return the neighbor link down reason string */ 7075 static const char *link_down_reason_str(u8 reason) 7076 { 7077 const char *str = NULL; 7078 7079 if (reason < ARRAY_SIZE(link_down_reason_strs)) 7080 str = link_down_reason_strs[reason]; 7081 if (!str) 7082 str = "(invalid)"; 7083 7084 return str; 7085 } 7086 7087 /* 7088 * Handle a link down interrupt from the 8051. 7089 * 7090 * This is a work-queue function outside of the interrupt. 7091 */ 7092 void handle_link_down(struct work_struct *work) 7093 { 7094 u8 lcl_reason, neigh_reason = 0; 7095 u8 link_down_reason; 7096 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7097 link_down_work); 7098 int was_up; 7099 static const char ldr_str[] = "Link down reason: "; 7100 7101 if ((ppd->host_link_state & 7102 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) && 7103 ppd->port_type == PORT_TYPE_FIXED) 7104 ppd->offline_disabled_reason = 7105 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED); 7106 7107 /* Go offline first, then deal with reading/writing through 8051 */ 7108 was_up = !!(ppd->host_link_state & HLS_UP); 7109 set_link_state(ppd, HLS_DN_OFFLINE); 7110 xchg(&ppd->is_link_down_queued, 0); 7111 7112 if (was_up) { 7113 lcl_reason = 0; 7114 /* link down reason is only valid if the link was up */ 7115 read_link_down_reason(ppd->dd, &link_down_reason); 7116 switch (link_down_reason) { 7117 case LDR_LINK_TRANSFER_ACTIVE_LOW: 7118 /* the link went down, no idle message reason */ 7119 dd_dev_info(ppd->dd, "%sUnexpected link down\n", 7120 ldr_str); 7121 break; 7122 case LDR_RECEIVED_LINKDOWN_IDLE_MSG: 7123 /* 7124 * The neighbor reason is only valid if an idle message 7125 * was received for it. 7126 */ 7127 read_planned_down_reason_code(ppd->dd, &neigh_reason); 7128 dd_dev_info(ppd->dd, 7129 "%sNeighbor link down message %d, %s\n", 7130 ldr_str, neigh_reason, 7131 link_down_reason_str(neigh_reason)); 7132 break; 7133 case LDR_RECEIVED_HOST_OFFLINE_REQ: 7134 dd_dev_info(ppd->dd, 7135 "%sHost requested link to go offline\n", 7136 ldr_str); 7137 break; 7138 default: 7139 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n", 7140 ldr_str, link_down_reason); 7141 break; 7142 } 7143 7144 /* 7145 * If no reason, assume peer-initiated but missed 7146 * LinkGoingDown idle flits. 7147 */ 7148 if (neigh_reason == 0) 7149 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN; 7150 } else { 7151 /* went down while polling or going up */ 7152 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT; 7153 } 7154 7155 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0); 7156 7157 /* inform the SMA when the link transitions from up to down */ 7158 if (was_up && ppd->local_link_down_reason.sma == 0 && 7159 ppd->neigh_link_down_reason.sma == 0) { 7160 ppd->local_link_down_reason.sma = 7161 ppd->local_link_down_reason.latest; 7162 ppd->neigh_link_down_reason.sma = 7163 ppd->neigh_link_down_reason.latest; 7164 } 7165 7166 reset_neighbor_info(ppd); 7167 7168 /* disable the port */ 7169 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 7170 7171 /* 7172 * If there is no cable attached, turn the DC off. Otherwise, 7173 * start the link bring up. 7174 */ 7175 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) 7176 dc_shutdown(ppd->dd); 7177 else 7178 start_link(ppd); 7179 } 7180 7181 void handle_link_bounce(struct work_struct *work) 7182 { 7183 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7184 link_bounce_work); 7185 7186 /* 7187 * Only do something if the link is currently up. 7188 */ 7189 if (ppd->host_link_state & HLS_UP) { 7190 set_link_state(ppd, HLS_DN_OFFLINE); 7191 start_link(ppd); 7192 } else { 7193 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n", 7194 __func__, link_state_name(ppd->host_link_state)); 7195 } 7196 } 7197 7198 /* 7199 * Mask conversion: Capability exchange to Port LTP. The capability 7200 * exchange has an implicit 16b CRC that is mandatory. 7201 */ 7202 static int cap_to_port_ltp(int cap) 7203 { 7204 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */ 7205 7206 if (cap & CAP_CRC_14B) 7207 port_ltp |= PORT_LTP_CRC_MODE_14; 7208 if (cap & CAP_CRC_48B) 7209 port_ltp |= PORT_LTP_CRC_MODE_48; 7210 if (cap & CAP_CRC_12B_16B_PER_LANE) 7211 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE; 7212 7213 return port_ltp; 7214 } 7215 7216 /* 7217 * Convert an OPA Port LTP mask to capability mask 7218 */ 7219 int port_ltp_to_cap(int port_ltp) 7220 { 7221 int cap_mask = 0; 7222 7223 if (port_ltp & PORT_LTP_CRC_MODE_14) 7224 cap_mask |= CAP_CRC_14B; 7225 if (port_ltp & PORT_LTP_CRC_MODE_48) 7226 cap_mask |= CAP_CRC_48B; 7227 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE) 7228 cap_mask |= CAP_CRC_12B_16B_PER_LANE; 7229 7230 return cap_mask; 7231 } 7232 7233 /* 7234 * Convert a single DC LCB CRC mode to an OPA Port LTP mask. 7235 */ 7236 static int lcb_to_port_ltp(int lcb_crc) 7237 { 7238 int port_ltp = 0; 7239 7240 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE) 7241 port_ltp = PORT_LTP_CRC_MODE_PER_LANE; 7242 else if (lcb_crc == LCB_CRC_48B) 7243 port_ltp = PORT_LTP_CRC_MODE_48; 7244 else if (lcb_crc == LCB_CRC_14B) 7245 port_ltp = PORT_LTP_CRC_MODE_14; 7246 else 7247 port_ltp = PORT_LTP_CRC_MODE_16; 7248 7249 return port_ltp; 7250 } 7251 7252 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd) 7253 { 7254 if (ppd->pkeys[2] != 0) { 7255 ppd->pkeys[2] = 0; 7256 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); 7257 hfi1_event_pkey_change(ppd->dd, ppd->port); 7258 } 7259 } 7260 7261 /* 7262 * Convert the given link width to the OPA link width bitmask. 7263 */ 7264 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width) 7265 { 7266 switch (width) { 7267 case 0: 7268 /* 7269 * Simulator and quick linkup do not set the width. 7270 * Just set it to 4x without complaint. 7271 */ 7272 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup) 7273 return OPA_LINK_WIDTH_4X; 7274 return 0; /* no lanes up */ 7275 case 1: return OPA_LINK_WIDTH_1X; 7276 case 2: return OPA_LINK_WIDTH_2X; 7277 case 3: return OPA_LINK_WIDTH_3X; 7278 case 4: return OPA_LINK_WIDTH_4X; 7279 default: 7280 dd_dev_info(dd, "%s: invalid width %d, using 4\n", 7281 __func__, width); 7282 return OPA_LINK_WIDTH_4X; 7283 } 7284 } 7285 7286 /* 7287 * Do a population count on the bottom nibble. 7288 */ 7289 static const u8 bit_counts[16] = { 7290 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 7291 }; 7292 7293 static inline u8 nibble_to_count(u8 nibble) 7294 { 7295 return bit_counts[nibble & 0xf]; 7296 } 7297 7298 /* 7299 * Read the active lane information from the 8051 registers and return 7300 * their widths. 7301 * 7302 * Active lane information is found in these 8051 registers: 7303 * enable_lane_tx 7304 * enable_lane_rx 7305 */ 7306 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width, 7307 u16 *rx_width) 7308 { 7309 u16 tx, rx; 7310 u8 enable_lane_rx; 7311 u8 enable_lane_tx; 7312 u8 tx_polarity_inversion; 7313 u8 rx_polarity_inversion; 7314 u8 max_rate; 7315 7316 /* read the active lanes */ 7317 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 7318 &rx_polarity_inversion, &max_rate); 7319 read_local_lni(dd, &enable_lane_rx); 7320 7321 /* convert to counts */ 7322 tx = nibble_to_count(enable_lane_tx); 7323 rx = nibble_to_count(enable_lane_rx); 7324 7325 /* 7326 * Set link_speed_active here, overriding what was set in 7327 * handle_verify_cap(). The ASIC 8051 firmware does not correctly 7328 * set the max_rate field in handle_verify_cap until v0.19. 7329 */ 7330 if ((dd->icode == ICODE_RTL_SILICON) && 7331 (dd->dc8051_ver < dc8051_ver(0, 19, 0))) { 7332 /* max_rate: 0 = 12.5G, 1 = 25G */ 7333 switch (max_rate) { 7334 case 0: 7335 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G; 7336 break; 7337 case 1: 7338 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; 7339 break; 7340 default: 7341 dd_dev_err(dd, 7342 "%s: unexpected max rate %d, using 25Gb\n", 7343 __func__, (int)max_rate); 7344 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; 7345 break; 7346 } 7347 } 7348 7349 dd_dev_info(dd, 7350 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n", 7351 enable_lane_tx, tx, enable_lane_rx, rx); 7352 *tx_width = link_width_to_bits(dd, tx); 7353 *rx_width = link_width_to_bits(dd, rx); 7354 } 7355 7356 /* 7357 * Read verify_cap_local_fm_link_width[1] to obtain the link widths. 7358 * Valid after the end of VerifyCap and during LinkUp. Does not change 7359 * after link up. I.e. look elsewhere for downgrade information. 7360 * 7361 * Bits are: 7362 * + bits [7:4] contain the number of active transmitters 7363 * + bits [3:0] contain the number of active receivers 7364 * These are numbers 1 through 4 and can be different values if the 7365 * link is asymmetric. 7366 * 7367 * verify_cap_local_fm_link_width[0] retains its original value. 7368 */ 7369 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width, 7370 u16 *rx_width) 7371 { 7372 u16 widths, tx, rx; 7373 u8 misc_bits, local_flags; 7374 u16 active_tx, active_rx; 7375 7376 read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths); 7377 tx = widths >> 12; 7378 rx = (widths >> 8) & 0xf; 7379 7380 *tx_width = link_width_to_bits(dd, tx); 7381 *rx_width = link_width_to_bits(dd, rx); 7382 7383 /* print the active widths */ 7384 get_link_widths(dd, &active_tx, &active_rx); 7385 } 7386 7387 /* 7388 * Set ppd->link_width_active and ppd->link_width_downgrade_active using 7389 * hardware information when the link first comes up. 7390 * 7391 * The link width is not available until after VerifyCap.AllFramesReceived 7392 * (the trigger for handle_verify_cap), so this is outside that routine 7393 * and should be called when the 8051 signals linkup. 7394 */ 7395 void get_linkup_link_widths(struct hfi1_pportdata *ppd) 7396 { 7397 u16 tx_width, rx_width; 7398 7399 /* get end-of-LNI link widths */ 7400 get_linkup_widths(ppd->dd, &tx_width, &rx_width); 7401 7402 /* use tx_width as the link is supposed to be symmetric on link up */ 7403 ppd->link_width_active = tx_width; 7404 /* link width downgrade active (LWD.A) starts out matching LW.A */ 7405 ppd->link_width_downgrade_tx_active = ppd->link_width_active; 7406 ppd->link_width_downgrade_rx_active = ppd->link_width_active; 7407 /* per OPA spec, on link up LWD.E resets to LWD.S */ 7408 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported; 7409 /* cache the active egress rate (units {10^6 bits/sec]) */ 7410 ppd->current_egress_rate = active_egress_rate(ppd); 7411 } 7412 7413 /* 7414 * Handle a verify capabilities interrupt from the 8051. 7415 * 7416 * This is a work-queue function outside of the interrupt. 7417 */ 7418 void handle_verify_cap(struct work_struct *work) 7419 { 7420 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7421 link_vc_work); 7422 struct hfi1_devdata *dd = ppd->dd; 7423 u64 reg; 7424 u8 power_management; 7425 u8 continuous; 7426 u8 vcu; 7427 u8 vau; 7428 u8 z; 7429 u16 vl15buf; 7430 u16 link_widths; 7431 u16 crc_mask; 7432 u16 crc_val; 7433 u16 device_id; 7434 u16 active_tx, active_rx; 7435 u8 partner_supported_crc; 7436 u8 remote_tx_rate; 7437 u8 device_rev; 7438 7439 set_link_state(ppd, HLS_VERIFY_CAP); 7440 7441 lcb_shutdown(dd, 0); 7442 adjust_lcb_for_fpga_serdes(dd); 7443 7444 read_vc_remote_phy(dd, &power_management, &continuous); 7445 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf, 7446 &partner_supported_crc); 7447 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths); 7448 read_remote_device_id(dd, &device_id, &device_rev); 7449 7450 /* print the active widths */ 7451 get_link_widths(dd, &active_tx, &active_rx); 7452 dd_dev_info(dd, 7453 "Peer PHY: power management 0x%x, continuous updates 0x%x\n", 7454 (int)power_management, (int)continuous); 7455 dd_dev_info(dd, 7456 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n", 7457 (int)vau, (int)z, (int)vcu, (int)vl15buf, 7458 (int)partner_supported_crc); 7459 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n", 7460 (u32)remote_tx_rate, (u32)link_widths); 7461 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n", 7462 (u32)device_id, (u32)device_rev); 7463 /* 7464 * The peer vAU value just read is the peer receiver value. HFI does 7465 * not support a transmit vAU of 0 (AU == 8). We advertised that 7466 * with Z=1 in the fabric capabilities sent to the peer. The peer 7467 * will see our Z=1, and, if it advertised a vAU of 0, will move its 7468 * receive to vAU of 1 (AU == 16). Do the same here. We do not care 7469 * about the peer Z value - our sent vAU is 3 (hardwired) and is not 7470 * subject to the Z value exception. 7471 */ 7472 if (vau == 0) 7473 vau = 1; 7474 set_up_vau(dd, vau); 7475 7476 /* 7477 * Set VL15 credits to 0 in global credit register. Cache remote VL15 7478 * credits value and wait for link-up interrupt ot set it. 7479 */ 7480 set_up_vl15(dd, 0); 7481 dd->vl15buf_cached = vl15buf; 7482 7483 /* set up the LCB CRC mode */ 7484 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc; 7485 7486 /* order is important: use the lowest bit in common */ 7487 if (crc_mask & CAP_CRC_14B) 7488 crc_val = LCB_CRC_14B; 7489 else if (crc_mask & CAP_CRC_48B) 7490 crc_val = LCB_CRC_48B; 7491 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE) 7492 crc_val = LCB_CRC_12B_16B_PER_LANE; 7493 else 7494 crc_val = LCB_CRC_16B; 7495 7496 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val); 7497 write_csr(dd, DC_LCB_CFG_CRC_MODE, 7498 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT); 7499 7500 /* set (14b only) or clear sideband credit */ 7501 reg = read_csr(dd, SEND_CM_CTRL); 7502 if (crc_val == LCB_CRC_14B && crc_14b_sideband) { 7503 write_csr(dd, SEND_CM_CTRL, 7504 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7505 } else { 7506 write_csr(dd, SEND_CM_CTRL, 7507 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7508 } 7509 7510 ppd->link_speed_active = 0; /* invalid value */ 7511 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) { 7512 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */ 7513 switch (remote_tx_rate) { 7514 case 0: 7515 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7516 break; 7517 case 1: 7518 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7519 break; 7520 } 7521 } else { 7522 /* actual rate is highest bit of the ANDed rates */ 7523 u8 rate = remote_tx_rate & ppd->local_tx_rate; 7524 7525 if (rate & 2) 7526 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7527 else if (rate & 1) 7528 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7529 } 7530 if (ppd->link_speed_active == 0) { 7531 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n", 7532 __func__, (int)remote_tx_rate); 7533 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7534 } 7535 7536 /* 7537 * Cache the values of the supported, enabled, and active 7538 * LTP CRC modes to return in 'portinfo' queries. But the bit 7539 * flags that are returned in the portinfo query differ from 7540 * what's in the link_crc_mask, crc_sizes, and crc_val 7541 * variables. Convert these here. 7542 */ 7543 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 7544 /* supported crc modes */ 7545 ppd->port_ltp_crc_mode |= 7546 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4; 7547 /* enabled crc modes */ 7548 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val); 7549 /* active crc mode */ 7550 7551 /* set up the remote credit return table */ 7552 assign_remote_cm_au_table(dd, vcu); 7553 7554 /* 7555 * The LCB is reset on entry to handle_verify_cap(), so this must 7556 * be applied on every link up. 7557 * 7558 * Adjust LCB error kill enable to kill the link if 7559 * these RBUF errors are seen: 7560 * REPLAY_BUF_MBE_SMASK 7561 * FLIT_INPUT_BUF_MBE_SMASK 7562 */ 7563 if (is_ax(dd)) { /* fixed in B0 */ 7564 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN); 7565 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK 7566 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK; 7567 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg); 7568 } 7569 7570 /* pull LCB fifos out of reset - all fifo clocks must be stable */ 7571 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 7572 7573 /* give 8051 access to the LCB CSRs */ 7574 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 7575 set_8051_lcb_access(dd); 7576 7577 /* tell the 8051 to go to LinkUp */ 7578 set_link_state(ppd, HLS_GOING_UP); 7579 } 7580 7581 /** 7582 * apply_link_downgrade_policy - Apply the link width downgrade enabled 7583 * policy against the current active link widths. 7584 * @ppd: info of physical Hfi port 7585 * @refresh_widths: True indicates link downgrade event 7586 * @return: True indicates a successful link downgrade. False indicates 7587 * link downgrade event failed and the link will bounce back to 7588 * default link width. 7589 * 7590 * Called when the enabled policy changes or the active link widths 7591 * change. 7592 * Refresh_widths indicates that a link downgrade occurred. The 7593 * link_downgraded variable is set by refresh_widths and 7594 * determines the success/failure of the policy application. 7595 */ 7596 bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, 7597 bool refresh_widths) 7598 { 7599 int do_bounce = 0; 7600 int tries; 7601 u16 lwde; 7602 u16 tx, rx; 7603 bool link_downgraded = refresh_widths; 7604 7605 /* use the hls lock to avoid a race with actual link up */ 7606 tries = 0; 7607 retry: 7608 mutex_lock(&ppd->hls_lock); 7609 /* only apply if the link is up */ 7610 if (ppd->host_link_state & HLS_DOWN) { 7611 /* still going up..wait and retry */ 7612 if (ppd->host_link_state & HLS_GOING_UP) { 7613 if (++tries < 1000) { 7614 mutex_unlock(&ppd->hls_lock); 7615 usleep_range(100, 120); /* arbitrary */ 7616 goto retry; 7617 } 7618 dd_dev_err(ppd->dd, 7619 "%s: giving up waiting for link state change\n", 7620 __func__); 7621 } 7622 goto done; 7623 } 7624 7625 lwde = ppd->link_width_downgrade_enabled; 7626 7627 if (refresh_widths) { 7628 get_link_widths(ppd->dd, &tx, &rx); 7629 ppd->link_width_downgrade_tx_active = tx; 7630 ppd->link_width_downgrade_rx_active = rx; 7631 } 7632 7633 if (ppd->link_width_downgrade_tx_active == 0 || 7634 ppd->link_width_downgrade_rx_active == 0) { 7635 /* the 8051 reported a dead link as a downgrade */ 7636 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n"); 7637 link_downgraded = false; 7638 } else if (lwde == 0) { 7639 /* downgrade is disabled */ 7640 7641 /* bounce if not at starting active width */ 7642 if ((ppd->link_width_active != 7643 ppd->link_width_downgrade_tx_active) || 7644 (ppd->link_width_active != 7645 ppd->link_width_downgrade_rx_active)) { 7646 dd_dev_err(ppd->dd, 7647 "Link downgrade is disabled and link has downgraded, downing link\n"); 7648 dd_dev_err(ppd->dd, 7649 " original 0x%x, tx active 0x%x, rx active 0x%x\n", 7650 ppd->link_width_active, 7651 ppd->link_width_downgrade_tx_active, 7652 ppd->link_width_downgrade_rx_active); 7653 do_bounce = 1; 7654 link_downgraded = false; 7655 } 7656 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 || 7657 (lwde & ppd->link_width_downgrade_rx_active) == 0) { 7658 /* Tx or Rx is outside the enabled policy */ 7659 dd_dev_err(ppd->dd, 7660 "Link is outside of downgrade allowed, downing link\n"); 7661 dd_dev_err(ppd->dd, 7662 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n", 7663 lwde, ppd->link_width_downgrade_tx_active, 7664 ppd->link_width_downgrade_rx_active); 7665 do_bounce = 1; 7666 link_downgraded = false; 7667 } 7668 7669 done: 7670 mutex_unlock(&ppd->hls_lock); 7671 7672 if (do_bounce) { 7673 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0, 7674 OPA_LINKDOWN_REASON_WIDTH_POLICY); 7675 set_link_state(ppd, HLS_DN_OFFLINE); 7676 start_link(ppd); 7677 } 7678 7679 return link_downgraded; 7680 } 7681 7682 /* 7683 * Handle a link downgrade interrupt from the 8051. 7684 * 7685 * This is a work-queue function outside of the interrupt. 7686 */ 7687 void handle_link_downgrade(struct work_struct *work) 7688 { 7689 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7690 link_downgrade_work); 7691 7692 dd_dev_info(ppd->dd, "8051: Link width downgrade\n"); 7693 if (apply_link_downgrade_policy(ppd, true)) 7694 update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active); 7695 } 7696 7697 static char *dcc_err_string(char *buf, int buf_len, u64 flags) 7698 { 7699 return flag_string(buf, buf_len, flags, dcc_err_flags, 7700 ARRAY_SIZE(dcc_err_flags)); 7701 } 7702 7703 static char *lcb_err_string(char *buf, int buf_len, u64 flags) 7704 { 7705 return flag_string(buf, buf_len, flags, lcb_err_flags, 7706 ARRAY_SIZE(lcb_err_flags)); 7707 } 7708 7709 static char *dc8051_err_string(char *buf, int buf_len, u64 flags) 7710 { 7711 return flag_string(buf, buf_len, flags, dc8051_err_flags, 7712 ARRAY_SIZE(dc8051_err_flags)); 7713 } 7714 7715 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags) 7716 { 7717 return flag_string(buf, buf_len, flags, dc8051_info_err_flags, 7718 ARRAY_SIZE(dc8051_info_err_flags)); 7719 } 7720 7721 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags) 7722 { 7723 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags, 7724 ARRAY_SIZE(dc8051_info_host_msg_flags)); 7725 } 7726 7727 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg) 7728 { 7729 struct hfi1_pportdata *ppd = dd->pport; 7730 u64 info, err, host_msg; 7731 int queue_link_down = 0; 7732 char buf[96]; 7733 7734 /* look at the flags */ 7735 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) { 7736 /* 8051 information set by firmware */ 7737 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */ 7738 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051); 7739 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT) 7740 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK; 7741 host_msg = (info >> 7742 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT) 7743 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK; 7744 7745 /* 7746 * Handle error flags. 7747 */ 7748 if (err & FAILED_LNI) { 7749 /* 7750 * LNI error indications are cleared by the 8051 7751 * only when starting polling. Only pay attention 7752 * to them when in the states that occur during 7753 * LNI. 7754 */ 7755 if (ppd->host_link_state 7756 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 7757 queue_link_down = 1; 7758 dd_dev_info(dd, "Link error: %s\n", 7759 dc8051_info_err_string(buf, 7760 sizeof(buf), 7761 err & 7762 FAILED_LNI)); 7763 } 7764 err &= ~(u64)FAILED_LNI; 7765 } 7766 /* unknown frames can happen durning LNI, just count */ 7767 if (err & UNKNOWN_FRAME) { 7768 ppd->unknown_frame_count++; 7769 err &= ~(u64)UNKNOWN_FRAME; 7770 } 7771 if (err) { 7772 /* report remaining errors, but do not do anything */ 7773 dd_dev_err(dd, "8051 info error: %s\n", 7774 dc8051_info_err_string(buf, sizeof(buf), 7775 err)); 7776 } 7777 7778 /* 7779 * Handle host message flags. 7780 */ 7781 if (host_msg & HOST_REQ_DONE) { 7782 /* 7783 * Presently, the driver does a busy wait for 7784 * host requests to complete. This is only an 7785 * informational message. 7786 * NOTE: The 8051 clears the host message 7787 * information *on the next 8051 command*. 7788 * Therefore, when linkup is achieved, 7789 * this flag will still be set. 7790 */ 7791 host_msg &= ~(u64)HOST_REQ_DONE; 7792 } 7793 if (host_msg & BC_SMA_MSG) { 7794 queue_work(ppd->link_wq, &ppd->sma_message_work); 7795 host_msg &= ~(u64)BC_SMA_MSG; 7796 } 7797 if (host_msg & LINKUP_ACHIEVED) { 7798 dd_dev_info(dd, "8051: Link up\n"); 7799 queue_work(ppd->link_wq, &ppd->link_up_work); 7800 host_msg &= ~(u64)LINKUP_ACHIEVED; 7801 } 7802 if (host_msg & EXT_DEVICE_CFG_REQ) { 7803 handle_8051_request(ppd); 7804 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ; 7805 } 7806 if (host_msg & VERIFY_CAP_FRAME) { 7807 queue_work(ppd->link_wq, &ppd->link_vc_work); 7808 host_msg &= ~(u64)VERIFY_CAP_FRAME; 7809 } 7810 if (host_msg & LINK_GOING_DOWN) { 7811 const char *extra = ""; 7812 /* no downgrade action needed if going down */ 7813 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7814 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7815 extra = " (ignoring downgrade)"; 7816 } 7817 dd_dev_info(dd, "8051: Link down%s\n", extra); 7818 queue_link_down = 1; 7819 host_msg &= ~(u64)LINK_GOING_DOWN; 7820 } 7821 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7822 queue_work(ppd->link_wq, &ppd->link_downgrade_work); 7823 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7824 } 7825 if (host_msg) { 7826 /* report remaining messages, but do not do anything */ 7827 dd_dev_info(dd, "8051 info host message: %s\n", 7828 dc8051_info_host_msg_string(buf, 7829 sizeof(buf), 7830 host_msg)); 7831 } 7832 7833 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK; 7834 } 7835 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) { 7836 /* 7837 * Lost the 8051 heartbeat. If this happens, we 7838 * receive constant interrupts about it. Disable 7839 * the interrupt after the first. 7840 */ 7841 dd_dev_err(dd, "Lost 8051 heartbeat\n"); 7842 write_csr(dd, DC_DC8051_ERR_EN, 7843 read_csr(dd, DC_DC8051_ERR_EN) & 7844 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK); 7845 7846 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK; 7847 } 7848 if (reg) { 7849 /* report the error, but do not do anything */ 7850 dd_dev_err(dd, "8051 error: %s\n", 7851 dc8051_err_string(buf, sizeof(buf), reg)); 7852 } 7853 7854 if (queue_link_down) { 7855 /* 7856 * if the link is already going down or disabled, do not 7857 * queue another. If there's a link down entry already 7858 * queued, don't queue another one. 7859 */ 7860 if ((ppd->host_link_state & 7861 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) || 7862 ppd->link_enabled == 0) { 7863 dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n", 7864 __func__, ppd->host_link_state, 7865 ppd->link_enabled); 7866 } else { 7867 if (xchg(&ppd->is_link_down_queued, 1) == 1) 7868 dd_dev_info(dd, 7869 "%s: link down request already queued\n", 7870 __func__); 7871 else 7872 queue_work(ppd->link_wq, &ppd->link_down_work); 7873 } 7874 } 7875 } 7876 7877 static const char * const fm_config_txt[] = { 7878 [0] = 7879 "BadHeadDist: Distance violation between two head flits", 7880 [1] = 7881 "BadTailDist: Distance violation between two tail flits", 7882 [2] = 7883 "BadCtrlDist: Distance violation between two credit control flits", 7884 [3] = 7885 "BadCrdAck: Credits return for unsupported VL", 7886 [4] = 7887 "UnsupportedVLMarker: Received VL Marker", 7888 [5] = 7889 "BadPreempt: Exceeded the preemption nesting level", 7890 [6] = 7891 "BadControlFlit: Received unsupported control flit", 7892 /* no 7 */ 7893 [8] = 7894 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL", 7895 }; 7896 7897 static const char * const port_rcv_txt[] = { 7898 [1] = 7899 "BadPktLen: Illegal PktLen", 7900 [2] = 7901 "PktLenTooLong: Packet longer than PktLen", 7902 [3] = 7903 "PktLenTooShort: Packet shorter than PktLen", 7904 [4] = 7905 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)", 7906 [5] = 7907 "BadDLID: Illegal DLID (0, doesn't match HFI)", 7908 [6] = 7909 "BadL2: Illegal L2 opcode", 7910 [7] = 7911 "BadSC: Unsupported SC", 7912 [9] = 7913 "BadRC: Illegal RC", 7914 [11] = 7915 "PreemptError: Preempting with same VL", 7916 [12] = 7917 "PreemptVL15: Preempting a VL15 packet", 7918 }; 7919 7920 #define OPA_LDR_FMCONFIG_OFFSET 16 7921 #define OPA_LDR_PORTRCV_OFFSET 0 7922 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 7923 { 7924 u64 info, hdr0, hdr1; 7925 const char *extra; 7926 char buf[96]; 7927 struct hfi1_pportdata *ppd = dd->pport; 7928 u8 lcl_reason = 0; 7929 int do_bounce = 0; 7930 7931 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) { 7932 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) { 7933 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE); 7934 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK; 7935 /* set status bit */ 7936 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK; 7937 } 7938 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK; 7939 } 7940 7941 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) { 7942 struct hfi1_pportdata *ppd = dd->pport; 7943 /* this counter saturates at (2^32) - 1 */ 7944 if (ppd->link_downed < (u32)UINT_MAX) 7945 ppd->link_downed++; 7946 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK; 7947 } 7948 7949 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) { 7950 u8 reason_valid = 1; 7951 7952 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG); 7953 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) { 7954 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK; 7955 /* set status bit */ 7956 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK; 7957 } 7958 switch (info) { 7959 case 0: 7960 case 1: 7961 case 2: 7962 case 3: 7963 case 4: 7964 case 5: 7965 case 6: 7966 extra = fm_config_txt[info]; 7967 break; 7968 case 8: 7969 extra = fm_config_txt[info]; 7970 if (ppd->port_error_action & 7971 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) { 7972 do_bounce = 1; 7973 /* 7974 * lcl_reason cannot be derived from info 7975 * for this error 7976 */ 7977 lcl_reason = 7978 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER; 7979 } 7980 break; 7981 default: 7982 reason_valid = 0; 7983 snprintf(buf, sizeof(buf), "reserved%lld", info); 7984 extra = buf; 7985 break; 7986 } 7987 7988 if (reason_valid && !do_bounce) { 7989 do_bounce = ppd->port_error_action & 7990 (1 << (OPA_LDR_FMCONFIG_OFFSET + info)); 7991 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST; 7992 } 7993 7994 /* just report this */ 7995 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n", 7996 extra); 7997 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK; 7998 } 7999 8000 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) { 8001 u8 reason_valid = 1; 8002 8003 info = read_csr(dd, DCC_ERR_INFO_PORTRCV); 8004 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0); 8005 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1); 8006 if (!(dd->err_info_rcvport.status_and_code & 8007 OPA_EI_STATUS_SMASK)) { 8008 dd->err_info_rcvport.status_and_code = 8009 info & OPA_EI_CODE_SMASK; 8010 /* set status bit */ 8011 dd->err_info_rcvport.status_and_code |= 8012 OPA_EI_STATUS_SMASK; 8013 /* 8014 * save first 2 flits in the packet that caused 8015 * the error 8016 */ 8017 dd->err_info_rcvport.packet_flit1 = hdr0; 8018 dd->err_info_rcvport.packet_flit2 = hdr1; 8019 } 8020 switch (info) { 8021 case 1: 8022 case 2: 8023 case 3: 8024 case 4: 8025 case 5: 8026 case 6: 8027 case 7: 8028 case 9: 8029 case 11: 8030 case 12: 8031 extra = port_rcv_txt[info]; 8032 break; 8033 default: 8034 reason_valid = 0; 8035 snprintf(buf, sizeof(buf), "reserved%lld", info); 8036 extra = buf; 8037 break; 8038 } 8039 8040 if (reason_valid && !do_bounce) { 8041 do_bounce = ppd->port_error_action & 8042 (1 << (OPA_LDR_PORTRCV_OFFSET + info)); 8043 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0; 8044 } 8045 8046 /* just report this */ 8047 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n" 8048 " hdr0 0x%llx, hdr1 0x%llx\n", 8049 extra, hdr0, hdr1); 8050 8051 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK; 8052 } 8053 8054 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) { 8055 /* informative only */ 8056 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n"); 8057 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK; 8058 } 8059 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) { 8060 /* informative only */ 8061 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n"); 8062 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK; 8063 } 8064 8065 if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev))) 8066 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK; 8067 8068 /* report any remaining errors */ 8069 if (reg) 8070 dd_dev_info_ratelimited(dd, "DCC Error: %s\n", 8071 dcc_err_string(buf, sizeof(buf), reg)); 8072 8073 if (lcl_reason == 0) 8074 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN; 8075 8076 if (do_bounce) { 8077 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n", 8078 __func__); 8079 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason); 8080 queue_work(ppd->link_wq, &ppd->link_bounce_work); 8081 } 8082 } 8083 8084 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 8085 { 8086 char buf[96]; 8087 8088 dd_dev_info(dd, "LCB Error: %s\n", 8089 lcb_err_string(buf, sizeof(buf), reg)); 8090 } 8091 8092 /* 8093 * CCE block DC interrupt. Source is < 8. 8094 */ 8095 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source) 8096 { 8097 const struct err_reg_info *eri = &dc_errs[source]; 8098 8099 if (eri->handler) { 8100 interrupt_clear_down(dd, 0, eri); 8101 } else if (source == 3 /* dc_lbm_int */) { 8102 /* 8103 * This indicates that a parity error has occurred on the 8104 * address/control lines presented to the LBM. The error 8105 * is a single pulse, there is no associated error flag, 8106 * and it is non-maskable. This is because if a parity 8107 * error occurs on the request the request is dropped. 8108 * This should never occur, but it is nice to know if it 8109 * ever does. 8110 */ 8111 dd_dev_err(dd, "Parity error in DC LBM block\n"); 8112 } else { 8113 dd_dev_err(dd, "Invalid DC interrupt %u\n", source); 8114 } 8115 } 8116 8117 /* 8118 * TX block send credit interrupt. Source is < 160. 8119 */ 8120 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source) 8121 { 8122 sc_group_release_update(dd, source); 8123 } 8124 8125 /* 8126 * TX block SDMA interrupt. Source is < 48. 8127 * 8128 * SDMA interrupts are grouped by type: 8129 * 8130 * 0 - N-1 = SDma 8131 * N - 2N-1 = SDmaProgress 8132 * 2N - 3N-1 = SDmaIdle 8133 */ 8134 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source) 8135 { 8136 /* what interrupt */ 8137 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 8138 /* which engine */ 8139 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 8140 8141 #ifdef CONFIG_SDMA_VERBOSITY 8142 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which, 8143 slashstrip(__FILE__), __LINE__, __func__); 8144 sdma_dumpstate(&dd->per_sdma[which]); 8145 #endif 8146 8147 if (likely(what < 3 && which < dd->num_sdma)) { 8148 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source); 8149 } else { 8150 /* should not happen */ 8151 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source); 8152 } 8153 } 8154 8155 /** 8156 * is_rcv_avail_int() - User receive context available IRQ handler 8157 * @dd: valid dd 8158 * @source: logical IRQ source (offset from IS_RCVAVAIL_START) 8159 * 8160 * RX block receive available interrupt. Source is < 160. 8161 * 8162 * This is the general interrupt handler for user (PSM) receive contexts, 8163 * and can only be used for non-threaded IRQs. 8164 */ 8165 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source) 8166 { 8167 struct hfi1_ctxtdata *rcd; 8168 char *err_detail; 8169 8170 if (likely(source < dd->num_rcv_contexts)) { 8171 rcd = hfi1_rcd_get_by_index(dd, source); 8172 if (rcd) { 8173 handle_user_interrupt(rcd); 8174 hfi1_rcd_put(rcd); 8175 return; /* OK */ 8176 } 8177 /* received an interrupt, but no rcd */ 8178 err_detail = "dataless"; 8179 } else { 8180 /* received an interrupt, but are not using that context */ 8181 err_detail = "out of range"; 8182 } 8183 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n", 8184 err_detail, source); 8185 } 8186 8187 /** 8188 * is_rcv_urgent_int() - User receive context urgent IRQ handler 8189 * @dd: valid dd 8190 * @source: logical IRQ source (offset from IS_RCVURGENT_START) 8191 * 8192 * RX block receive urgent interrupt. Source is < 160. 8193 * 8194 * NOTE: kernel receive contexts specifically do NOT enable this IRQ. 8195 */ 8196 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source) 8197 { 8198 struct hfi1_ctxtdata *rcd; 8199 char *err_detail; 8200 8201 if (likely(source < dd->num_rcv_contexts)) { 8202 rcd = hfi1_rcd_get_by_index(dd, source); 8203 if (rcd) { 8204 handle_user_interrupt(rcd); 8205 hfi1_rcd_put(rcd); 8206 return; /* OK */ 8207 } 8208 /* received an interrupt, but no rcd */ 8209 err_detail = "dataless"; 8210 } else { 8211 /* received an interrupt, but are not using that context */ 8212 err_detail = "out of range"; 8213 } 8214 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n", 8215 err_detail, source); 8216 } 8217 8218 /* 8219 * Reserved range interrupt. Should not be called in normal operation. 8220 */ 8221 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source) 8222 { 8223 char name[64]; 8224 8225 dd_dev_err(dd, "unexpected %s interrupt\n", 8226 is_reserved_name(name, sizeof(name), source)); 8227 } 8228 8229 static const struct is_table is_table[] = { 8230 /* 8231 * start end 8232 * name func interrupt func 8233 */ 8234 { IS_GENERAL_ERR_START, IS_GENERAL_ERR_END, 8235 is_misc_err_name, is_misc_err_int }, 8236 { IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END, 8237 is_sdma_eng_err_name, is_sdma_eng_err_int }, 8238 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END, 8239 is_sendctxt_err_name, is_sendctxt_err_int }, 8240 { IS_SDMA_START, IS_SDMA_IDLE_END, 8241 is_sdma_eng_name, is_sdma_eng_int }, 8242 { IS_VARIOUS_START, IS_VARIOUS_END, 8243 is_various_name, is_various_int }, 8244 { IS_DC_START, IS_DC_END, 8245 is_dc_name, is_dc_int }, 8246 { IS_RCVAVAIL_START, IS_RCVAVAIL_END, 8247 is_rcv_avail_name, is_rcv_avail_int }, 8248 { IS_RCVURGENT_START, IS_RCVURGENT_END, 8249 is_rcv_urgent_name, is_rcv_urgent_int }, 8250 { IS_SENDCREDIT_START, IS_SENDCREDIT_END, 8251 is_send_credit_name, is_send_credit_int}, 8252 { IS_RESERVED_START, IS_RESERVED_END, 8253 is_reserved_name, is_reserved_int}, 8254 }; 8255 8256 /* 8257 * Interrupt source interrupt - called when the given source has an interrupt. 8258 * Source is a bit index into an array of 64-bit integers. 8259 */ 8260 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source) 8261 { 8262 const struct is_table *entry; 8263 8264 /* avoids a double compare by walking the table in-order */ 8265 for (entry = &is_table[0]; entry->is_name; entry++) { 8266 if (source <= entry->end) { 8267 trace_hfi1_interrupt(dd, entry, source); 8268 entry->is_int(dd, source - entry->start); 8269 return; 8270 } 8271 } 8272 /* fell off the end */ 8273 dd_dev_err(dd, "invalid interrupt source %u\n", source); 8274 } 8275 8276 /** 8277 * general_interrupt - General interrupt handler 8278 * @irq: MSIx IRQ vector 8279 * @data: hfi1 devdata 8280 * 8281 * This is able to correctly handle all non-threaded interrupts. Receive 8282 * context DATA IRQs are threaded and are not supported by this handler. 8283 * 8284 */ 8285 irqreturn_t general_interrupt(int irq, void *data) 8286 { 8287 struct hfi1_devdata *dd = data; 8288 u64 regs[CCE_NUM_INT_CSRS]; 8289 u32 bit; 8290 int i; 8291 irqreturn_t handled = IRQ_NONE; 8292 8293 this_cpu_inc(*dd->int_counter); 8294 8295 /* phase 1: scan and clear all handled interrupts */ 8296 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 8297 if (dd->gi_mask[i] == 0) { 8298 regs[i] = 0; /* used later */ 8299 continue; 8300 } 8301 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) & 8302 dd->gi_mask[i]; 8303 /* only clear if anything is set */ 8304 if (regs[i]) 8305 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]); 8306 } 8307 8308 /* phase 2: call the appropriate handler */ 8309 for_each_set_bit(bit, (unsigned long *)®s[0], 8310 CCE_NUM_INT_CSRS * 64) { 8311 is_interrupt(dd, bit); 8312 handled = IRQ_HANDLED; 8313 } 8314 8315 return handled; 8316 } 8317 8318 irqreturn_t sdma_interrupt(int irq, void *data) 8319 { 8320 struct sdma_engine *sde = data; 8321 struct hfi1_devdata *dd = sde->dd; 8322 u64 status; 8323 8324 #ifdef CONFIG_SDMA_VERBOSITY 8325 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 8326 slashstrip(__FILE__), __LINE__, __func__); 8327 sdma_dumpstate(sde); 8328 #endif 8329 8330 this_cpu_inc(*dd->int_counter); 8331 8332 /* This read_csr is really bad in the hot path */ 8333 status = read_csr(dd, 8334 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64))) 8335 & sde->imask; 8336 if (likely(status)) { 8337 /* clear the interrupt(s) */ 8338 write_csr(dd, 8339 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)), 8340 status); 8341 8342 /* handle the interrupt(s) */ 8343 sdma_engine_interrupt(sde, status); 8344 } else { 8345 dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", 8346 sde->this_idx); 8347 } 8348 return IRQ_HANDLED; 8349 } 8350 8351 /* 8352 * Clear the receive interrupt. Use a read of the interrupt clear CSR 8353 * to insure that the write completed. This does NOT guarantee that 8354 * queued DMA writes to memory from the chip are pushed. 8355 */ 8356 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd) 8357 { 8358 struct hfi1_devdata *dd = rcd->dd; 8359 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg); 8360 8361 write_csr(dd, addr, rcd->imask); 8362 /* force the above write on the chip and get a value back */ 8363 (void)read_csr(dd, addr); 8364 } 8365 8366 /* force the receive interrupt */ 8367 void force_recv_intr(struct hfi1_ctxtdata *rcd) 8368 { 8369 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask); 8370 } 8371 8372 /* 8373 * Return non-zero if a packet is present. 8374 * 8375 * This routine is called when rechecking for packets after the RcvAvail 8376 * interrupt has been cleared down. First, do a quick check of memory for 8377 * a packet present. If not found, use an expensive CSR read of the context 8378 * tail to determine the actual tail. The CSR read is necessary because there 8379 * is no method to push pending DMAs to memory other than an interrupt and we 8380 * are trying to determine if we need to force an interrupt. 8381 */ 8382 static inline int check_packet_present(struct hfi1_ctxtdata *rcd) 8383 { 8384 u32 tail; 8385 8386 if (hfi1_packet_present(rcd)) 8387 return 1; 8388 8389 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */ 8390 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 8391 return hfi1_rcd_head(rcd) != tail; 8392 } 8393 8394 /* 8395 * Common code for receive contexts interrupt handlers. 8396 * Update traces, increment kernel IRQ counter and 8397 * setup ASPM when needed. 8398 */ 8399 static void receive_interrupt_common(struct hfi1_ctxtdata *rcd) 8400 { 8401 struct hfi1_devdata *dd = rcd->dd; 8402 8403 trace_hfi1_receive_interrupt(dd, rcd); 8404 this_cpu_inc(*dd->int_counter); 8405 aspm_ctx_disable(rcd); 8406 } 8407 8408 /* 8409 * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt 8410 * when there are packets present in the queue. When calling 8411 * with interrupts enabled please use hfi1_rcd_eoi_intr. 8412 * 8413 * @rcd: valid receive context 8414 */ 8415 static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) 8416 { 8417 clear_recv_intr(rcd); 8418 if (check_packet_present(rcd)) 8419 force_recv_intr(rcd); 8420 } 8421 8422 /** 8423 * hfi1_rcd_eoi_intr() - End of Interrupt processing action 8424 * 8425 * @rcd: Ptr to hfi1_ctxtdata of receive context 8426 * 8427 * Hold IRQs so we can safely clear the interrupt and 8428 * recheck for a packet that may have arrived after the previous 8429 * check and the interrupt clear. If a packet arrived, force another 8430 * interrupt. This routine can be called at the end of receive packet 8431 * processing in interrupt service routines, interrupt service thread 8432 * and softirqs 8433 */ 8434 static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) 8435 { 8436 unsigned long flags; 8437 8438 local_irq_save(flags); 8439 __hfi1_rcd_eoi_intr(rcd); 8440 local_irq_restore(flags); 8441 } 8442 8443 /** 8444 * hfi1_netdev_rx_napi - napi poll function to move eoi inline 8445 * @napi: pointer to napi object 8446 * @budget: netdev budget 8447 */ 8448 int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget) 8449 { 8450 struct hfi1_netdev_rxq *rxq = container_of(napi, 8451 struct hfi1_netdev_rxq, napi); 8452 struct hfi1_ctxtdata *rcd = rxq->rcd; 8453 int work_done = 0; 8454 8455 work_done = rcd->do_interrupt(rcd, budget); 8456 8457 if (work_done < budget) { 8458 napi_complete_done(napi, work_done); 8459 hfi1_rcd_eoi_intr(rcd); 8460 } 8461 8462 return work_done; 8463 } 8464 8465 /* Receive packet napi handler for netdevs VNIC and AIP */ 8466 irqreturn_t receive_context_interrupt_napi(int irq, void *data) 8467 { 8468 struct hfi1_ctxtdata *rcd = data; 8469 8470 receive_interrupt_common(rcd); 8471 8472 if (likely(rcd->napi)) { 8473 if (likely(napi_schedule_prep(rcd->napi))) 8474 __napi_schedule_irqoff(rcd->napi); 8475 else 8476 __hfi1_rcd_eoi_intr(rcd); 8477 } else { 8478 WARN_ONCE(1, "Napi IRQ handler without napi set up ctxt=%d\n", 8479 rcd->ctxt); 8480 __hfi1_rcd_eoi_intr(rcd); 8481 } 8482 8483 return IRQ_HANDLED; 8484 } 8485 8486 /* 8487 * Receive packet IRQ handler. This routine expects to be on its own IRQ. 8488 * This routine will try to handle packets immediately (latency), but if 8489 * it finds too many, it will invoke the thread handler (bandwitdh). The 8490 * chip receive interrupt is *not* cleared down until this or the thread (if 8491 * invoked) is finished. The intent is to avoid extra interrupts while we 8492 * are processing packets anyway. 8493 */ 8494 irqreturn_t receive_context_interrupt(int irq, void *data) 8495 { 8496 struct hfi1_ctxtdata *rcd = data; 8497 int disposition; 8498 8499 receive_interrupt_common(rcd); 8500 8501 /* receive interrupt remains blocked while processing packets */ 8502 disposition = rcd->do_interrupt(rcd, 0); 8503 8504 /* 8505 * Too many packets were seen while processing packets in this 8506 * IRQ handler. Invoke the handler thread. The receive interrupt 8507 * remains blocked. 8508 */ 8509 if (disposition == RCV_PKT_LIMIT) 8510 return IRQ_WAKE_THREAD; 8511 8512 __hfi1_rcd_eoi_intr(rcd); 8513 return IRQ_HANDLED; 8514 } 8515 8516 /* 8517 * Receive packet thread handler. This expects to be invoked with the 8518 * receive interrupt still blocked. 8519 */ 8520 irqreturn_t receive_context_thread(int irq, void *data) 8521 { 8522 struct hfi1_ctxtdata *rcd = data; 8523 8524 /* receive interrupt is still blocked from the IRQ handler */ 8525 (void)rcd->do_interrupt(rcd, 1); 8526 8527 hfi1_rcd_eoi_intr(rcd); 8528 8529 return IRQ_HANDLED; 8530 } 8531 8532 /* ========================================================================= */ 8533 8534 u32 read_physical_state(struct hfi1_devdata *dd) 8535 { 8536 u64 reg; 8537 8538 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE); 8539 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT) 8540 & DC_DC8051_STS_CUR_STATE_PORT_MASK; 8541 } 8542 8543 u32 read_logical_state(struct hfi1_devdata *dd) 8544 { 8545 u64 reg; 8546 8547 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8548 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT) 8549 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK; 8550 } 8551 8552 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate) 8553 { 8554 u64 reg; 8555 8556 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8557 /* clear current state, set new state */ 8558 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK; 8559 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT; 8560 write_csr(dd, DCC_CFG_PORT_CONFIG, reg); 8561 } 8562 8563 /* 8564 * Use the 8051 to read a LCB CSR. 8565 */ 8566 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data) 8567 { 8568 u32 regno; 8569 int ret; 8570 8571 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 8572 if (acquire_lcb_access(dd, 0) == 0) { 8573 *data = read_csr(dd, addr); 8574 release_lcb_access(dd, 0); 8575 return 0; 8576 } 8577 return -EBUSY; 8578 } 8579 8580 /* register is an index of LCB registers: (offset - base) / 8 */ 8581 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8582 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data); 8583 if (ret != HCMD_SUCCESS) 8584 return -EBUSY; 8585 return 0; 8586 } 8587 8588 /* 8589 * Provide a cache for some of the LCB registers in case the LCB is 8590 * unavailable. 8591 * (The LCB is unavailable in certain link states, for example.) 8592 */ 8593 struct lcb_datum { 8594 u32 off; 8595 u64 val; 8596 }; 8597 8598 static struct lcb_datum lcb_cache[] = { 8599 { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0}, 8600 { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 }, 8601 { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 }, 8602 }; 8603 8604 static void update_lcb_cache(struct hfi1_devdata *dd) 8605 { 8606 int i; 8607 int ret; 8608 u64 val; 8609 8610 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) { 8611 ret = read_lcb_csr(dd, lcb_cache[i].off, &val); 8612 8613 /* Update if we get good data */ 8614 if (likely(ret != -EBUSY)) 8615 lcb_cache[i].val = val; 8616 } 8617 } 8618 8619 static int read_lcb_cache(u32 off, u64 *val) 8620 { 8621 int i; 8622 8623 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) { 8624 if (lcb_cache[i].off == off) { 8625 *val = lcb_cache[i].val; 8626 return 0; 8627 } 8628 } 8629 8630 pr_warn("%s bad offset 0x%x\n", __func__, off); 8631 return -1; 8632 } 8633 8634 /* 8635 * Read an LCB CSR. Access may not be in host control, so check. 8636 * Return 0 on success, -EBUSY on failure. 8637 */ 8638 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data) 8639 { 8640 struct hfi1_pportdata *ppd = dd->pport; 8641 8642 /* if up, go through the 8051 for the value */ 8643 if (ppd->host_link_state & HLS_UP) 8644 return read_lcb_via_8051(dd, addr, data); 8645 /* if going up or down, check the cache, otherwise, no access */ 8646 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) { 8647 if (read_lcb_cache(addr, data)) 8648 return -EBUSY; 8649 return 0; 8650 } 8651 8652 /* otherwise, host has access */ 8653 *data = read_csr(dd, addr); 8654 return 0; 8655 } 8656 8657 /* 8658 * Use the 8051 to write a LCB CSR. 8659 */ 8660 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data) 8661 { 8662 u32 regno; 8663 int ret; 8664 8665 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || 8666 (dd->dc8051_ver < dc8051_ver(0, 20, 0))) { 8667 if (acquire_lcb_access(dd, 0) == 0) { 8668 write_csr(dd, addr, data); 8669 release_lcb_access(dd, 0); 8670 return 0; 8671 } 8672 return -EBUSY; 8673 } 8674 8675 /* register is an index of LCB registers: (offset - base) / 8 */ 8676 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8677 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data); 8678 if (ret != HCMD_SUCCESS) 8679 return -EBUSY; 8680 return 0; 8681 } 8682 8683 /* 8684 * Write an LCB CSR. Access may not be in host control, so check. 8685 * Return 0 on success, -EBUSY on failure. 8686 */ 8687 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data) 8688 { 8689 struct hfi1_pportdata *ppd = dd->pport; 8690 8691 /* if up, go through the 8051 for the value */ 8692 if (ppd->host_link_state & HLS_UP) 8693 return write_lcb_via_8051(dd, addr, data); 8694 /* if going up or down, no access */ 8695 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) 8696 return -EBUSY; 8697 /* otherwise, host has access */ 8698 write_csr(dd, addr, data); 8699 return 0; 8700 } 8701 8702 /* 8703 * Returns: 8704 * < 0 = Linux error, not able to get access 8705 * > 0 = 8051 command RETURN_CODE 8706 */ 8707 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 8708 u64 *out_data) 8709 { 8710 u64 reg, completed; 8711 int return_code; 8712 unsigned long timeout; 8713 8714 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data); 8715 8716 mutex_lock(&dd->dc8051_lock); 8717 8718 /* We can't send any commands to the 8051 if it's in reset */ 8719 if (dd->dc_shutdown) { 8720 return_code = -ENODEV; 8721 goto fail; 8722 } 8723 8724 /* 8725 * If an 8051 host command timed out previously, then the 8051 is 8726 * stuck. 8727 * 8728 * On first timeout, attempt to reset and restart the entire DC 8729 * block (including 8051). (Is this too big of a hammer?) 8730 * 8731 * If the 8051 times out a second time, the reset did not bring it 8732 * back to healthy life. In that case, fail any subsequent commands. 8733 */ 8734 if (dd->dc8051_timed_out) { 8735 if (dd->dc8051_timed_out > 1) { 8736 dd_dev_err(dd, 8737 "Previous 8051 host command timed out, skipping command %u\n", 8738 type); 8739 return_code = -ENXIO; 8740 goto fail; 8741 } 8742 _dc_shutdown(dd); 8743 _dc_start(dd); 8744 } 8745 8746 /* 8747 * If there is no timeout, then the 8051 command interface is 8748 * waiting for a command. 8749 */ 8750 8751 /* 8752 * When writing a LCB CSR, out_data contains the full value to 8753 * to be written, while in_data contains the relative LCB 8754 * address in 7:0. Do the work here, rather than the caller, 8755 * of distrubting the write data to where it needs to go: 8756 * 8757 * Write data 8758 * 39:00 -> in_data[47:8] 8759 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE 8760 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA 8761 */ 8762 if (type == HCMD_WRITE_LCB_CSR) { 8763 in_data |= ((*out_data) & 0xffffffffffull) << 8; 8764 /* must preserve COMPLETED - it is tied to hardware */ 8765 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0); 8766 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK; 8767 reg |= ((((*out_data) >> 40) & 0xff) << 8768 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT) 8769 | ((((*out_data) >> 48) & 0xffff) << 8770 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 8771 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg); 8772 } 8773 8774 /* 8775 * Do two writes: the first to stabilize the type and req_data, the 8776 * second to activate. 8777 */ 8778 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK) 8779 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT 8780 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK) 8781 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT; 8782 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8783 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK; 8784 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8785 8786 /* wait for completion, alternate: interrupt */ 8787 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT); 8788 while (1) { 8789 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1); 8790 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK; 8791 if (completed) 8792 break; 8793 if (time_after(jiffies, timeout)) { 8794 dd->dc8051_timed_out++; 8795 dd_dev_err(dd, "8051 host command %u timeout\n", type); 8796 if (out_data) 8797 *out_data = 0; 8798 return_code = -ETIMEDOUT; 8799 goto fail; 8800 } 8801 udelay(2); 8802 } 8803 8804 if (out_data) { 8805 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT) 8806 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK; 8807 if (type == HCMD_READ_LCB_CSR) { 8808 /* top 16 bits are in a different register */ 8809 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1) 8810 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK) 8811 << (48 8812 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT); 8813 } 8814 } 8815 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT) 8816 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK; 8817 dd->dc8051_timed_out = 0; 8818 /* 8819 * Clear command for next user. 8820 */ 8821 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0); 8822 8823 fail: 8824 mutex_unlock(&dd->dc8051_lock); 8825 return return_code; 8826 } 8827 8828 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state) 8829 { 8830 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL); 8831 } 8832 8833 int load_8051_config(struct hfi1_devdata *dd, u8 field_id, 8834 u8 lane_id, u32 config_data) 8835 { 8836 u64 data; 8837 int ret; 8838 8839 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT 8840 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT 8841 | (u64)config_data << LOAD_DATA_DATA_SHIFT; 8842 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL); 8843 if (ret != HCMD_SUCCESS) { 8844 dd_dev_err(dd, 8845 "load 8051 config: field id %d, lane %d, err %d\n", 8846 (int)field_id, (int)lane_id, ret); 8847 } 8848 return ret; 8849 } 8850 8851 /* 8852 * Read the 8051 firmware "registers". Use the RAM directly. Always 8853 * set the result, even on error. 8854 * Return 0 on success, -errno on failure 8855 */ 8856 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id, 8857 u32 *result) 8858 { 8859 u64 big_data; 8860 u32 addr; 8861 int ret; 8862 8863 /* address start depends on the lane_id */ 8864 if (lane_id < 4) 8865 addr = (4 * NUM_GENERAL_FIELDS) 8866 + (lane_id * 4 * NUM_LANE_FIELDS); 8867 else 8868 addr = 0; 8869 addr += field_id * 4; 8870 8871 /* read is in 8-byte chunks, hardware will truncate the address down */ 8872 ret = read_8051_data(dd, addr, 8, &big_data); 8873 8874 if (ret == 0) { 8875 /* extract the 4 bytes we want */ 8876 if (addr & 0x4) 8877 *result = (u32)(big_data >> 32); 8878 else 8879 *result = (u32)big_data; 8880 } else { 8881 *result = 0; 8882 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n", 8883 __func__, lane_id, field_id); 8884 } 8885 8886 return ret; 8887 } 8888 8889 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management, 8890 u8 continuous) 8891 { 8892 u32 frame; 8893 8894 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT 8895 | power_management << POWER_MANAGEMENT_SHIFT; 8896 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY, 8897 GENERAL_CONFIG, frame); 8898 } 8899 8900 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu, 8901 u16 vl15buf, u8 crc_sizes) 8902 { 8903 u32 frame; 8904 8905 frame = (u32)vau << VAU_SHIFT 8906 | (u32)z << Z_SHIFT 8907 | (u32)vcu << VCU_SHIFT 8908 | (u32)vl15buf << VL15BUF_SHIFT 8909 | (u32)crc_sizes << CRC_SIZES_SHIFT; 8910 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC, 8911 GENERAL_CONFIG, frame); 8912 } 8913 8914 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits, 8915 u8 *flag_bits, u16 *link_widths) 8916 { 8917 u32 frame; 8918 8919 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG, 8920 &frame); 8921 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK; 8922 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK; 8923 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 8924 } 8925 8926 static int write_vc_local_link_mode(struct hfi1_devdata *dd, 8927 u8 misc_bits, 8928 u8 flag_bits, 8929 u16 link_widths) 8930 { 8931 u32 frame; 8932 8933 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT 8934 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT 8935 | (u32)link_widths << LINK_WIDTH_SHIFT; 8936 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG, 8937 frame); 8938 } 8939 8940 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id, 8941 u8 device_rev) 8942 { 8943 u32 frame; 8944 8945 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT) 8946 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT); 8947 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame); 8948 } 8949 8950 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 8951 u8 *device_rev) 8952 { 8953 u32 frame; 8954 8955 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame); 8956 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK; 8957 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT) 8958 & REMOTE_DEVICE_REV_MASK; 8959 } 8960 8961 int write_host_interface_version(struct hfi1_devdata *dd, u8 version) 8962 { 8963 u32 frame; 8964 u32 mask; 8965 8966 mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT); 8967 read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame); 8968 /* Clear, then set field */ 8969 frame &= ~mask; 8970 frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT); 8971 return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, 8972 frame); 8973 } 8974 8975 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor, 8976 u8 *ver_patch) 8977 { 8978 u32 frame; 8979 8980 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame); 8981 *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) & 8982 STS_FM_VERSION_MAJOR_MASK; 8983 *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) & 8984 STS_FM_VERSION_MINOR_MASK; 8985 8986 read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame); 8987 *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) & 8988 STS_FM_VERSION_PATCH_MASK; 8989 } 8990 8991 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 8992 u8 *continuous) 8993 { 8994 u32 frame; 8995 8996 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame); 8997 *power_management = (frame >> POWER_MANAGEMENT_SHIFT) 8998 & POWER_MANAGEMENT_MASK; 8999 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT) 9000 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK; 9001 } 9002 9003 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 9004 u8 *vcu, u16 *vl15buf, u8 *crc_sizes) 9005 { 9006 u32 frame; 9007 9008 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame); 9009 *vau = (frame >> VAU_SHIFT) & VAU_MASK; 9010 *z = (frame >> Z_SHIFT) & Z_MASK; 9011 *vcu = (frame >> VCU_SHIFT) & VCU_MASK; 9012 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK; 9013 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK; 9014 } 9015 9016 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 9017 u8 *remote_tx_rate, 9018 u16 *link_widths) 9019 { 9020 u32 frame; 9021 9022 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG, 9023 &frame); 9024 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT) 9025 & REMOTE_TX_RATE_MASK; 9026 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 9027 } 9028 9029 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx) 9030 { 9031 u32 frame; 9032 9033 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame); 9034 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK; 9035 } 9036 9037 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls) 9038 { 9039 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls); 9040 } 9041 9042 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs) 9043 { 9044 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs); 9045 } 9046 9047 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality) 9048 { 9049 u32 frame; 9050 int ret; 9051 9052 *link_quality = 0; 9053 if (dd->pport->host_link_state & HLS_UP) { 9054 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, 9055 &frame); 9056 if (ret == 0) 9057 *link_quality = (frame >> LINK_QUALITY_SHIFT) 9058 & LINK_QUALITY_MASK; 9059 } 9060 } 9061 9062 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc) 9063 { 9064 u32 frame; 9065 9066 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame); 9067 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK; 9068 } 9069 9070 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr) 9071 { 9072 u32 frame; 9073 9074 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame); 9075 *ldr = (frame & 0xff); 9076 } 9077 9078 static int read_tx_settings(struct hfi1_devdata *dd, 9079 u8 *enable_lane_tx, 9080 u8 *tx_polarity_inversion, 9081 u8 *rx_polarity_inversion, 9082 u8 *max_rate) 9083 { 9084 u32 frame; 9085 int ret; 9086 9087 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame); 9088 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT) 9089 & ENABLE_LANE_TX_MASK; 9090 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT) 9091 & TX_POLARITY_INVERSION_MASK; 9092 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT) 9093 & RX_POLARITY_INVERSION_MASK; 9094 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK; 9095 return ret; 9096 } 9097 9098 static int write_tx_settings(struct hfi1_devdata *dd, 9099 u8 enable_lane_tx, 9100 u8 tx_polarity_inversion, 9101 u8 rx_polarity_inversion, 9102 u8 max_rate) 9103 { 9104 u32 frame; 9105 9106 /* no need to mask, all variable sizes match field widths */ 9107 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT 9108 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT 9109 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT 9110 | max_rate << MAX_RATE_SHIFT; 9111 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame); 9112 } 9113 9114 /* 9115 * Read an idle LCB message. 9116 * 9117 * Returns 0 on success, -EINVAL on error 9118 */ 9119 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out) 9120 { 9121 int ret; 9122 9123 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out); 9124 if (ret != HCMD_SUCCESS) { 9125 dd_dev_err(dd, "read idle message: type %d, err %d\n", 9126 (u32)type, ret); 9127 return -EINVAL; 9128 } 9129 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out); 9130 /* return only the payload as we already know the type */ 9131 *data_out >>= IDLE_PAYLOAD_SHIFT; 9132 return 0; 9133 } 9134 9135 /* 9136 * Read an idle SMA message. To be done in response to a notification from 9137 * the 8051. 9138 * 9139 * Returns 0 on success, -EINVAL on error 9140 */ 9141 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data) 9142 { 9143 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT, 9144 data); 9145 } 9146 9147 /* 9148 * Send an idle LCB message. 9149 * 9150 * Returns 0 on success, -EINVAL on error 9151 */ 9152 static int send_idle_message(struct hfi1_devdata *dd, u64 data) 9153 { 9154 int ret; 9155 9156 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data); 9157 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL); 9158 if (ret != HCMD_SUCCESS) { 9159 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n", 9160 data, ret); 9161 return -EINVAL; 9162 } 9163 return 0; 9164 } 9165 9166 /* 9167 * Send an idle SMA message. 9168 * 9169 * Returns 0 on success, -EINVAL on error 9170 */ 9171 int send_idle_sma(struct hfi1_devdata *dd, u64 message) 9172 { 9173 u64 data; 9174 9175 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) | 9176 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT); 9177 return send_idle_message(dd, data); 9178 } 9179 9180 /* 9181 * Initialize the LCB then do a quick link up. This may or may not be 9182 * in loopback. 9183 * 9184 * return 0 on success, -errno on error 9185 */ 9186 static int do_quick_linkup(struct hfi1_devdata *dd) 9187 { 9188 int ret; 9189 9190 lcb_shutdown(dd, 0); 9191 9192 if (loopback) { 9193 /* LCB_CFG_LOOPBACK.VAL = 2 */ 9194 /* LCB_CFG_LANE_WIDTH.VAL = 0 */ 9195 write_csr(dd, DC_LCB_CFG_LOOPBACK, 9196 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT); 9197 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 9198 } 9199 9200 /* start the LCBs */ 9201 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */ 9202 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 9203 9204 /* simulator only loopback steps */ 9205 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 9206 /* LCB_CFG_RUN.EN = 1 */ 9207 write_csr(dd, DC_LCB_CFG_RUN, 9208 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 9209 9210 ret = wait_link_transfer_active(dd, 10); 9211 if (ret) 9212 return ret; 9213 9214 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 9215 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT); 9216 } 9217 9218 if (!loopback) { 9219 /* 9220 * When doing quick linkup and not in loopback, both 9221 * sides must be done with LCB set-up before either 9222 * starts the quick linkup. Put a delay here so that 9223 * both sides can be started and have a chance to be 9224 * done with LCB set up before resuming. 9225 */ 9226 dd_dev_err(dd, 9227 "Pausing for peer to be finished with LCB set up\n"); 9228 msleep(5000); 9229 dd_dev_err(dd, "Continuing with quick linkup\n"); 9230 } 9231 9232 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 9233 set_8051_lcb_access(dd); 9234 9235 /* 9236 * State "quick" LinkUp request sets the physical link state to 9237 * LinkUp without a verify capability sequence. 9238 * This state is in simulator v37 and later. 9239 */ 9240 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP); 9241 if (ret != HCMD_SUCCESS) { 9242 dd_dev_err(dd, 9243 "%s: set physical link state to quick LinkUp failed with return %d\n", 9244 __func__, ret); 9245 9246 set_host_lcb_access(dd); 9247 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 9248 9249 if (ret >= 0) 9250 ret = -EINVAL; 9251 return ret; 9252 } 9253 9254 return 0; /* success */ 9255 } 9256 9257 /* 9258 * Do all special steps to set up loopback. 9259 */ 9260 static int init_loopback(struct hfi1_devdata *dd) 9261 { 9262 dd_dev_info(dd, "Entering loopback mode\n"); 9263 9264 /* all loopbacks should disable self GUID check */ 9265 write_csr(dd, DC_DC8051_CFG_MODE, 9266 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK)); 9267 9268 /* 9269 * The simulator has only one loopback option - LCB. Switch 9270 * to that option, which includes quick link up. 9271 * 9272 * Accept all valid loopback values. 9273 */ 9274 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) && 9275 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB || 9276 loopback == LOOPBACK_CABLE)) { 9277 loopback = LOOPBACK_LCB; 9278 quick_linkup = 1; 9279 return 0; 9280 } 9281 9282 /* 9283 * SerDes loopback init sequence is handled in set_local_link_attributes 9284 */ 9285 if (loopback == LOOPBACK_SERDES) 9286 return 0; 9287 9288 /* LCB loopback - handled at poll time */ 9289 if (loopback == LOOPBACK_LCB) { 9290 quick_linkup = 1; /* LCB is always quick linkup */ 9291 9292 /* not supported in emulation due to emulation RTL changes */ 9293 if (dd->icode == ICODE_FPGA_EMULATION) { 9294 dd_dev_err(dd, 9295 "LCB loopback not supported in emulation\n"); 9296 return -EINVAL; 9297 } 9298 return 0; 9299 } 9300 9301 /* external cable loopback requires no extra steps */ 9302 if (loopback == LOOPBACK_CABLE) 9303 return 0; 9304 9305 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback); 9306 return -EINVAL; 9307 } 9308 9309 /* 9310 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits 9311 * used in the Verify Capability link width attribute. 9312 */ 9313 static u16 opa_to_vc_link_widths(u16 opa_widths) 9314 { 9315 int i; 9316 u16 result = 0; 9317 9318 static const struct link_bits { 9319 u16 from; 9320 u16 to; 9321 } opa_link_xlate[] = { 9322 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) }, 9323 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) }, 9324 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) }, 9325 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) }, 9326 }; 9327 9328 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) { 9329 if (opa_widths & opa_link_xlate[i].from) 9330 result |= opa_link_xlate[i].to; 9331 } 9332 return result; 9333 } 9334 9335 /* 9336 * Set link attributes before moving to polling. 9337 */ 9338 static int set_local_link_attributes(struct hfi1_pportdata *ppd) 9339 { 9340 struct hfi1_devdata *dd = ppd->dd; 9341 u8 enable_lane_tx; 9342 u8 tx_polarity_inversion; 9343 u8 rx_polarity_inversion; 9344 int ret; 9345 u32 misc_bits = 0; 9346 /* reset our fabric serdes to clear any lingering problems */ 9347 fabric_serdes_reset(dd); 9348 9349 /* set the local tx rate - need to read-modify-write */ 9350 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 9351 &rx_polarity_inversion, &ppd->local_tx_rate); 9352 if (ret) 9353 goto set_local_link_attributes_fail; 9354 9355 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) { 9356 /* set the tx rate to the fastest enabled */ 9357 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9358 ppd->local_tx_rate = 1; 9359 else 9360 ppd->local_tx_rate = 0; 9361 } else { 9362 /* set the tx rate to all enabled */ 9363 ppd->local_tx_rate = 0; 9364 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9365 ppd->local_tx_rate |= 2; 9366 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G) 9367 ppd->local_tx_rate |= 1; 9368 } 9369 9370 enable_lane_tx = 0xF; /* enable all four lanes */ 9371 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion, 9372 rx_polarity_inversion, ppd->local_tx_rate); 9373 if (ret != HCMD_SUCCESS) 9374 goto set_local_link_attributes_fail; 9375 9376 ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION); 9377 if (ret != HCMD_SUCCESS) { 9378 dd_dev_err(dd, 9379 "Failed to set host interface version, return 0x%x\n", 9380 ret); 9381 goto set_local_link_attributes_fail; 9382 } 9383 9384 /* 9385 * DC supports continuous updates. 9386 */ 9387 ret = write_vc_local_phy(dd, 9388 0 /* no power management */, 9389 1 /* continuous updates */); 9390 if (ret != HCMD_SUCCESS) 9391 goto set_local_link_attributes_fail; 9392 9393 /* z=1 in the next call: AU of 0 is not supported by the hardware */ 9394 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init, 9395 ppd->port_crc_mode_enabled); 9396 if (ret != HCMD_SUCCESS) 9397 goto set_local_link_attributes_fail; 9398 9399 /* 9400 * SerDes loopback init sequence requires 9401 * setting bit 0 of MISC_CONFIG_BITS 9402 */ 9403 if (loopback == LOOPBACK_SERDES) 9404 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT; 9405 9406 /* 9407 * An external device configuration request is used to reset the LCB 9408 * to retry to obtain operational lanes when the first attempt is 9409 * unsuccesful. 9410 */ 9411 if (dd->dc8051_ver >= dc8051_ver(1, 25, 0)) 9412 misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT; 9413 9414 ret = write_vc_local_link_mode(dd, misc_bits, 0, 9415 opa_to_vc_link_widths( 9416 ppd->link_width_enabled)); 9417 if (ret != HCMD_SUCCESS) 9418 goto set_local_link_attributes_fail; 9419 9420 /* let peer know who we are */ 9421 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev); 9422 if (ret == HCMD_SUCCESS) 9423 return 0; 9424 9425 set_local_link_attributes_fail: 9426 dd_dev_err(dd, 9427 "Failed to set local link attributes, return 0x%x\n", 9428 ret); 9429 return ret; 9430 } 9431 9432 /* 9433 * Call this to start the link. 9434 * Do not do anything if the link is disabled. 9435 * Returns 0 if link is disabled, moved to polling, or the driver is not ready. 9436 */ 9437 int start_link(struct hfi1_pportdata *ppd) 9438 { 9439 /* 9440 * Tune the SerDes to a ballpark setting for optimal signal and bit 9441 * error rate. Needs to be done before starting the link. 9442 */ 9443 tune_serdes(ppd); 9444 9445 if (!ppd->driver_link_ready) { 9446 dd_dev_info(ppd->dd, 9447 "%s: stopping link start because driver is not ready\n", 9448 __func__); 9449 return 0; 9450 } 9451 9452 /* 9453 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the 9454 * pkey table can be configured properly if the HFI unit is connected 9455 * to switch port with MgmtAllowed=NO 9456 */ 9457 clear_full_mgmt_pkey(ppd); 9458 9459 return set_link_state(ppd, HLS_DN_POLL); 9460 } 9461 9462 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd) 9463 { 9464 struct hfi1_devdata *dd = ppd->dd; 9465 u64 mask; 9466 unsigned long timeout; 9467 9468 /* 9469 * Some QSFP cables have a quirk that asserts the IntN line as a side 9470 * effect of power up on plug-in. We ignore this false positive 9471 * interrupt until the module has finished powering up by waiting for 9472 * a minimum timeout of the module inrush initialization time of 9473 * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the 9474 * module have stabilized. 9475 */ 9476 msleep(500); 9477 9478 /* 9479 * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1) 9480 */ 9481 timeout = jiffies + msecs_to_jiffies(2000); 9482 while (1) { 9483 mask = read_csr(dd, dd->hfi1_id ? 9484 ASIC_QSFP2_IN : ASIC_QSFP1_IN); 9485 if (!(mask & QSFP_HFI0_INT_N)) 9486 break; 9487 if (time_after(jiffies, timeout)) { 9488 dd_dev_info(dd, "%s: No IntN detected, reset complete\n", 9489 __func__); 9490 break; 9491 } 9492 udelay(2); 9493 } 9494 } 9495 9496 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable) 9497 { 9498 struct hfi1_devdata *dd = ppd->dd; 9499 u64 mask; 9500 9501 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK); 9502 if (enable) { 9503 /* 9504 * Clear the status register to avoid an immediate interrupt 9505 * when we re-enable the IntN pin 9506 */ 9507 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9508 QSFP_HFI0_INT_N); 9509 mask |= (u64)QSFP_HFI0_INT_N; 9510 } else { 9511 mask &= ~(u64)QSFP_HFI0_INT_N; 9512 } 9513 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask); 9514 } 9515 9516 int reset_qsfp(struct hfi1_pportdata *ppd) 9517 { 9518 struct hfi1_devdata *dd = ppd->dd; 9519 u64 mask, qsfp_mask; 9520 9521 /* Disable INT_N from triggering QSFP interrupts */ 9522 set_qsfp_int_n(ppd, 0); 9523 9524 /* Reset the QSFP */ 9525 mask = (u64)QSFP_HFI0_RESET_N; 9526 9527 qsfp_mask = read_csr(dd, 9528 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT); 9529 qsfp_mask &= ~mask; 9530 write_csr(dd, 9531 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9532 9533 udelay(10); 9534 9535 qsfp_mask |= mask; 9536 write_csr(dd, 9537 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9538 9539 wait_for_qsfp_init(ppd); 9540 9541 /* 9542 * Allow INT_N to trigger the QSFP interrupt to watch 9543 * for alarms and warnings 9544 */ 9545 set_qsfp_int_n(ppd, 1); 9546 9547 /* 9548 * After the reset, AOC transmitters are enabled by default. They need 9549 * to be turned off to complete the QSFP setup before they can be 9550 * enabled again. 9551 */ 9552 return set_qsfp_tx(ppd, 0); 9553 } 9554 9555 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd, 9556 u8 *qsfp_interrupt_status) 9557 { 9558 struct hfi1_devdata *dd = ppd->dd; 9559 9560 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) || 9561 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING)) 9562 dd_dev_err(dd, "%s: QSFP cable temperature too high\n", 9563 __func__); 9564 9565 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) || 9566 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING)) 9567 dd_dev_err(dd, "%s: QSFP cable temperature too low\n", 9568 __func__); 9569 9570 /* 9571 * The remaining alarms/warnings don't matter if the link is down. 9572 */ 9573 if (ppd->host_link_state & HLS_DOWN) 9574 return 0; 9575 9576 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) || 9577 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING)) 9578 dd_dev_err(dd, "%s: QSFP supply voltage too high\n", 9579 __func__); 9580 9581 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) || 9582 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING)) 9583 dd_dev_err(dd, "%s: QSFP supply voltage too low\n", 9584 __func__); 9585 9586 /* Byte 2 is vendor specific */ 9587 9588 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) || 9589 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING)) 9590 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n", 9591 __func__); 9592 9593 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) || 9594 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING)) 9595 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n", 9596 __func__); 9597 9598 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) || 9599 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING)) 9600 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n", 9601 __func__); 9602 9603 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) || 9604 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING)) 9605 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n", 9606 __func__); 9607 9608 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) || 9609 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING)) 9610 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n", 9611 __func__); 9612 9613 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) || 9614 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING)) 9615 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n", 9616 __func__); 9617 9618 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) || 9619 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING)) 9620 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n", 9621 __func__); 9622 9623 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) || 9624 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING)) 9625 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n", 9626 __func__); 9627 9628 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) || 9629 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING)) 9630 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n", 9631 __func__); 9632 9633 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) || 9634 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING)) 9635 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n", 9636 __func__); 9637 9638 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) || 9639 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING)) 9640 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n", 9641 __func__); 9642 9643 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) || 9644 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING)) 9645 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n", 9646 __func__); 9647 9648 /* Bytes 9-10 and 11-12 are reserved */ 9649 /* Bytes 13-15 are vendor specific */ 9650 9651 return 0; 9652 } 9653 9654 /* This routine will only be scheduled if the QSFP module present is asserted */ 9655 void qsfp_event(struct work_struct *work) 9656 { 9657 struct qsfp_data *qd; 9658 struct hfi1_pportdata *ppd; 9659 struct hfi1_devdata *dd; 9660 9661 qd = container_of(work, struct qsfp_data, qsfp_work); 9662 ppd = qd->ppd; 9663 dd = ppd->dd; 9664 9665 /* Sanity check */ 9666 if (!qsfp_mod_present(ppd)) 9667 return; 9668 9669 if (ppd->host_link_state == HLS_DN_DISABLE) { 9670 dd_dev_info(ppd->dd, 9671 "%s: stopping link start because link is disabled\n", 9672 __func__); 9673 return; 9674 } 9675 9676 /* 9677 * Turn DC back on after cable has been re-inserted. Up until 9678 * now, the DC has been in reset to save power. 9679 */ 9680 dc_start(dd); 9681 9682 if (qd->cache_refresh_required) { 9683 set_qsfp_int_n(ppd, 0); 9684 9685 wait_for_qsfp_init(ppd); 9686 9687 /* 9688 * Allow INT_N to trigger the QSFP interrupt to watch 9689 * for alarms and warnings 9690 */ 9691 set_qsfp_int_n(ppd, 1); 9692 9693 start_link(ppd); 9694 } 9695 9696 if (qd->check_interrupt_flags) { 9697 u8 qsfp_interrupt_status[16] = {0,}; 9698 9699 if (one_qsfp_read(ppd, dd->hfi1_id, 6, 9700 &qsfp_interrupt_status[0], 16) != 16) { 9701 dd_dev_info(dd, 9702 "%s: Failed to read status of QSFP module\n", 9703 __func__); 9704 } else { 9705 unsigned long flags; 9706 9707 handle_qsfp_error_conditions( 9708 ppd, qsfp_interrupt_status); 9709 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 9710 ppd->qsfp_info.check_interrupt_flags = 0; 9711 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 9712 flags); 9713 } 9714 } 9715 } 9716 9717 void init_qsfp_int(struct hfi1_devdata *dd) 9718 { 9719 struct hfi1_pportdata *ppd = dd->pport; 9720 u64 qsfp_mask; 9721 9722 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 9723 /* Clear current status to avoid spurious interrupts */ 9724 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9725 qsfp_mask); 9726 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, 9727 qsfp_mask); 9728 9729 set_qsfp_int_n(ppd, 0); 9730 9731 /* Handle active low nature of INT_N and MODPRST_N pins */ 9732 if (qsfp_mod_present(ppd)) 9733 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N; 9734 write_csr(dd, 9735 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT, 9736 qsfp_mask); 9737 9738 /* Enable the appropriate QSFP IRQ source */ 9739 if (!dd->hfi1_id) 9740 set_intr_bits(dd, QSFP1_INT, QSFP1_INT, true); 9741 else 9742 set_intr_bits(dd, QSFP2_INT, QSFP2_INT, true); 9743 } 9744 9745 /* 9746 * Do a one-time initialize of the LCB block. 9747 */ 9748 static void init_lcb(struct hfi1_devdata *dd) 9749 { 9750 /* simulator does not correctly handle LCB cclk loopback, skip */ 9751 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 9752 return; 9753 9754 /* the DC has been reset earlier in the driver load */ 9755 9756 /* set LCB for cclk loopback on the port */ 9757 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01); 9758 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00); 9759 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00); 9760 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 9761 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08); 9762 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02); 9763 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00); 9764 } 9765 9766 /* 9767 * Perform a test read on the QSFP. Return 0 on success, -ERRNO 9768 * on error. 9769 */ 9770 static int test_qsfp_read(struct hfi1_pportdata *ppd) 9771 { 9772 int ret; 9773 u8 status; 9774 9775 /* 9776 * Report success if not a QSFP or, if it is a QSFP, but the cable is 9777 * not present 9778 */ 9779 if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd)) 9780 return 0; 9781 9782 /* read byte 2, the status byte */ 9783 ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1); 9784 if (ret < 0) 9785 return ret; 9786 if (ret != 1) 9787 return -EIO; 9788 9789 return 0; /* success */ 9790 } 9791 9792 /* 9793 * Values for QSFP retry. 9794 * 9795 * Give up after 10s (20 x 500ms). The overall timeout was empirically 9796 * arrived at from experience on a large cluster. 9797 */ 9798 #define MAX_QSFP_RETRIES 20 9799 #define QSFP_RETRY_WAIT 500 /* msec */ 9800 9801 /* 9802 * Try a QSFP read. If it fails, schedule a retry for later. 9803 * Called on first link activation after driver load. 9804 */ 9805 static void try_start_link(struct hfi1_pportdata *ppd) 9806 { 9807 if (test_qsfp_read(ppd)) { 9808 /* read failed */ 9809 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) { 9810 dd_dev_err(ppd->dd, "QSFP not responding, giving up\n"); 9811 return; 9812 } 9813 dd_dev_info(ppd->dd, 9814 "QSFP not responding, waiting and retrying %d\n", 9815 (int)ppd->qsfp_retry_count); 9816 ppd->qsfp_retry_count++; 9817 queue_delayed_work(ppd->link_wq, &ppd->start_link_work, 9818 msecs_to_jiffies(QSFP_RETRY_WAIT)); 9819 return; 9820 } 9821 ppd->qsfp_retry_count = 0; 9822 9823 start_link(ppd); 9824 } 9825 9826 /* 9827 * Workqueue function to start the link after a delay. 9828 */ 9829 void handle_start_link(struct work_struct *work) 9830 { 9831 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 9832 start_link_work.work); 9833 try_start_link(ppd); 9834 } 9835 9836 int bringup_serdes(struct hfi1_pportdata *ppd) 9837 { 9838 struct hfi1_devdata *dd = ppd->dd; 9839 u64 guid; 9840 int ret; 9841 9842 if (HFI1_CAP_IS_KSET(EXTENDED_PSN)) 9843 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK); 9844 9845 guid = ppd->guids[HFI1_PORT_GUID_INDEX]; 9846 if (!guid) { 9847 if (dd->base_guid) 9848 guid = dd->base_guid + ppd->port - 1; 9849 ppd->guids[HFI1_PORT_GUID_INDEX] = guid; 9850 } 9851 9852 /* Set linkinit_reason on power up per OPA spec */ 9853 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP; 9854 9855 /* one-time init of the LCB */ 9856 init_lcb(dd); 9857 9858 if (loopback) { 9859 ret = init_loopback(dd); 9860 if (ret < 0) 9861 return ret; 9862 } 9863 9864 get_port_type(ppd); 9865 if (ppd->port_type == PORT_TYPE_QSFP) { 9866 set_qsfp_int_n(ppd, 0); 9867 wait_for_qsfp_init(ppd); 9868 set_qsfp_int_n(ppd, 1); 9869 } 9870 9871 try_start_link(ppd); 9872 return 0; 9873 } 9874 9875 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd) 9876 { 9877 struct hfi1_devdata *dd = ppd->dd; 9878 9879 /* 9880 * Shut down the link and keep it down. First turn off that the 9881 * driver wants to allow the link to be up (driver_link_ready). 9882 * Then make sure the link is not automatically restarted 9883 * (link_enabled). Cancel any pending restart. And finally 9884 * go offline. 9885 */ 9886 ppd->driver_link_ready = 0; 9887 ppd->link_enabled = 0; 9888 9889 ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */ 9890 flush_delayed_work(&ppd->start_link_work); 9891 cancel_delayed_work_sync(&ppd->start_link_work); 9892 9893 ppd->offline_disabled_reason = 9894 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT); 9895 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0, 9896 OPA_LINKDOWN_REASON_REBOOT); 9897 set_link_state(ppd, HLS_DN_OFFLINE); 9898 9899 /* disable the port */ 9900 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 9901 cancel_work_sync(&ppd->freeze_work); 9902 } 9903 9904 static inline int init_cpu_counters(struct hfi1_devdata *dd) 9905 { 9906 struct hfi1_pportdata *ppd; 9907 int i; 9908 9909 ppd = (struct hfi1_pportdata *)(dd + 1); 9910 for (i = 0; i < dd->num_pports; i++, ppd++) { 9911 ppd->ibport_data.rvp.rc_acks = NULL; 9912 ppd->ibport_data.rvp.rc_qacks = NULL; 9913 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64); 9914 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64); 9915 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64); 9916 if (!ppd->ibport_data.rvp.rc_acks || 9917 !ppd->ibport_data.rvp.rc_delayed_comp || 9918 !ppd->ibport_data.rvp.rc_qacks) 9919 return -ENOMEM; 9920 } 9921 9922 return 0; 9923 } 9924 9925 /* 9926 * index is the index into the receive array 9927 */ 9928 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index, 9929 u32 type, unsigned long pa, u16 order) 9930 { 9931 u64 reg; 9932 9933 if (!(dd->flags & HFI1_PRESENT)) 9934 goto done; 9935 9936 if (type == PT_INVALID || type == PT_INVALID_FLUSH) { 9937 pa = 0; 9938 order = 0; 9939 } else if (type > PT_INVALID) { 9940 dd_dev_err(dd, 9941 "unexpected receive array type %u for index %u, not handled\n", 9942 type, index); 9943 goto done; 9944 } 9945 trace_hfi1_put_tid(dd, index, type, pa, order); 9946 9947 #define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */ 9948 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK 9949 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT 9950 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK) 9951 << RCV_ARRAY_RT_ADDR_SHIFT; 9952 trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg); 9953 writeq(reg, dd->rcvarray_wc + (index * 8)); 9954 9955 if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3) 9956 /* 9957 * Eager entries are written and flushed 9958 * 9959 * Expected entries are flushed every 4 writes 9960 */ 9961 flush_wc(); 9962 done: 9963 return; 9964 } 9965 9966 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd) 9967 { 9968 struct hfi1_devdata *dd = rcd->dd; 9969 u32 i; 9970 9971 /* this could be optimized */ 9972 for (i = rcd->eager_base; i < rcd->eager_base + 9973 rcd->egrbufs.alloced; i++) 9974 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9975 9976 for (i = rcd->expected_base; 9977 i < rcd->expected_base + rcd->expected_count; i++) 9978 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9979 } 9980 9981 static const char * const ib_cfg_name_strings[] = { 9982 "HFI1_IB_CFG_LIDLMC", 9983 "HFI1_IB_CFG_LWID_DG_ENB", 9984 "HFI1_IB_CFG_LWID_ENB", 9985 "HFI1_IB_CFG_LWID", 9986 "HFI1_IB_CFG_SPD_ENB", 9987 "HFI1_IB_CFG_SPD", 9988 "HFI1_IB_CFG_RXPOL_ENB", 9989 "HFI1_IB_CFG_LREV_ENB", 9990 "HFI1_IB_CFG_LINKLATENCY", 9991 "HFI1_IB_CFG_HRTBT", 9992 "HFI1_IB_CFG_OP_VLS", 9993 "HFI1_IB_CFG_VL_HIGH_CAP", 9994 "HFI1_IB_CFG_VL_LOW_CAP", 9995 "HFI1_IB_CFG_OVERRUN_THRESH", 9996 "HFI1_IB_CFG_PHYERR_THRESH", 9997 "HFI1_IB_CFG_LINKDEFAULT", 9998 "HFI1_IB_CFG_PKEYS", 9999 "HFI1_IB_CFG_MTU", 10000 "HFI1_IB_CFG_LSTATE", 10001 "HFI1_IB_CFG_VL_HIGH_LIMIT", 10002 "HFI1_IB_CFG_PMA_TICKS", 10003 "HFI1_IB_CFG_PORT" 10004 }; 10005 10006 static const char *ib_cfg_name(int which) 10007 { 10008 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings)) 10009 return "invalid"; 10010 return ib_cfg_name_strings[which]; 10011 } 10012 10013 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which) 10014 { 10015 struct hfi1_devdata *dd = ppd->dd; 10016 int val = 0; 10017 10018 switch (which) { 10019 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */ 10020 val = ppd->link_width_enabled; 10021 break; 10022 case HFI1_IB_CFG_LWID: /* currently active Link-width */ 10023 val = ppd->link_width_active; 10024 break; 10025 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 10026 val = ppd->link_speed_enabled; 10027 break; 10028 case HFI1_IB_CFG_SPD: /* current Link speed */ 10029 val = ppd->link_speed_active; 10030 break; 10031 10032 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */ 10033 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */ 10034 case HFI1_IB_CFG_LINKLATENCY: 10035 goto unimplemented; 10036 10037 case HFI1_IB_CFG_OP_VLS: 10038 val = ppd->actual_vls_operational; 10039 break; 10040 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */ 10041 val = VL_ARB_HIGH_PRIO_TABLE_SIZE; 10042 break; 10043 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */ 10044 val = VL_ARB_LOW_PRIO_TABLE_SIZE; 10045 break; 10046 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 10047 val = ppd->overrun_threshold; 10048 break; 10049 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 10050 val = ppd->phy_error_threshold; 10051 break; 10052 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 10053 val = HLS_DEFAULT; 10054 break; 10055 10056 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */ 10057 case HFI1_IB_CFG_PMA_TICKS: 10058 default: 10059 unimplemented: 10060 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 10061 dd_dev_info( 10062 dd, 10063 "%s: which %s: not implemented\n", 10064 __func__, 10065 ib_cfg_name(which)); 10066 break; 10067 } 10068 10069 return val; 10070 } 10071 10072 /* 10073 * The largest MAD packet size. 10074 */ 10075 #define MAX_MAD_PACKET 2048 10076 10077 /* 10078 * Return the maximum header bytes that can go on the _wire_ 10079 * for this device. This count includes the ICRC which is 10080 * not part of the packet held in memory but it is appended 10081 * by the HW. 10082 * This is dependent on the device's receive header entry size. 10083 * HFI allows this to be set per-receive context, but the 10084 * driver presently enforces a global value. 10085 */ 10086 u32 lrh_max_header_bytes(struct hfi1_devdata *dd) 10087 { 10088 /* 10089 * The maximum non-payload (MTU) bytes in LRH.PktLen are 10090 * the Receive Header Entry Size minus the PBC (or RHF) size 10091 * plus one DW for the ICRC appended by HW. 10092 * 10093 * dd->rcd[0].rcvhdrqentsize is in DW. 10094 * We use rcd[0] as all context will have the same value. Also, 10095 * the first kernel context would have been allocated by now so 10096 * we are guaranteed a valid value. 10097 */ 10098 return (get_hdrqentsize(dd->rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2; 10099 } 10100 10101 /* 10102 * Set Send Length 10103 * @ppd: per port data 10104 * 10105 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck* 10106 * registers compare against LRH.PktLen, so use the max bytes included 10107 * in the LRH. 10108 * 10109 * This routine changes all VL values except VL15, which it maintains at 10110 * the same value. 10111 */ 10112 static void set_send_length(struct hfi1_pportdata *ppd) 10113 { 10114 struct hfi1_devdata *dd = ppd->dd; 10115 u32 max_hb = lrh_max_header_bytes(dd), dcmtu; 10116 u32 maxvlmtu = dd->vld[15].mtu; 10117 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2) 10118 & SEND_LEN_CHECK1_LEN_VL15_MASK) << 10119 SEND_LEN_CHECK1_LEN_VL15_SHIFT; 10120 int i, j; 10121 u32 thres; 10122 10123 for (i = 0; i < ppd->vls_supported; i++) { 10124 if (dd->vld[i].mtu > maxvlmtu) 10125 maxvlmtu = dd->vld[i].mtu; 10126 if (i <= 3) 10127 len1 |= (((dd->vld[i].mtu + max_hb) >> 2) 10128 & SEND_LEN_CHECK0_LEN_VL0_MASK) << 10129 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT); 10130 else 10131 len2 |= (((dd->vld[i].mtu + max_hb) >> 2) 10132 & SEND_LEN_CHECK1_LEN_VL4_MASK) << 10133 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT); 10134 } 10135 write_csr(dd, SEND_LEN_CHECK0, len1); 10136 write_csr(dd, SEND_LEN_CHECK1, len2); 10137 /* adjust kernel credit return thresholds based on new MTUs */ 10138 /* all kernel receive contexts have the same hdrqentsize */ 10139 for (i = 0; i < ppd->vls_supported; i++) { 10140 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50), 10141 sc_mtu_to_threshold(dd->vld[i].sc, 10142 dd->vld[i].mtu, 10143 get_hdrqentsize(dd->rcd[0]))); 10144 for (j = 0; j < INIT_SC_PER_VL; j++) 10145 sc_set_cr_threshold( 10146 pio_select_send_context_vl(dd, j, i), 10147 thres); 10148 } 10149 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50), 10150 sc_mtu_to_threshold(dd->vld[15].sc, 10151 dd->vld[15].mtu, 10152 dd->rcd[0]->rcvhdrqentsize)); 10153 sc_set_cr_threshold(dd->vld[15].sc, thres); 10154 10155 /* Adjust maximum MTU for the port in DC */ 10156 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 : 10157 (ilog2(maxvlmtu >> 8) + 1); 10158 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG); 10159 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK; 10160 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) << 10161 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT; 10162 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1); 10163 } 10164 10165 static void set_lidlmc(struct hfi1_pportdata *ppd) 10166 { 10167 int i; 10168 u64 sreg = 0; 10169 struct hfi1_devdata *dd = ppd->dd; 10170 u32 mask = ~((1U << ppd->lmc) - 1); 10171 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1); 10172 u32 lid; 10173 10174 /* 10175 * Program 0 in CSR if port lid is extended. This prevents 10176 * 9B packets being sent out for large lids. 10177 */ 10178 lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid; 10179 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK 10180 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK); 10181 c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK) 10182 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) | 10183 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK) 10184 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT); 10185 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1); 10186 10187 /* 10188 * Iterate over all the send contexts and set their SLID check 10189 */ 10190 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) << 10191 SEND_CTXT_CHECK_SLID_MASK_SHIFT) | 10192 (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) << 10193 SEND_CTXT_CHECK_SLID_VALUE_SHIFT); 10194 10195 for (i = 0; i < chip_send_contexts(dd); i++) { 10196 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x", 10197 i, (u32)sreg); 10198 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg); 10199 } 10200 10201 /* Now we have to do the same thing for the sdma engines */ 10202 sdma_update_lmc(dd, mask, lid); 10203 } 10204 10205 static const char *state_completed_string(u32 completed) 10206 { 10207 static const char * const state_completed[] = { 10208 "EstablishComm", 10209 "OptimizeEQ", 10210 "VerifyCap" 10211 }; 10212 10213 if (completed < ARRAY_SIZE(state_completed)) 10214 return state_completed[completed]; 10215 10216 return "unknown"; 10217 } 10218 10219 static const char all_lanes_dead_timeout_expired[] = 10220 "All lanes were inactive – was the interconnect media removed?"; 10221 static const char tx_out_of_policy[] = 10222 "Passing lanes on local port do not meet the local link width policy"; 10223 static const char no_state_complete[] = 10224 "State timeout occurred before link partner completed the state"; 10225 static const char * const state_complete_reasons[] = { 10226 [0x00] = "Reason unknown", 10227 [0x01] = "Link was halted by driver, refer to LinkDownReason", 10228 [0x02] = "Link partner reported failure", 10229 [0x10] = "Unable to achieve frame sync on any lane", 10230 [0x11] = 10231 "Unable to find a common bit rate with the link partner", 10232 [0x12] = 10233 "Unable to achieve frame sync on sufficient lanes to meet the local link width policy", 10234 [0x13] = 10235 "Unable to identify preset equalization on sufficient lanes to meet the local link width policy", 10236 [0x14] = no_state_complete, 10237 [0x15] = 10238 "State timeout occurred before link partner identified equalization presets", 10239 [0x16] = 10240 "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy", 10241 [0x17] = tx_out_of_policy, 10242 [0x20] = all_lanes_dead_timeout_expired, 10243 [0x21] = 10244 "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy", 10245 [0x22] = no_state_complete, 10246 [0x23] = 10247 "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy", 10248 [0x24] = tx_out_of_policy, 10249 [0x30] = all_lanes_dead_timeout_expired, 10250 [0x31] = 10251 "State timeout occurred waiting for host to process received frames", 10252 [0x32] = no_state_complete, 10253 [0x33] = 10254 "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy", 10255 [0x34] = tx_out_of_policy, 10256 [0x35] = "Negotiated link width is mutually exclusive", 10257 [0x36] = 10258 "Timed out before receiving verifycap frames in VerifyCap.Exchange", 10259 [0x37] = "Unable to resolve secure data exchange", 10260 }; 10261 10262 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd, 10263 u32 code) 10264 { 10265 const char *str = NULL; 10266 10267 if (code < ARRAY_SIZE(state_complete_reasons)) 10268 str = state_complete_reasons[code]; 10269 10270 if (str) 10271 return str; 10272 return "Reserved"; 10273 } 10274 10275 /* describe the given last state complete frame */ 10276 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame, 10277 const char *prefix) 10278 { 10279 struct hfi1_devdata *dd = ppd->dd; 10280 u32 success; 10281 u32 state; 10282 u32 reason; 10283 u32 lanes; 10284 10285 /* 10286 * Decode frame: 10287 * [ 0: 0] - success 10288 * [ 3: 1] - state 10289 * [ 7: 4] - next state timeout 10290 * [15: 8] - reason code 10291 * [31:16] - lanes 10292 */ 10293 success = frame & 0x1; 10294 state = (frame >> 1) & 0x7; 10295 reason = (frame >> 8) & 0xff; 10296 lanes = (frame >> 16) & 0xffff; 10297 10298 dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n", 10299 prefix, frame); 10300 dd_dev_err(dd, " last reported state state: %s (0x%x)\n", 10301 state_completed_string(state), state); 10302 dd_dev_err(dd, " state successfully completed: %s\n", 10303 success ? "yes" : "no"); 10304 dd_dev_err(dd, " fail reason 0x%x: %s\n", 10305 reason, state_complete_reason_code_string(ppd, reason)); 10306 dd_dev_err(dd, " passing lane mask: 0x%x", lanes); 10307 } 10308 10309 /* 10310 * Read the last state complete frames and explain them. This routine 10311 * expects to be called if the link went down during link negotiation 10312 * and initialization (LNI). That is, anywhere between polling and link up. 10313 */ 10314 static void check_lni_states(struct hfi1_pportdata *ppd) 10315 { 10316 u32 last_local_state; 10317 u32 last_remote_state; 10318 10319 read_last_local_state(ppd->dd, &last_local_state); 10320 read_last_remote_state(ppd->dd, &last_remote_state); 10321 10322 /* 10323 * Don't report anything if there is nothing to report. A value of 10324 * 0 means the link was taken down while polling and there was no 10325 * training in-process. 10326 */ 10327 if (last_local_state == 0 && last_remote_state == 0) 10328 return; 10329 10330 decode_state_complete(ppd, last_local_state, "transmitted"); 10331 decode_state_complete(ppd, last_remote_state, "received"); 10332 } 10333 10334 /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */ 10335 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms) 10336 { 10337 u64 reg; 10338 unsigned long timeout; 10339 10340 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */ 10341 timeout = jiffies + msecs_to_jiffies(wait_ms); 10342 while (1) { 10343 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE); 10344 if (reg) 10345 break; 10346 if (time_after(jiffies, timeout)) { 10347 dd_dev_err(dd, 10348 "timeout waiting for LINK_TRANSFER_ACTIVE\n"); 10349 return -ETIMEDOUT; 10350 } 10351 udelay(2); 10352 } 10353 return 0; 10354 } 10355 10356 /* called when the logical link state is not down as it should be */ 10357 static void force_logical_link_state_down(struct hfi1_pportdata *ppd) 10358 { 10359 struct hfi1_devdata *dd = ppd->dd; 10360 10361 /* 10362 * Bring link up in LCB loopback 10363 */ 10364 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); 10365 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 10366 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 10367 10368 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 10369 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0); 10370 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 10371 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2); 10372 10373 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 10374 (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET); 10375 udelay(3); 10376 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1); 10377 write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 10378 10379 wait_link_transfer_active(dd, 100); 10380 10381 /* 10382 * Bring the link down again. 10383 */ 10384 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); 10385 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0); 10386 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0); 10387 10388 dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n"); 10389 } 10390 10391 /* 10392 * Helper for set_link_state(). Do not call except from that routine. 10393 * Expects ppd->hls_mutex to be held. 10394 * 10395 * @rem_reason value to be sent to the neighbor 10396 * 10397 * LinkDownReasons only set if transition succeeds. 10398 */ 10399 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) 10400 { 10401 struct hfi1_devdata *dd = ppd->dd; 10402 u32 previous_state; 10403 int offline_state_ret; 10404 int ret; 10405 10406 update_lcb_cache(dd); 10407 10408 previous_state = ppd->host_link_state; 10409 ppd->host_link_state = HLS_GOING_OFFLINE; 10410 10411 /* start offline transition */ 10412 ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE); 10413 10414 if (ret != HCMD_SUCCESS) { 10415 dd_dev_err(dd, 10416 "Failed to transition to Offline link state, return %d\n", 10417 ret); 10418 return -EINVAL; 10419 } 10420 if (ppd->offline_disabled_reason == 10421 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)) 10422 ppd->offline_disabled_reason = 10423 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 10424 10425 offline_state_ret = wait_phys_link_offline_substates(ppd, 10000); 10426 if (offline_state_ret < 0) 10427 return offline_state_ret; 10428 10429 /* Disabling AOC transmitters */ 10430 if (ppd->port_type == PORT_TYPE_QSFP && 10431 ppd->qsfp_info.limiting_active && 10432 qsfp_mod_present(ppd)) { 10433 int ret; 10434 10435 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT); 10436 if (ret == 0) { 10437 set_qsfp_tx(ppd, 0); 10438 release_chip_resource(dd, qsfp_resource(dd)); 10439 } else { 10440 /* not fatal, but should warn */ 10441 dd_dev_err(dd, 10442 "Unable to acquire lock to turn off QSFP TX\n"); 10443 } 10444 } 10445 10446 /* 10447 * Wait for the offline.Quiet transition if it hasn't happened yet. It 10448 * can take a while for the link to go down. 10449 */ 10450 if (offline_state_ret != PLS_OFFLINE_QUIET) { 10451 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000); 10452 if (ret < 0) 10453 return ret; 10454 } 10455 10456 /* 10457 * Now in charge of LCB - must be after the physical state is 10458 * offline.quiet and before host_link_state is changed. 10459 */ 10460 set_host_lcb_access(dd); 10461 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 10462 10463 /* make sure the logical state is also down */ 10464 ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000); 10465 if (ret) 10466 force_logical_link_state_down(ppd); 10467 10468 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */ 10469 update_statusp(ppd, IB_PORT_DOWN); 10470 10471 /* 10472 * The LNI has a mandatory wait time after the physical state 10473 * moves to Offline.Quiet. The wait time may be different 10474 * depending on how the link went down. The 8051 firmware 10475 * will observe the needed wait time and only move to ready 10476 * when that is completed. The largest of the quiet timeouts 10477 * is 6s, so wait that long and then at least 0.5s more for 10478 * other transitions, and another 0.5s for a buffer. 10479 */ 10480 ret = wait_fm_ready(dd, 7000); 10481 if (ret) { 10482 dd_dev_err(dd, 10483 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n"); 10484 /* state is really offline, so make it so */ 10485 ppd->host_link_state = HLS_DN_OFFLINE; 10486 return ret; 10487 } 10488 10489 /* 10490 * The state is now offline and the 8051 is ready to accept host 10491 * requests. 10492 * - change our state 10493 * - notify others if we were previously in a linkup state 10494 */ 10495 ppd->host_link_state = HLS_DN_OFFLINE; 10496 if (previous_state & HLS_UP) { 10497 /* went down while link was up */ 10498 handle_linkup_change(dd, 0); 10499 } else if (previous_state 10500 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 10501 /* went down while attempting link up */ 10502 check_lni_states(ppd); 10503 10504 /* The QSFP doesn't need to be reset on LNI failure */ 10505 ppd->qsfp_info.reset_needed = 0; 10506 } 10507 10508 /* the active link width (downgrade) is 0 on link down */ 10509 ppd->link_width_active = 0; 10510 ppd->link_width_downgrade_tx_active = 0; 10511 ppd->link_width_downgrade_rx_active = 0; 10512 ppd->current_egress_rate = 0; 10513 return 0; 10514 } 10515 10516 /* return the link state name */ 10517 static const char *link_state_name(u32 state) 10518 { 10519 const char *name; 10520 int n = ilog2(state); 10521 static const char * const names[] = { 10522 [__HLS_UP_INIT_BP] = "INIT", 10523 [__HLS_UP_ARMED_BP] = "ARMED", 10524 [__HLS_UP_ACTIVE_BP] = "ACTIVE", 10525 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF", 10526 [__HLS_DN_POLL_BP] = "POLL", 10527 [__HLS_DN_DISABLE_BP] = "DISABLE", 10528 [__HLS_DN_OFFLINE_BP] = "OFFLINE", 10529 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP", 10530 [__HLS_GOING_UP_BP] = "GOING_UP", 10531 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE", 10532 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN" 10533 }; 10534 10535 name = n < ARRAY_SIZE(names) ? names[n] : NULL; 10536 return name ? name : "unknown"; 10537 } 10538 10539 /* return the link state reason name */ 10540 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state) 10541 { 10542 if (state == HLS_UP_INIT) { 10543 switch (ppd->linkinit_reason) { 10544 case OPA_LINKINIT_REASON_LINKUP: 10545 return "(LINKUP)"; 10546 case OPA_LINKINIT_REASON_FLAPPING: 10547 return "(FLAPPING)"; 10548 case OPA_LINKINIT_OUTSIDE_POLICY: 10549 return "(OUTSIDE_POLICY)"; 10550 case OPA_LINKINIT_QUARANTINED: 10551 return "(QUARANTINED)"; 10552 case OPA_LINKINIT_INSUFIC_CAPABILITY: 10553 return "(INSUFIC_CAPABILITY)"; 10554 default: 10555 break; 10556 } 10557 } 10558 return ""; 10559 } 10560 10561 /* 10562 * driver_pstate - convert the driver's notion of a port's 10563 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*). 10564 * Return -1 (converted to a u32) to indicate error. 10565 */ 10566 u32 driver_pstate(struct hfi1_pportdata *ppd) 10567 { 10568 switch (ppd->host_link_state) { 10569 case HLS_UP_INIT: 10570 case HLS_UP_ARMED: 10571 case HLS_UP_ACTIVE: 10572 return IB_PORTPHYSSTATE_LINKUP; 10573 case HLS_DN_POLL: 10574 return IB_PORTPHYSSTATE_POLLING; 10575 case HLS_DN_DISABLE: 10576 return IB_PORTPHYSSTATE_DISABLED; 10577 case HLS_DN_OFFLINE: 10578 return OPA_PORTPHYSSTATE_OFFLINE; 10579 case HLS_VERIFY_CAP: 10580 return IB_PORTPHYSSTATE_TRAINING; 10581 case HLS_GOING_UP: 10582 return IB_PORTPHYSSTATE_TRAINING; 10583 case HLS_GOING_OFFLINE: 10584 return OPA_PORTPHYSSTATE_OFFLINE; 10585 case HLS_LINK_COOLDOWN: 10586 return OPA_PORTPHYSSTATE_OFFLINE; 10587 case HLS_DN_DOWNDEF: 10588 default: 10589 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10590 ppd->host_link_state); 10591 return -1; 10592 } 10593 } 10594 10595 /* 10596 * driver_lstate - convert the driver's notion of a port's 10597 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1 10598 * (converted to a u32) to indicate error. 10599 */ 10600 u32 driver_lstate(struct hfi1_pportdata *ppd) 10601 { 10602 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN)) 10603 return IB_PORT_DOWN; 10604 10605 switch (ppd->host_link_state & HLS_UP) { 10606 case HLS_UP_INIT: 10607 return IB_PORT_INIT; 10608 case HLS_UP_ARMED: 10609 return IB_PORT_ARMED; 10610 case HLS_UP_ACTIVE: 10611 return IB_PORT_ACTIVE; 10612 default: 10613 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10614 ppd->host_link_state); 10615 return -1; 10616 } 10617 } 10618 10619 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason, 10620 u8 neigh_reason, u8 rem_reason) 10621 { 10622 if (ppd->local_link_down_reason.latest == 0 && 10623 ppd->neigh_link_down_reason.latest == 0) { 10624 ppd->local_link_down_reason.latest = lcl_reason; 10625 ppd->neigh_link_down_reason.latest = neigh_reason; 10626 ppd->remote_link_down_reason = rem_reason; 10627 } 10628 } 10629 10630 /** 10631 * data_vls_operational() - Verify if data VL BCT credits and MTU 10632 * are both set. 10633 * @ppd: pointer to hfi1_pportdata structure 10634 * 10635 * Return: true - Ok, false -otherwise. 10636 */ 10637 static inline bool data_vls_operational(struct hfi1_pportdata *ppd) 10638 { 10639 int i; 10640 u64 reg; 10641 10642 if (!ppd->actual_vls_operational) 10643 return false; 10644 10645 for (i = 0; i < ppd->vls_supported; i++) { 10646 reg = read_csr(ppd->dd, SEND_CM_CREDIT_VL + (8 * i)); 10647 if ((reg && !ppd->dd->vld[i].mtu) || 10648 (!reg && ppd->dd->vld[i].mtu)) 10649 return false; 10650 } 10651 10652 return true; 10653 } 10654 10655 /* 10656 * Change the physical and/or logical link state. 10657 * 10658 * Do not call this routine while inside an interrupt. It contains 10659 * calls to routines that can take multiple seconds to finish. 10660 * 10661 * Returns 0 on success, -errno on failure. 10662 */ 10663 int set_link_state(struct hfi1_pportdata *ppd, u32 state) 10664 { 10665 struct hfi1_devdata *dd = ppd->dd; 10666 struct ib_event event = {.device = NULL}; 10667 int ret1, ret = 0; 10668 int orig_new_state, poll_bounce; 10669 10670 mutex_lock(&ppd->hls_lock); 10671 10672 orig_new_state = state; 10673 if (state == HLS_DN_DOWNDEF) 10674 state = HLS_DEFAULT; 10675 10676 /* interpret poll -> poll as a link bounce */ 10677 poll_bounce = ppd->host_link_state == HLS_DN_POLL && 10678 state == HLS_DN_POLL; 10679 10680 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__, 10681 link_state_name(ppd->host_link_state), 10682 link_state_name(orig_new_state), 10683 poll_bounce ? "(bounce) " : "", 10684 link_state_reason_name(ppd, state)); 10685 10686 /* 10687 * If we're going to a (HLS_*) link state that implies the logical 10688 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then 10689 * reset is_sm_config_started to 0. 10690 */ 10691 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE))) 10692 ppd->is_sm_config_started = 0; 10693 10694 /* 10695 * Do nothing if the states match. Let a poll to poll link bounce 10696 * go through. 10697 */ 10698 if (ppd->host_link_state == state && !poll_bounce) 10699 goto done; 10700 10701 switch (state) { 10702 case HLS_UP_INIT: 10703 if (ppd->host_link_state == HLS_DN_POLL && 10704 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) { 10705 /* 10706 * Quick link up jumps from polling to here. 10707 * 10708 * Whether in normal or loopback mode, the 10709 * simulator jumps from polling to link up. 10710 * Accept that here. 10711 */ 10712 /* OK */ 10713 } else if (ppd->host_link_state != HLS_GOING_UP) { 10714 goto unexpected; 10715 } 10716 10717 /* 10718 * Wait for Link_Up physical state. 10719 * Physical and Logical states should already be 10720 * be transitioned to LinkUp and LinkInit respectively. 10721 */ 10722 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000); 10723 if (ret) { 10724 dd_dev_err(dd, 10725 "%s: physical state did not change to LINK-UP\n", 10726 __func__); 10727 break; 10728 } 10729 10730 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000); 10731 if (ret) { 10732 dd_dev_err(dd, 10733 "%s: logical state did not change to INIT\n", 10734 __func__); 10735 break; 10736 } 10737 10738 /* clear old transient LINKINIT_REASON code */ 10739 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR) 10740 ppd->linkinit_reason = 10741 OPA_LINKINIT_REASON_LINKUP; 10742 10743 /* enable the port */ 10744 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 10745 10746 handle_linkup_change(dd, 1); 10747 pio_kernel_linkup(dd); 10748 10749 /* 10750 * After link up, a new link width will have been set. 10751 * Update the xmit counters with regards to the new 10752 * link width. 10753 */ 10754 update_xmit_counters(ppd, ppd->link_width_active); 10755 10756 ppd->host_link_state = HLS_UP_INIT; 10757 update_statusp(ppd, IB_PORT_INIT); 10758 break; 10759 case HLS_UP_ARMED: 10760 if (ppd->host_link_state != HLS_UP_INIT) 10761 goto unexpected; 10762 10763 if (!data_vls_operational(ppd)) { 10764 dd_dev_err(dd, 10765 "%s: Invalid data VL credits or mtu\n", 10766 __func__); 10767 ret = -EINVAL; 10768 break; 10769 } 10770 10771 set_logical_state(dd, LSTATE_ARMED); 10772 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000); 10773 if (ret) { 10774 dd_dev_err(dd, 10775 "%s: logical state did not change to ARMED\n", 10776 __func__); 10777 break; 10778 } 10779 ppd->host_link_state = HLS_UP_ARMED; 10780 update_statusp(ppd, IB_PORT_ARMED); 10781 /* 10782 * The simulator does not currently implement SMA messages, 10783 * so neighbor_normal is not set. Set it here when we first 10784 * move to Armed. 10785 */ 10786 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 10787 ppd->neighbor_normal = 1; 10788 break; 10789 case HLS_UP_ACTIVE: 10790 if (ppd->host_link_state != HLS_UP_ARMED) 10791 goto unexpected; 10792 10793 set_logical_state(dd, LSTATE_ACTIVE); 10794 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000); 10795 if (ret) { 10796 dd_dev_err(dd, 10797 "%s: logical state did not change to ACTIVE\n", 10798 __func__); 10799 } else { 10800 /* tell all engines to go running */ 10801 sdma_all_running(dd); 10802 ppd->host_link_state = HLS_UP_ACTIVE; 10803 update_statusp(ppd, IB_PORT_ACTIVE); 10804 10805 /* Signal the IB layer that the port has went active */ 10806 event.device = &dd->verbs_dev.rdi.ibdev; 10807 event.element.port_num = ppd->port; 10808 event.event = IB_EVENT_PORT_ACTIVE; 10809 } 10810 break; 10811 case HLS_DN_POLL: 10812 if ((ppd->host_link_state == HLS_DN_DISABLE || 10813 ppd->host_link_state == HLS_DN_OFFLINE) && 10814 dd->dc_shutdown) 10815 dc_start(dd); 10816 /* Hand LED control to the DC */ 10817 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 10818 10819 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10820 u8 tmp = ppd->link_enabled; 10821 10822 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10823 if (ret) { 10824 ppd->link_enabled = tmp; 10825 break; 10826 } 10827 ppd->remote_link_down_reason = 0; 10828 10829 if (ppd->driver_link_ready) 10830 ppd->link_enabled = 1; 10831 } 10832 10833 set_all_slowpath(ppd->dd); 10834 ret = set_local_link_attributes(ppd); 10835 if (ret) 10836 break; 10837 10838 ppd->port_error_action = 0; 10839 10840 if (quick_linkup) { 10841 /* quick linkup does not go into polling */ 10842 ret = do_quick_linkup(dd); 10843 } else { 10844 ret1 = set_physical_link_state(dd, PLS_POLLING); 10845 if (!ret1) 10846 ret1 = wait_phys_link_out_of_offline(ppd, 10847 3000); 10848 if (ret1 != HCMD_SUCCESS) { 10849 dd_dev_err(dd, 10850 "Failed to transition to Polling link state, return 0x%x\n", 10851 ret1); 10852 ret = -EINVAL; 10853 } 10854 } 10855 10856 /* 10857 * Change the host link state after requesting DC8051 to 10858 * change its physical state so that we can ignore any 10859 * interrupt with stale LNI(XX) error, which will not be 10860 * cleared until DC8051 transitions to Polling state. 10861 */ 10862 ppd->host_link_state = HLS_DN_POLL; 10863 ppd->offline_disabled_reason = 10864 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE); 10865 /* 10866 * If an error occurred above, go back to offline. The 10867 * caller may reschedule another attempt. 10868 */ 10869 if (ret) 10870 goto_offline(ppd, 0); 10871 else 10872 log_physical_state(ppd, PLS_POLLING); 10873 break; 10874 case HLS_DN_DISABLE: 10875 /* link is disabled */ 10876 ppd->link_enabled = 0; 10877 10878 /* allow any state to transition to disabled */ 10879 10880 /* must transition to offline first */ 10881 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10882 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10883 if (ret) 10884 break; 10885 ppd->remote_link_down_reason = 0; 10886 } 10887 10888 if (!dd->dc_shutdown) { 10889 ret1 = set_physical_link_state(dd, PLS_DISABLED); 10890 if (ret1 != HCMD_SUCCESS) { 10891 dd_dev_err(dd, 10892 "Failed to transition to Disabled link state, return 0x%x\n", 10893 ret1); 10894 ret = -EINVAL; 10895 break; 10896 } 10897 ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000); 10898 if (ret) { 10899 dd_dev_err(dd, 10900 "%s: physical state did not change to DISABLED\n", 10901 __func__); 10902 break; 10903 } 10904 dc_shutdown(dd); 10905 } 10906 ppd->host_link_state = HLS_DN_DISABLE; 10907 break; 10908 case HLS_DN_OFFLINE: 10909 if (ppd->host_link_state == HLS_DN_DISABLE) 10910 dc_start(dd); 10911 10912 /* allow any state to transition to offline */ 10913 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10914 if (!ret) 10915 ppd->remote_link_down_reason = 0; 10916 break; 10917 case HLS_VERIFY_CAP: 10918 if (ppd->host_link_state != HLS_DN_POLL) 10919 goto unexpected; 10920 ppd->host_link_state = HLS_VERIFY_CAP; 10921 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP); 10922 break; 10923 case HLS_GOING_UP: 10924 if (ppd->host_link_state != HLS_VERIFY_CAP) 10925 goto unexpected; 10926 10927 ret1 = set_physical_link_state(dd, PLS_LINKUP); 10928 if (ret1 != HCMD_SUCCESS) { 10929 dd_dev_err(dd, 10930 "Failed to transition to link up state, return 0x%x\n", 10931 ret1); 10932 ret = -EINVAL; 10933 break; 10934 } 10935 ppd->host_link_state = HLS_GOING_UP; 10936 break; 10937 10938 case HLS_GOING_OFFLINE: /* transient within goto_offline() */ 10939 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */ 10940 default: 10941 dd_dev_info(dd, "%s: state 0x%x: not supported\n", 10942 __func__, state); 10943 ret = -EINVAL; 10944 break; 10945 } 10946 10947 goto done; 10948 10949 unexpected: 10950 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n", 10951 __func__, link_state_name(ppd->host_link_state), 10952 link_state_name(state)); 10953 ret = -EINVAL; 10954 10955 done: 10956 mutex_unlock(&ppd->hls_lock); 10957 10958 if (event.device) 10959 ib_dispatch_event(&event); 10960 10961 return ret; 10962 } 10963 10964 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val) 10965 { 10966 u64 reg; 10967 int ret = 0; 10968 10969 switch (which) { 10970 case HFI1_IB_CFG_LIDLMC: 10971 set_lidlmc(ppd); 10972 break; 10973 case HFI1_IB_CFG_VL_HIGH_LIMIT: 10974 /* 10975 * The VL Arbitrator high limit is sent in units of 4k 10976 * bytes, while HFI stores it in units of 64 bytes. 10977 */ 10978 val *= 4096 / 64; 10979 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK) 10980 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT; 10981 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg); 10982 break; 10983 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 10984 /* HFI only supports POLL as the default link down state */ 10985 if (val != HLS_DN_POLL) 10986 ret = -EINVAL; 10987 break; 10988 case HFI1_IB_CFG_OP_VLS: 10989 if (ppd->vls_operational != val) { 10990 ppd->vls_operational = val; 10991 if (!ppd->port) 10992 ret = -EINVAL; 10993 } 10994 break; 10995 /* 10996 * For link width, link width downgrade, and speed enable, always AND 10997 * the setting with what is actually supported. This has two benefits. 10998 * First, enabled can't have unsupported values, no matter what the 10999 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean 11000 * "fill in with your supported value" have all the bits in the 11001 * field set, so simply ANDing with supported has the desired result. 11002 */ 11003 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */ 11004 ppd->link_width_enabled = val & ppd->link_width_supported; 11005 break; 11006 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */ 11007 ppd->link_width_downgrade_enabled = 11008 val & ppd->link_width_downgrade_supported; 11009 break; 11010 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 11011 ppd->link_speed_enabled = val & ppd->link_speed_supported; 11012 break; 11013 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 11014 /* 11015 * HFI does not follow IB specs, save this value 11016 * so we can report it, if asked. 11017 */ 11018 ppd->overrun_threshold = val; 11019 break; 11020 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 11021 /* 11022 * HFI does not follow IB specs, save this value 11023 * so we can report it, if asked. 11024 */ 11025 ppd->phy_error_threshold = val; 11026 break; 11027 11028 case HFI1_IB_CFG_MTU: 11029 set_send_length(ppd); 11030 break; 11031 11032 case HFI1_IB_CFG_PKEYS: 11033 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 11034 set_partition_keys(ppd); 11035 break; 11036 11037 default: 11038 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 11039 dd_dev_info(ppd->dd, 11040 "%s: which %s, val 0x%x: not implemented\n", 11041 __func__, ib_cfg_name(which), val); 11042 break; 11043 } 11044 return ret; 11045 } 11046 11047 /* begin functions related to vl arbitration table caching */ 11048 static void init_vl_arb_caches(struct hfi1_pportdata *ppd) 11049 { 11050 int i; 11051 11052 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 11053 VL_ARB_LOW_PRIO_TABLE_SIZE); 11054 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 11055 VL_ARB_HIGH_PRIO_TABLE_SIZE); 11056 11057 /* 11058 * Note that we always return values directly from the 11059 * 'vl_arb_cache' (and do no CSR reads) in response to a 11060 * 'Get(VLArbTable)'. This is obviously correct after a 11061 * 'Set(VLArbTable)', since the cache will then be up to 11062 * date. But it's also correct prior to any 'Set(VLArbTable)' 11063 * since then both the cache, and the relevant h/w registers 11064 * will be zeroed. 11065 */ 11066 11067 for (i = 0; i < MAX_PRIO_TABLE; i++) 11068 spin_lock_init(&ppd->vl_arb_cache[i].lock); 11069 } 11070 11071 /* 11072 * vl_arb_lock_cache 11073 * 11074 * All other vl_arb_* functions should be called only after locking 11075 * the cache. 11076 */ 11077 static inline struct vl_arb_cache * 11078 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx) 11079 { 11080 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE) 11081 return NULL; 11082 spin_lock(&ppd->vl_arb_cache[idx].lock); 11083 return &ppd->vl_arb_cache[idx]; 11084 } 11085 11086 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx) 11087 { 11088 spin_unlock(&ppd->vl_arb_cache[idx].lock); 11089 } 11090 11091 static void vl_arb_get_cache(struct vl_arb_cache *cache, 11092 struct ib_vl_weight_elem *vl) 11093 { 11094 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl)); 11095 } 11096 11097 static void vl_arb_set_cache(struct vl_arb_cache *cache, 11098 struct ib_vl_weight_elem *vl) 11099 { 11100 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 11101 } 11102 11103 static int vl_arb_match_cache(struct vl_arb_cache *cache, 11104 struct ib_vl_weight_elem *vl) 11105 { 11106 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 11107 } 11108 11109 /* end functions related to vl arbitration table caching */ 11110 11111 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target, 11112 u32 size, struct ib_vl_weight_elem *vl) 11113 { 11114 struct hfi1_devdata *dd = ppd->dd; 11115 u64 reg; 11116 unsigned int i, is_up = 0; 11117 int drain, ret = 0; 11118 11119 mutex_lock(&ppd->hls_lock); 11120 11121 if (ppd->host_link_state & HLS_UP) 11122 is_up = 1; 11123 11124 drain = !is_ax(dd) && is_up; 11125 11126 if (drain) 11127 /* 11128 * Before adjusting VL arbitration weights, empty per-VL 11129 * FIFOs, otherwise a packet whose VL weight is being 11130 * set to 0 could get stuck in a FIFO with no chance to 11131 * egress. 11132 */ 11133 ret = stop_drain_data_vls(dd); 11134 11135 if (ret) { 11136 dd_dev_err( 11137 dd, 11138 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n", 11139 __func__); 11140 goto err; 11141 } 11142 11143 for (i = 0; i < size; i++, vl++) { 11144 /* 11145 * NOTE: The low priority shift and mask are used here, but 11146 * they are the same for both the low and high registers. 11147 */ 11148 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK) 11149 << SEND_LOW_PRIORITY_LIST_VL_SHIFT) 11150 | (((u64)vl->weight 11151 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK) 11152 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT); 11153 write_csr(dd, target + (i * 8), reg); 11154 } 11155 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE); 11156 11157 if (drain) 11158 open_fill_data_vls(dd); /* reopen all VLs */ 11159 11160 err: 11161 mutex_unlock(&ppd->hls_lock); 11162 11163 return ret; 11164 } 11165 11166 /* 11167 * Read one credit merge VL register. 11168 */ 11169 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr, 11170 struct vl_limit *vll) 11171 { 11172 u64 reg = read_csr(dd, csr); 11173 11174 vll->dedicated = cpu_to_be16( 11175 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT) 11176 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK); 11177 vll->shared = cpu_to_be16( 11178 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT) 11179 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK); 11180 } 11181 11182 /* 11183 * Read the current credit merge limits. 11184 */ 11185 static int get_buffer_control(struct hfi1_devdata *dd, 11186 struct buffer_control *bc, u16 *overall_limit) 11187 { 11188 u64 reg; 11189 int i; 11190 11191 /* not all entries are filled in */ 11192 memset(bc, 0, sizeof(*bc)); 11193 11194 /* OPA and HFI have a 1-1 mapping */ 11195 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11196 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]); 11197 11198 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */ 11199 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]); 11200 11201 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11202 bc->overall_shared_limit = cpu_to_be16( 11203 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) 11204 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK); 11205 if (overall_limit) 11206 *overall_limit = (reg 11207 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) 11208 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK; 11209 return sizeof(struct buffer_control); 11210 } 11211 11212 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 11213 { 11214 u64 reg; 11215 int i; 11216 11217 /* each register contains 16 SC->VLnt mappings, 4 bits each */ 11218 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0); 11219 for (i = 0; i < sizeof(u64); i++) { 11220 u8 byte = *(((u8 *)®) + i); 11221 11222 dp->vlnt[2 * i] = byte & 0xf; 11223 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4; 11224 } 11225 11226 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16); 11227 for (i = 0; i < sizeof(u64); i++) { 11228 u8 byte = *(((u8 *)®) + i); 11229 11230 dp->vlnt[16 + (2 * i)] = byte & 0xf; 11231 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4; 11232 } 11233 return sizeof(struct sc2vlnt); 11234 } 11235 11236 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems, 11237 struct ib_vl_weight_elem *vl) 11238 { 11239 unsigned int i; 11240 11241 for (i = 0; i < nelems; i++, vl++) { 11242 vl->vl = 0xf; 11243 vl->weight = 0; 11244 } 11245 } 11246 11247 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 11248 { 11249 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, 11250 DC_SC_VL_VAL(15_0, 11251 0, dp->vlnt[0] & 0xf, 11252 1, dp->vlnt[1] & 0xf, 11253 2, dp->vlnt[2] & 0xf, 11254 3, dp->vlnt[3] & 0xf, 11255 4, dp->vlnt[4] & 0xf, 11256 5, dp->vlnt[5] & 0xf, 11257 6, dp->vlnt[6] & 0xf, 11258 7, dp->vlnt[7] & 0xf, 11259 8, dp->vlnt[8] & 0xf, 11260 9, dp->vlnt[9] & 0xf, 11261 10, dp->vlnt[10] & 0xf, 11262 11, dp->vlnt[11] & 0xf, 11263 12, dp->vlnt[12] & 0xf, 11264 13, dp->vlnt[13] & 0xf, 11265 14, dp->vlnt[14] & 0xf, 11266 15, dp->vlnt[15] & 0xf)); 11267 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, 11268 DC_SC_VL_VAL(31_16, 11269 16, dp->vlnt[16] & 0xf, 11270 17, dp->vlnt[17] & 0xf, 11271 18, dp->vlnt[18] & 0xf, 11272 19, dp->vlnt[19] & 0xf, 11273 20, dp->vlnt[20] & 0xf, 11274 21, dp->vlnt[21] & 0xf, 11275 22, dp->vlnt[22] & 0xf, 11276 23, dp->vlnt[23] & 0xf, 11277 24, dp->vlnt[24] & 0xf, 11278 25, dp->vlnt[25] & 0xf, 11279 26, dp->vlnt[26] & 0xf, 11280 27, dp->vlnt[27] & 0xf, 11281 28, dp->vlnt[28] & 0xf, 11282 29, dp->vlnt[29] & 0xf, 11283 30, dp->vlnt[30] & 0xf, 11284 31, dp->vlnt[31] & 0xf)); 11285 } 11286 11287 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what, 11288 u16 limit) 11289 { 11290 if (limit != 0) 11291 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n", 11292 what, (int)limit, idx); 11293 } 11294 11295 /* change only the shared limit portion of SendCmGLobalCredit */ 11296 static void set_global_shared(struct hfi1_devdata *dd, u16 limit) 11297 { 11298 u64 reg; 11299 11300 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11301 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK; 11302 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT; 11303 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 11304 } 11305 11306 /* change only the total credit limit portion of SendCmGLobalCredit */ 11307 static void set_global_limit(struct hfi1_devdata *dd, u16 limit) 11308 { 11309 u64 reg; 11310 11311 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11312 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK; 11313 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 11314 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 11315 } 11316 11317 /* set the given per-VL shared limit */ 11318 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit) 11319 { 11320 u64 reg; 11321 u32 addr; 11322 11323 if (vl < TXE_NUM_DATA_VL) 11324 addr = SEND_CM_CREDIT_VL + (8 * vl); 11325 else 11326 addr = SEND_CM_CREDIT_VL15; 11327 11328 reg = read_csr(dd, addr); 11329 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK; 11330 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT; 11331 write_csr(dd, addr, reg); 11332 } 11333 11334 /* set the given per-VL dedicated limit */ 11335 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit) 11336 { 11337 u64 reg; 11338 u32 addr; 11339 11340 if (vl < TXE_NUM_DATA_VL) 11341 addr = SEND_CM_CREDIT_VL + (8 * vl); 11342 else 11343 addr = SEND_CM_CREDIT_VL15; 11344 11345 reg = read_csr(dd, addr); 11346 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK; 11347 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT; 11348 write_csr(dd, addr, reg); 11349 } 11350 11351 /* spin until the given per-VL status mask bits clear */ 11352 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask, 11353 const char *which) 11354 { 11355 unsigned long timeout; 11356 u64 reg; 11357 11358 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT); 11359 while (1) { 11360 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask; 11361 11362 if (reg == 0) 11363 return; /* success */ 11364 if (time_after(jiffies, timeout)) 11365 break; /* timed out */ 11366 udelay(1); 11367 } 11368 11369 dd_dev_err(dd, 11370 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n", 11371 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg); 11372 /* 11373 * If this occurs, it is likely there was a credit loss on the link. 11374 * The only recovery from that is a link bounce. 11375 */ 11376 dd_dev_err(dd, 11377 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n"); 11378 } 11379 11380 /* 11381 * The number of credits on the VLs may be changed while everything 11382 * is "live", but the following algorithm must be followed due to 11383 * how the hardware is actually implemented. In particular, 11384 * Return_Credit_Status[] is the only correct status check. 11385 * 11386 * if (reducing Global_Shared_Credit_Limit or any shared limit changing) 11387 * set Global_Shared_Credit_Limit = 0 11388 * use_all_vl = 1 11389 * mask0 = all VLs that are changing either dedicated or shared limits 11390 * set Shared_Limit[mask0] = 0 11391 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0 11392 * if (changing any dedicated limit) 11393 * mask1 = all VLs that are lowering dedicated limits 11394 * lower Dedicated_Limit[mask1] 11395 * spin until Return_Credit_Status[mask1] == 0 11396 * raise Dedicated_Limits 11397 * raise Shared_Limits 11398 * raise Global_Shared_Credit_Limit 11399 * 11400 * lower = if the new limit is lower, set the limit to the new value 11401 * raise = if the new limit is higher than the current value (may be changed 11402 * earlier in the algorithm), set the new limit to the new value 11403 */ 11404 int set_buffer_control(struct hfi1_pportdata *ppd, 11405 struct buffer_control *new_bc) 11406 { 11407 struct hfi1_devdata *dd = ppd->dd; 11408 u64 changing_mask, ld_mask, stat_mask; 11409 int change_count; 11410 int i, use_all_mask; 11411 int this_shared_changing; 11412 int vl_count = 0, ret; 11413 /* 11414 * A0: add the variable any_shared_limit_changing below and in the 11415 * algorithm above. If removing A0 support, it can be removed. 11416 */ 11417 int any_shared_limit_changing; 11418 struct buffer_control cur_bc; 11419 u8 changing[OPA_MAX_VLS]; 11420 u8 lowering_dedicated[OPA_MAX_VLS]; 11421 u16 cur_total; 11422 u32 new_total = 0; 11423 const u64 all_mask = 11424 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK 11425 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK 11426 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK 11427 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK 11428 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK 11429 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK 11430 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK 11431 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK 11432 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK; 11433 11434 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15) 11435 #define NUM_USABLE_VLS 16 /* look at VL15 and less */ 11436 11437 /* find the new total credits, do sanity check on unused VLs */ 11438 for (i = 0; i < OPA_MAX_VLS; i++) { 11439 if (valid_vl(i)) { 11440 new_total += be16_to_cpu(new_bc->vl[i].dedicated); 11441 continue; 11442 } 11443 nonzero_msg(dd, i, "dedicated", 11444 be16_to_cpu(new_bc->vl[i].dedicated)); 11445 nonzero_msg(dd, i, "shared", 11446 be16_to_cpu(new_bc->vl[i].shared)); 11447 new_bc->vl[i].dedicated = 0; 11448 new_bc->vl[i].shared = 0; 11449 } 11450 new_total += be16_to_cpu(new_bc->overall_shared_limit); 11451 11452 /* fetch the current values */ 11453 get_buffer_control(dd, &cur_bc, &cur_total); 11454 11455 /* 11456 * Create the masks we will use. 11457 */ 11458 memset(changing, 0, sizeof(changing)); 11459 memset(lowering_dedicated, 0, sizeof(lowering_dedicated)); 11460 /* 11461 * NOTE: Assumes that the individual VL bits are adjacent and in 11462 * increasing order 11463 */ 11464 stat_mask = 11465 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK; 11466 changing_mask = 0; 11467 ld_mask = 0; 11468 change_count = 0; 11469 any_shared_limit_changing = 0; 11470 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) { 11471 if (!valid_vl(i)) 11472 continue; 11473 this_shared_changing = new_bc->vl[i].shared 11474 != cur_bc.vl[i].shared; 11475 if (this_shared_changing) 11476 any_shared_limit_changing = 1; 11477 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated || 11478 this_shared_changing) { 11479 changing[i] = 1; 11480 changing_mask |= stat_mask; 11481 change_count++; 11482 } 11483 if (be16_to_cpu(new_bc->vl[i].dedicated) < 11484 be16_to_cpu(cur_bc.vl[i].dedicated)) { 11485 lowering_dedicated[i] = 1; 11486 ld_mask |= stat_mask; 11487 } 11488 } 11489 11490 /* bracket the credit change with a total adjustment */ 11491 if (new_total > cur_total) 11492 set_global_limit(dd, new_total); 11493 11494 /* 11495 * Start the credit change algorithm. 11496 */ 11497 use_all_mask = 0; 11498 if ((be16_to_cpu(new_bc->overall_shared_limit) < 11499 be16_to_cpu(cur_bc.overall_shared_limit)) || 11500 (is_ax(dd) && any_shared_limit_changing)) { 11501 set_global_shared(dd, 0); 11502 cur_bc.overall_shared_limit = 0; 11503 use_all_mask = 1; 11504 } 11505 11506 for (i = 0; i < NUM_USABLE_VLS; i++) { 11507 if (!valid_vl(i)) 11508 continue; 11509 11510 if (changing[i]) { 11511 set_vl_shared(dd, i, 0); 11512 cur_bc.vl[i].shared = 0; 11513 } 11514 } 11515 11516 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask, 11517 "shared"); 11518 11519 if (change_count > 0) { 11520 for (i = 0; i < NUM_USABLE_VLS; i++) { 11521 if (!valid_vl(i)) 11522 continue; 11523 11524 if (lowering_dedicated[i]) { 11525 set_vl_dedicated(dd, i, 11526 be16_to_cpu(new_bc-> 11527 vl[i].dedicated)); 11528 cur_bc.vl[i].dedicated = 11529 new_bc->vl[i].dedicated; 11530 } 11531 } 11532 11533 wait_for_vl_status_clear(dd, ld_mask, "dedicated"); 11534 11535 /* now raise all dedicated that are going up */ 11536 for (i = 0; i < NUM_USABLE_VLS; i++) { 11537 if (!valid_vl(i)) 11538 continue; 11539 11540 if (be16_to_cpu(new_bc->vl[i].dedicated) > 11541 be16_to_cpu(cur_bc.vl[i].dedicated)) 11542 set_vl_dedicated(dd, i, 11543 be16_to_cpu(new_bc-> 11544 vl[i].dedicated)); 11545 } 11546 } 11547 11548 /* next raise all shared that are going up */ 11549 for (i = 0; i < NUM_USABLE_VLS; i++) { 11550 if (!valid_vl(i)) 11551 continue; 11552 11553 if (be16_to_cpu(new_bc->vl[i].shared) > 11554 be16_to_cpu(cur_bc.vl[i].shared)) 11555 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared)); 11556 } 11557 11558 /* finally raise the global shared */ 11559 if (be16_to_cpu(new_bc->overall_shared_limit) > 11560 be16_to_cpu(cur_bc.overall_shared_limit)) 11561 set_global_shared(dd, 11562 be16_to_cpu(new_bc->overall_shared_limit)); 11563 11564 /* bracket the credit change with a total adjustment */ 11565 if (new_total < cur_total) 11566 set_global_limit(dd, new_total); 11567 11568 /* 11569 * Determine the actual number of operational VLS using the number of 11570 * dedicated and shared credits for each VL. 11571 */ 11572 if (change_count > 0) { 11573 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11574 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 || 11575 be16_to_cpu(new_bc->vl[i].shared) > 0) 11576 vl_count++; 11577 ppd->actual_vls_operational = vl_count; 11578 ret = sdma_map_init(dd, ppd->port - 1, vl_count ? 11579 ppd->actual_vls_operational : 11580 ppd->vls_operational, 11581 NULL); 11582 if (ret == 0) 11583 ret = pio_map_init(dd, ppd->port - 1, vl_count ? 11584 ppd->actual_vls_operational : 11585 ppd->vls_operational, NULL); 11586 if (ret) 11587 return ret; 11588 } 11589 return 0; 11590 } 11591 11592 /* 11593 * Read the given fabric manager table. Return the size of the 11594 * table (in bytes) on success, and a negative error code on 11595 * failure. 11596 */ 11597 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t) 11598 11599 { 11600 int size; 11601 struct vl_arb_cache *vlc; 11602 11603 switch (which) { 11604 case FM_TBL_VL_HIGH_ARB: 11605 size = 256; 11606 /* 11607 * OPA specifies 128 elements (of 2 bytes each), though 11608 * HFI supports only 16 elements in h/w. 11609 */ 11610 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11611 vl_arb_get_cache(vlc, t); 11612 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11613 break; 11614 case FM_TBL_VL_LOW_ARB: 11615 size = 256; 11616 /* 11617 * OPA specifies 128 elements (of 2 bytes each), though 11618 * HFI supports only 16 elements in h/w. 11619 */ 11620 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11621 vl_arb_get_cache(vlc, t); 11622 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11623 break; 11624 case FM_TBL_BUFFER_CONTROL: 11625 size = get_buffer_control(ppd->dd, t, NULL); 11626 break; 11627 case FM_TBL_SC2VLNT: 11628 size = get_sc2vlnt(ppd->dd, t); 11629 break; 11630 case FM_TBL_VL_PREEMPT_ELEMS: 11631 size = 256; 11632 /* OPA specifies 128 elements, of 2 bytes each */ 11633 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t); 11634 break; 11635 case FM_TBL_VL_PREEMPT_MATRIX: 11636 size = 256; 11637 /* 11638 * OPA specifies that this is the same size as the VL 11639 * arbitration tables (i.e., 256 bytes). 11640 */ 11641 break; 11642 default: 11643 return -EINVAL; 11644 } 11645 return size; 11646 } 11647 11648 /* 11649 * Write the given fabric manager table. 11650 */ 11651 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t) 11652 { 11653 int ret = 0; 11654 struct vl_arb_cache *vlc; 11655 11656 switch (which) { 11657 case FM_TBL_VL_HIGH_ARB: 11658 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11659 if (vl_arb_match_cache(vlc, t)) { 11660 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11661 break; 11662 } 11663 vl_arb_set_cache(vlc, t); 11664 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11665 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST, 11666 VL_ARB_HIGH_PRIO_TABLE_SIZE, t); 11667 break; 11668 case FM_TBL_VL_LOW_ARB: 11669 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11670 if (vl_arb_match_cache(vlc, t)) { 11671 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11672 break; 11673 } 11674 vl_arb_set_cache(vlc, t); 11675 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11676 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST, 11677 VL_ARB_LOW_PRIO_TABLE_SIZE, t); 11678 break; 11679 case FM_TBL_BUFFER_CONTROL: 11680 ret = set_buffer_control(ppd, t); 11681 break; 11682 case FM_TBL_SC2VLNT: 11683 set_sc2vlnt(ppd->dd, t); 11684 break; 11685 default: 11686 ret = -EINVAL; 11687 } 11688 return ret; 11689 } 11690 11691 /* 11692 * Disable all data VLs. 11693 * 11694 * Return 0 if disabled, non-zero if the VLs cannot be disabled. 11695 */ 11696 static int disable_data_vls(struct hfi1_devdata *dd) 11697 { 11698 if (is_ax(dd)) 11699 return 1; 11700 11701 pio_send_control(dd, PSC_DATA_VL_DISABLE); 11702 11703 return 0; 11704 } 11705 11706 /* 11707 * open_fill_data_vls() - the counterpart to stop_drain_data_vls(). 11708 * Just re-enables all data VLs (the "fill" part happens 11709 * automatically - the name was chosen for symmetry with 11710 * stop_drain_data_vls()). 11711 * 11712 * Return 0 if successful, non-zero if the VLs cannot be enabled. 11713 */ 11714 int open_fill_data_vls(struct hfi1_devdata *dd) 11715 { 11716 if (is_ax(dd)) 11717 return 1; 11718 11719 pio_send_control(dd, PSC_DATA_VL_ENABLE); 11720 11721 return 0; 11722 } 11723 11724 /* 11725 * drain_data_vls() - assumes that disable_data_vls() has been called, 11726 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA 11727 * engines to drop to 0. 11728 */ 11729 static void drain_data_vls(struct hfi1_devdata *dd) 11730 { 11731 sc_wait(dd); 11732 sdma_wait(dd); 11733 pause_for_credit_return(dd); 11734 } 11735 11736 /* 11737 * stop_drain_data_vls() - disable, then drain all per-VL fifos. 11738 * 11739 * Use open_fill_data_vls() to resume using data VLs. This pair is 11740 * meant to be used like this: 11741 * 11742 * stop_drain_data_vls(dd); 11743 * // do things with per-VL resources 11744 * open_fill_data_vls(dd); 11745 */ 11746 int stop_drain_data_vls(struct hfi1_devdata *dd) 11747 { 11748 int ret; 11749 11750 ret = disable_data_vls(dd); 11751 if (ret == 0) 11752 drain_data_vls(dd); 11753 11754 return ret; 11755 } 11756 11757 /* 11758 * Convert a nanosecond time to a cclock count. No matter how slow 11759 * the cclock, a non-zero ns will always have a non-zero result. 11760 */ 11761 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns) 11762 { 11763 u32 cclocks; 11764 11765 if (dd->icode == ICODE_FPGA_EMULATION) 11766 cclocks = (ns * 1000) / FPGA_CCLOCK_PS; 11767 else /* simulation pretends to be ASIC */ 11768 cclocks = (ns * 1000) / ASIC_CCLOCK_PS; 11769 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */ 11770 cclocks = 1; 11771 return cclocks; 11772 } 11773 11774 /* 11775 * Convert a cclock count to nanoseconds. Not matter how slow 11776 * the cclock, a non-zero cclocks will always have a non-zero result. 11777 */ 11778 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks) 11779 { 11780 u32 ns; 11781 11782 if (dd->icode == ICODE_FPGA_EMULATION) 11783 ns = (cclocks * FPGA_CCLOCK_PS) / 1000; 11784 else /* simulation pretends to be ASIC */ 11785 ns = (cclocks * ASIC_CCLOCK_PS) / 1000; 11786 if (cclocks && !ns) 11787 ns = 1; 11788 return ns; 11789 } 11790 11791 /* 11792 * Dynamically adjust the receive interrupt timeout for a context based on 11793 * incoming packet rate. 11794 * 11795 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero. 11796 */ 11797 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts) 11798 { 11799 struct hfi1_devdata *dd = rcd->dd; 11800 u32 timeout = rcd->rcvavail_timeout; 11801 11802 /* 11803 * This algorithm doubles or halves the timeout depending on whether 11804 * the number of packets received in this interrupt were less than or 11805 * greater equal the interrupt count. 11806 * 11807 * The calculations below do not allow a steady state to be achieved. 11808 * Only at the endpoints it is possible to have an unchanging 11809 * timeout. 11810 */ 11811 if (npkts < rcv_intr_count) { 11812 /* 11813 * Not enough packets arrived before the timeout, adjust 11814 * timeout downward. 11815 */ 11816 if (timeout < 2) /* already at minimum? */ 11817 return; 11818 timeout >>= 1; 11819 } else { 11820 /* 11821 * More than enough packets arrived before the timeout, adjust 11822 * timeout upward. 11823 */ 11824 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */ 11825 return; 11826 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr); 11827 } 11828 11829 rcd->rcvavail_timeout = timeout; 11830 /* 11831 * timeout cannot be larger than rcv_intr_timeout_csr which has already 11832 * been verified to be in range 11833 */ 11834 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT, 11835 (u64)timeout << 11836 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 11837 } 11838 11839 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, 11840 u32 intr_adjust, u32 npkts) 11841 { 11842 struct hfi1_devdata *dd = rcd->dd; 11843 u64 reg; 11844 u32 ctxt = rcd->ctxt; 11845 11846 /* 11847 * Need to write timeout register before updating RcvHdrHead to ensure 11848 * that a new value is used when the HW decides to restart counting. 11849 */ 11850 if (intr_adjust) 11851 adjust_rcv_timeout(rcd, npkts); 11852 if (updegr) { 11853 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK) 11854 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT; 11855 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg); 11856 } 11857 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) | 11858 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK) 11859 << RCV_HDR_HEAD_HEAD_SHIFT); 11860 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 11861 } 11862 11863 u32 hdrqempty(struct hfi1_ctxtdata *rcd) 11864 { 11865 u32 head, tail; 11866 11867 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) 11868 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT; 11869 11870 if (hfi1_rcvhdrtail_kvaddr(rcd)) 11871 tail = get_rcvhdrtail(rcd); 11872 else 11873 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 11874 11875 return head == tail; 11876 } 11877 11878 /* 11879 * Context Control and Receive Array encoding for buffer size: 11880 * 0x0 invalid 11881 * 0x1 4 KB 11882 * 0x2 8 KB 11883 * 0x3 16 KB 11884 * 0x4 32 KB 11885 * 0x5 64 KB 11886 * 0x6 128 KB 11887 * 0x7 256 KB 11888 * 0x8 512 KB (Receive Array only) 11889 * 0x9 1 MB (Receive Array only) 11890 * 0xa 2 MB (Receive Array only) 11891 * 11892 * 0xB-0xF - reserved (Receive Array only) 11893 * 11894 * 11895 * This routine assumes that the value has already been sanity checked. 11896 */ 11897 static u32 encoded_size(u32 size) 11898 { 11899 switch (size) { 11900 case 4 * 1024: return 0x1; 11901 case 8 * 1024: return 0x2; 11902 case 16 * 1024: return 0x3; 11903 case 32 * 1024: return 0x4; 11904 case 64 * 1024: return 0x5; 11905 case 128 * 1024: return 0x6; 11906 case 256 * 1024: return 0x7; 11907 case 512 * 1024: return 0x8; 11908 case 1 * 1024 * 1024: return 0x9; 11909 case 2 * 1024 * 1024: return 0xa; 11910 } 11911 return 0x1; /* if invalid, go with the minimum size */ 11912 } 11913 11914 /** 11915 * encode_rcv_header_entry_size - return chip specific encoding for size 11916 * @size: size in dwords 11917 * 11918 * Convert a receive header entry size that to the encoding used in the CSR. 11919 * 11920 * Return a zero if the given size is invalid, otherwise the encoding. 11921 */ 11922 u8 encode_rcv_header_entry_size(u8 size) 11923 { 11924 /* there are only 3 valid receive header entry sizes */ 11925 if (size == 2) 11926 return 1; 11927 if (size == 16) 11928 return 2; 11929 if (size == 32) 11930 return 4; 11931 return 0; /* invalid */ 11932 } 11933 11934 /** 11935 * hfi1_validate_rcvhdrcnt - validate hdrcnt 11936 * @dd: the device data 11937 * @thecnt: the header count 11938 */ 11939 int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt) 11940 { 11941 if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) { 11942 dd_dev_err(dd, "Receive header queue count too small\n"); 11943 return -EINVAL; 11944 } 11945 11946 if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) { 11947 dd_dev_err(dd, 11948 "Receive header queue count cannot be greater than %u\n", 11949 HFI1_MAX_HDRQ_EGRBUF_CNT); 11950 return -EINVAL; 11951 } 11952 11953 if (thecnt % HDRQ_INCREMENT) { 11954 dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n", 11955 thecnt, HDRQ_INCREMENT); 11956 return -EINVAL; 11957 } 11958 11959 return 0; 11960 } 11961 11962 /** 11963 * set_hdrq_regs - set header queue registers for context 11964 * @dd: the device data 11965 * @ctxt: the context 11966 * @entsize: the dword entry size 11967 * @hdrcnt: the number of header entries 11968 */ 11969 void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt) 11970 { 11971 u64 reg; 11972 11973 reg = (((u64)hdrcnt >> HDRQ_SIZE_SHIFT) & RCV_HDR_CNT_CNT_MASK) << 11974 RCV_HDR_CNT_CNT_SHIFT; 11975 write_kctxt_csr(dd, ctxt, RCV_HDR_CNT, reg); 11976 reg = ((u64)encode_rcv_header_entry_size(entsize) & 11977 RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) << 11978 RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT; 11979 write_kctxt_csr(dd, ctxt, RCV_HDR_ENT_SIZE, reg); 11980 reg = ((u64)DEFAULT_RCVHDRSIZE & RCV_HDR_SIZE_HDR_SIZE_MASK) << 11981 RCV_HDR_SIZE_HDR_SIZE_SHIFT; 11982 write_kctxt_csr(dd, ctxt, RCV_HDR_SIZE, reg); 11983 11984 /* 11985 * Program dummy tail address for every receive context 11986 * before enabling any receive context 11987 */ 11988 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11989 dd->rcvhdrtail_dummy_dma); 11990 } 11991 11992 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, 11993 struct hfi1_ctxtdata *rcd) 11994 { 11995 u64 rcvctrl, reg; 11996 int did_enable = 0; 11997 u16 ctxt; 11998 11999 if (!rcd) 12000 return; 12001 12002 ctxt = rcd->ctxt; 12003 12004 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op); 12005 12006 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL); 12007 /* if the context already enabled, don't do the extra steps */ 12008 if ((op & HFI1_RCVCTRL_CTXT_ENB) && 12009 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) { 12010 /* reset the tail and hdr addresses, and sequence count */ 12011 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR, 12012 rcd->rcvhdrq_dma); 12013 if (hfi1_rcvhdrtail_kvaddr(rcd)) 12014 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 12015 rcd->rcvhdrqtailaddr_dma); 12016 hfi1_set_seq_cnt(rcd, 1); 12017 12018 /* reset the cached receive header queue head value */ 12019 hfi1_set_rcd_head(rcd, 0); 12020 12021 /* 12022 * Zero the receive header queue so we don't get false 12023 * positives when checking the sequence number. The 12024 * sequence numbers could land exactly on the same spot. 12025 * E.g. a rcd restart before the receive header wrapped. 12026 */ 12027 memset(rcd->rcvhdrq, 0, rcvhdrq_size(rcd)); 12028 12029 /* starting timeout */ 12030 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr; 12031 12032 /* enable the context */ 12033 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK; 12034 12035 /* clean the egr buffer size first */ 12036 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 12037 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size) 12038 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK) 12039 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT; 12040 12041 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */ 12042 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0); 12043 did_enable = 1; 12044 12045 /* zero RcvEgrIndexHead */ 12046 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0); 12047 12048 /* set eager count and base index */ 12049 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT) 12050 & RCV_EGR_CTRL_EGR_CNT_MASK) 12051 << RCV_EGR_CTRL_EGR_CNT_SHIFT) | 12052 (((rcd->eager_base >> RCV_SHIFT) 12053 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK) 12054 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT); 12055 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg); 12056 12057 /* 12058 * Set TID (expected) count and base index. 12059 * rcd->expected_count is set to individual RcvArray entries, 12060 * not pairs, and the CSR takes a pair-count in groups of 12061 * four, so divide by 8. 12062 */ 12063 reg = (((rcd->expected_count >> RCV_SHIFT) 12064 & RCV_TID_CTRL_TID_PAIR_CNT_MASK) 12065 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) | 12066 (((rcd->expected_base >> RCV_SHIFT) 12067 & RCV_TID_CTRL_TID_BASE_INDEX_MASK) 12068 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT); 12069 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg); 12070 if (ctxt == HFI1_CTRL_CTXT) 12071 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT); 12072 } 12073 if (op & HFI1_RCVCTRL_CTXT_DIS) { 12074 write_csr(dd, RCV_VL15, 0); 12075 /* 12076 * When receive context is being disabled turn on tail 12077 * update with a dummy tail address and then disable 12078 * receive context. 12079 */ 12080 if (dd->rcvhdrtail_dummy_dma) { 12081 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 12082 dd->rcvhdrtail_dummy_dma); 12083 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */ 12084 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 12085 } 12086 12087 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK; 12088 } 12089 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) { 12090 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt, 12091 IS_RCVAVAIL_START + rcd->ctxt, true); 12092 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 12093 } 12094 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) { 12095 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt, 12096 IS_RCVAVAIL_START + rcd->ctxt, false); 12097 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 12098 } 12099 if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && hfi1_rcvhdrtail_kvaddr(rcd)) 12100 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 12101 if (op & HFI1_RCVCTRL_TAILUPD_DIS) { 12102 /* See comment on RcvCtxtCtrl.TailUpd above */ 12103 if (!(op & HFI1_RCVCTRL_CTXT_DIS)) 12104 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK; 12105 } 12106 if (op & HFI1_RCVCTRL_TIDFLOW_ENB) 12107 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 12108 if (op & HFI1_RCVCTRL_TIDFLOW_DIS) 12109 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 12110 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) { 12111 /* 12112 * In one-packet-per-eager mode, the size comes from 12113 * the RcvArray entry. 12114 */ 12115 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 12116 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 12117 } 12118 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS) 12119 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 12120 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB) 12121 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 12122 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS) 12123 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 12124 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB) 12125 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 12126 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS) 12127 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 12128 if (op & HFI1_RCVCTRL_URGENT_ENB) 12129 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt, 12130 IS_RCVURGENT_START + rcd->ctxt, true); 12131 if (op & HFI1_RCVCTRL_URGENT_DIS) 12132 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt, 12133 IS_RCVURGENT_START + rcd->ctxt, false); 12134 12135 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl); 12136 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl); 12137 12138 /* work around sticky RcvCtxtStatus.BlockedRHQFull */ 12139 if (did_enable && 12140 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) { 12141 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 12142 if (reg != 0) { 12143 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n", 12144 ctxt, reg); 12145 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 12146 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10); 12147 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00); 12148 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 12149 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 12150 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n", 12151 ctxt, reg, reg == 0 ? "not" : "still"); 12152 } 12153 } 12154 12155 if (did_enable) { 12156 /* 12157 * The interrupt timeout and count must be set after 12158 * the context is enabled to take effect. 12159 */ 12160 /* set interrupt timeout */ 12161 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT, 12162 (u64)rcd->rcvavail_timeout << 12163 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 12164 12165 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */ 12166 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT; 12167 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 12168 } 12169 12170 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS)) 12171 /* 12172 * If the context has been disabled and the Tail Update has 12173 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address 12174 * so it doesn't contain an address that is invalid. 12175 */ 12176 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 12177 dd->rcvhdrtail_dummy_dma); 12178 } 12179 12180 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp) 12181 { 12182 int ret; 12183 u64 val = 0; 12184 12185 if (namep) { 12186 ret = dd->cntrnameslen; 12187 *namep = dd->cntrnames; 12188 } else { 12189 const struct cntr_entry *entry; 12190 int i, j; 12191 12192 ret = (dd->ndevcntrs) * sizeof(u64); 12193 12194 /* Get the start of the block of counters */ 12195 *cntrp = dd->cntrs; 12196 12197 /* 12198 * Now go and fill in each counter in the block. 12199 */ 12200 for (i = 0; i < DEV_CNTR_LAST; i++) { 12201 entry = &dev_cntrs[i]; 12202 hfi1_cdbg(CNTR, "reading %s", entry->name); 12203 if (entry->flags & CNTR_DISABLED) { 12204 /* Nothing */ 12205 hfi1_cdbg(CNTR, "\tDisabled\n"); 12206 } else { 12207 if (entry->flags & CNTR_VL) { 12208 hfi1_cdbg(CNTR, "\tPer VL\n"); 12209 for (j = 0; j < C_VL_COUNT; j++) { 12210 val = entry->rw_cntr(entry, 12211 dd, j, 12212 CNTR_MODE_R, 12213 0); 12214 hfi1_cdbg( 12215 CNTR, 12216 "\t\tRead 0x%llx for %d\n", 12217 val, j); 12218 dd->cntrs[entry->offset + j] = 12219 val; 12220 } 12221 } else if (entry->flags & CNTR_SDMA) { 12222 hfi1_cdbg(CNTR, 12223 "\t Per SDMA Engine\n"); 12224 for (j = 0; j < chip_sdma_engines(dd); 12225 j++) { 12226 val = 12227 entry->rw_cntr(entry, dd, j, 12228 CNTR_MODE_R, 0); 12229 hfi1_cdbg(CNTR, 12230 "\t\tRead 0x%llx for %d\n", 12231 val, j); 12232 dd->cntrs[entry->offset + j] = 12233 val; 12234 } 12235 } else { 12236 val = entry->rw_cntr(entry, dd, 12237 CNTR_INVALID_VL, 12238 CNTR_MODE_R, 0); 12239 dd->cntrs[entry->offset] = val; 12240 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 12241 } 12242 } 12243 } 12244 } 12245 return ret; 12246 } 12247 12248 /* 12249 * Used by sysfs to create files for hfi stats to read 12250 */ 12251 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp) 12252 { 12253 int ret; 12254 u64 val = 0; 12255 12256 if (namep) { 12257 ret = ppd->dd->portcntrnameslen; 12258 *namep = ppd->dd->portcntrnames; 12259 } else { 12260 const struct cntr_entry *entry; 12261 int i, j; 12262 12263 ret = ppd->dd->nportcntrs * sizeof(u64); 12264 *cntrp = ppd->cntrs; 12265 12266 for (i = 0; i < PORT_CNTR_LAST; i++) { 12267 entry = &port_cntrs[i]; 12268 hfi1_cdbg(CNTR, "reading %s", entry->name); 12269 if (entry->flags & CNTR_DISABLED) { 12270 /* Nothing */ 12271 hfi1_cdbg(CNTR, "\tDisabled\n"); 12272 continue; 12273 } 12274 12275 if (entry->flags & CNTR_VL) { 12276 hfi1_cdbg(CNTR, "\tPer VL"); 12277 for (j = 0; j < C_VL_COUNT; j++) { 12278 val = entry->rw_cntr(entry, ppd, j, 12279 CNTR_MODE_R, 12280 0); 12281 hfi1_cdbg( 12282 CNTR, 12283 "\t\tRead 0x%llx for %d", 12284 val, j); 12285 ppd->cntrs[entry->offset + j] = val; 12286 } 12287 } else { 12288 val = entry->rw_cntr(entry, ppd, 12289 CNTR_INVALID_VL, 12290 CNTR_MODE_R, 12291 0); 12292 ppd->cntrs[entry->offset] = val; 12293 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 12294 } 12295 } 12296 } 12297 return ret; 12298 } 12299 12300 static void free_cntrs(struct hfi1_devdata *dd) 12301 { 12302 struct hfi1_pportdata *ppd; 12303 int i; 12304 12305 if (dd->synth_stats_timer.function) 12306 del_timer_sync(&dd->synth_stats_timer); 12307 ppd = (struct hfi1_pportdata *)(dd + 1); 12308 for (i = 0; i < dd->num_pports; i++, ppd++) { 12309 kfree(ppd->cntrs); 12310 kfree(ppd->scntrs); 12311 free_percpu(ppd->ibport_data.rvp.rc_acks); 12312 free_percpu(ppd->ibport_data.rvp.rc_qacks); 12313 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp); 12314 ppd->cntrs = NULL; 12315 ppd->scntrs = NULL; 12316 ppd->ibport_data.rvp.rc_acks = NULL; 12317 ppd->ibport_data.rvp.rc_qacks = NULL; 12318 ppd->ibport_data.rvp.rc_delayed_comp = NULL; 12319 } 12320 kfree(dd->portcntrnames); 12321 dd->portcntrnames = NULL; 12322 kfree(dd->cntrs); 12323 dd->cntrs = NULL; 12324 kfree(dd->scntrs); 12325 dd->scntrs = NULL; 12326 kfree(dd->cntrnames); 12327 dd->cntrnames = NULL; 12328 if (dd->update_cntr_wq) { 12329 destroy_workqueue(dd->update_cntr_wq); 12330 dd->update_cntr_wq = NULL; 12331 } 12332 } 12333 12334 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry, 12335 u64 *psval, void *context, int vl) 12336 { 12337 u64 val; 12338 u64 sval = *psval; 12339 12340 if (entry->flags & CNTR_DISABLED) { 12341 dd_dev_err(dd, "Counter %s not enabled", entry->name); 12342 return 0; 12343 } 12344 12345 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 12346 12347 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0); 12348 12349 /* If its a synthetic counter there is more work we need to do */ 12350 if (entry->flags & CNTR_SYNTH) { 12351 if (sval == CNTR_MAX) { 12352 /* No need to read already saturated */ 12353 return CNTR_MAX; 12354 } 12355 12356 if (entry->flags & CNTR_32BIT) { 12357 /* 32bit counters can wrap multiple times */ 12358 u64 upper = sval >> 32; 12359 u64 lower = (sval << 32) >> 32; 12360 12361 if (lower > val) { /* hw wrapped */ 12362 if (upper == CNTR_32BIT_MAX) 12363 val = CNTR_MAX; 12364 else 12365 upper++; 12366 } 12367 12368 if (val != CNTR_MAX) 12369 val = (upper << 32) | val; 12370 12371 } else { 12372 /* If we rolled we are saturated */ 12373 if ((val < sval) || (val > CNTR_MAX)) 12374 val = CNTR_MAX; 12375 } 12376 } 12377 12378 *psval = val; 12379 12380 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 12381 12382 return val; 12383 } 12384 12385 static u64 write_dev_port_cntr(struct hfi1_devdata *dd, 12386 struct cntr_entry *entry, 12387 u64 *psval, void *context, int vl, u64 data) 12388 { 12389 u64 val; 12390 12391 if (entry->flags & CNTR_DISABLED) { 12392 dd_dev_err(dd, "Counter %s not enabled", entry->name); 12393 return 0; 12394 } 12395 12396 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 12397 12398 if (entry->flags & CNTR_SYNTH) { 12399 *psval = data; 12400 if (entry->flags & CNTR_32BIT) { 12401 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 12402 (data << 32) >> 32); 12403 val = data; /* return the full 64bit value */ 12404 } else { 12405 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 12406 data); 12407 } 12408 } else { 12409 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data); 12410 } 12411 12412 *psval = val; 12413 12414 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 12415 12416 return val; 12417 } 12418 12419 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl) 12420 { 12421 struct cntr_entry *entry; 12422 u64 *sval; 12423 12424 entry = &dev_cntrs[index]; 12425 sval = dd->scntrs + entry->offset; 12426 12427 if (vl != CNTR_INVALID_VL) 12428 sval += vl; 12429 12430 return read_dev_port_cntr(dd, entry, sval, dd, vl); 12431 } 12432 12433 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data) 12434 { 12435 struct cntr_entry *entry; 12436 u64 *sval; 12437 12438 entry = &dev_cntrs[index]; 12439 sval = dd->scntrs + entry->offset; 12440 12441 if (vl != CNTR_INVALID_VL) 12442 sval += vl; 12443 12444 return write_dev_port_cntr(dd, entry, sval, dd, vl, data); 12445 } 12446 12447 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl) 12448 { 12449 struct cntr_entry *entry; 12450 u64 *sval; 12451 12452 entry = &port_cntrs[index]; 12453 sval = ppd->scntrs + entry->offset; 12454 12455 if (vl != CNTR_INVALID_VL) 12456 sval += vl; 12457 12458 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 12459 (index <= C_RCV_HDR_OVF_LAST)) { 12460 /* We do not want to bother for disabled contexts */ 12461 return 0; 12462 } 12463 12464 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl); 12465 } 12466 12467 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data) 12468 { 12469 struct cntr_entry *entry; 12470 u64 *sval; 12471 12472 entry = &port_cntrs[index]; 12473 sval = ppd->scntrs + entry->offset; 12474 12475 if (vl != CNTR_INVALID_VL) 12476 sval += vl; 12477 12478 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 12479 (index <= C_RCV_HDR_OVF_LAST)) { 12480 /* We do not want to bother for disabled contexts */ 12481 return 0; 12482 } 12483 12484 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data); 12485 } 12486 12487 static void do_update_synth_timer(struct work_struct *work) 12488 { 12489 u64 cur_tx; 12490 u64 cur_rx; 12491 u64 total_flits; 12492 u8 update = 0; 12493 int i, j, vl; 12494 struct hfi1_pportdata *ppd; 12495 struct cntr_entry *entry; 12496 struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata, 12497 update_cntr_work); 12498 12499 /* 12500 * Rather than keep beating on the CSRs pick a minimal set that we can 12501 * check to watch for potential roll over. We can do this by looking at 12502 * the number of flits sent/recv. If the total flits exceeds 32bits then 12503 * we have to iterate all the counters and update. 12504 */ 12505 entry = &dev_cntrs[C_DC_RCV_FLITS]; 12506 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 12507 12508 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 12509 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 12510 12511 hfi1_cdbg( 12512 CNTR, 12513 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n", 12514 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx); 12515 12516 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) { 12517 /* 12518 * May not be strictly necessary to update but it won't hurt and 12519 * simplifies the logic here. 12520 */ 12521 update = 1; 12522 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating", 12523 dd->unit); 12524 } else { 12525 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx); 12526 hfi1_cdbg(CNTR, 12527 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit, 12528 total_flits, (u64)CNTR_32BIT_MAX); 12529 if (total_flits >= CNTR_32BIT_MAX) { 12530 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating", 12531 dd->unit); 12532 update = 1; 12533 } 12534 } 12535 12536 if (update) { 12537 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit); 12538 for (i = 0; i < DEV_CNTR_LAST; i++) { 12539 entry = &dev_cntrs[i]; 12540 if (entry->flags & CNTR_VL) { 12541 for (vl = 0; vl < C_VL_COUNT; vl++) 12542 read_dev_cntr(dd, i, vl); 12543 } else { 12544 read_dev_cntr(dd, i, CNTR_INVALID_VL); 12545 } 12546 } 12547 ppd = (struct hfi1_pportdata *)(dd + 1); 12548 for (i = 0; i < dd->num_pports; i++, ppd++) { 12549 for (j = 0; j < PORT_CNTR_LAST; j++) { 12550 entry = &port_cntrs[j]; 12551 if (entry->flags & CNTR_VL) { 12552 for (vl = 0; vl < C_VL_COUNT; vl++) 12553 read_port_cntr(ppd, j, vl); 12554 } else { 12555 read_port_cntr(ppd, j, CNTR_INVALID_VL); 12556 } 12557 } 12558 } 12559 12560 /* 12561 * We want the value in the register. The goal is to keep track 12562 * of the number of "ticks" not the counter value. In other 12563 * words if the register rolls we want to notice it and go ahead 12564 * and force an update. 12565 */ 12566 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 12567 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 12568 CNTR_MODE_R, 0); 12569 12570 entry = &dev_cntrs[C_DC_RCV_FLITS]; 12571 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 12572 CNTR_MODE_R, 0); 12573 12574 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx", 12575 dd->unit, dd->last_tx, dd->last_rx); 12576 12577 } else { 12578 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit); 12579 } 12580 } 12581 12582 static void update_synth_timer(struct timer_list *t) 12583 { 12584 struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer); 12585 12586 queue_work(dd->update_cntr_wq, &dd->update_cntr_work); 12587 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12588 } 12589 12590 #define C_MAX_NAME 16 /* 15 chars + one for /0 */ 12591 static int init_cntrs(struct hfi1_devdata *dd) 12592 { 12593 int i, rcv_ctxts, j; 12594 size_t sz; 12595 char *p; 12596 char name[C_MAX_NAME]; 12597 struct hfi1_pportdata *ppd; 12598 const char *bit_type_32 = ",32"; 12599 const int bit_type_32_sz = strlen(bit_type_32); 12600 u32 sdma_engines = chip_sdma_engines(dd); 12601 12602 /* set up the stats timer; the add_timer is done at the end */ 12603 timer_setup(&dd->synth_stats_timer, update_synth_timer, 0); 12604 12605 /***********************/ 12606 /* per device counters */ 12607 /***********************/ 12608 12609 /* size names and determine how many we have*/ 12610 dd->ndevcntrs = 0; 12611 sz = 0; 12612 12613 for (i = 0; i < DEV_CNTR_LAST; i++) { 12614 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12615 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name); 12616 continue; 12617 } 12618 12619 if (dev_cntrs[i].flags & CNTR_VL) { 12620 dev_cntrs[i].offset = dd->ndevcntrs; 12621 for (j = 0; j < C_VL_COUNT; j++) { 12622 snprintf(name, C_MAX_NAME, "%s%d", 12623 dev_cntrs[i].name, vl_from_idx(j)); 12624 sz += strlen(name); 12625 /* Add ",32" for 32-bit counters */ 12626 if (dev_cntrs[i].flags & CNTR_32BIT) 12627 sz += bit_type_32_sz; 12628 sz++; 12629 dd->ndevcntrs++; 12630 } 12631 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12632 dev_cntrs[i].offset = dd->ndevcntrs; 12633 for (j = 0; j < sdma_engines; j++) { 12634 snprintf(name, C_MAX_NAME, "%s%d", 12635 dev_cntrs[i].name, j); 12636 sz += strlen(name); 12637 /* Add ",32" for 32-bit counters */ 12638 if (dev_cntrs[i].flags & CNTR_32BIT) 12639 sz += bit_type_32_sz; 12640 sz++; 12641 dd->ndevcntrs++; 12642 } 12643 } else { 12644 /* +1 for newline. */ 12645 sz += strlen(dev_cntrs[i].name) + 1; 12646 /* Add ",32" for 32-bit counters */ 12647 if (dev_cntrs[i].flags & CNTR_32BIT) 12648 sz += bit_type_32_sz; 12649 dev_cntrs[i].offset = dd->ndevcntrs; 12650 dd->ndevcntrs++; 12651 } 12652 } 12653 12654 /* allocate space for the counter values */ 12655 dd->cntrs = kcalloc(dd->ndevcntrs + num_driver_cntrs, sizeof(u64), 12656 GFP_KERNEL); 12657 if (!dd->cntrs) 12658 goto bail; 12659 12660 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL); 12661 if (!dd->scntrs) 12662 goto bail; 12663 12664 /* allocate space for the counter names */ 12665 dd->cntrnameslen = sz; 12666 dd->cntrnames = kmalloc(sz, GFP_KERNEL); 12667 if (!dd->cntrnames) 12668 goto bail; 12669 12670 /* fill in the names */ 12671 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) { 12672 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12673 /* Nothing */ 12674 } else if (dev_cntrs[i].flags & CNTR_VL) { 12675 for (j = 0; j < C_VL_COUNT; j++) { 12676 snprintf(name, C_MAX_NAME, "%s%d", 12677 dev_cntrs[i].name, 12678 vl_from_idx(j)); 12679 memcpy(p, name, strlen(name)); 12680 p += strlen(name); 12681 12682 /* Counter is 32 bits */ 12683 if (dev_cntrs[i].flags & CNTR_32BIT) { 12684 memcpy(p, bit_type_32, bit_type_32_sz); 12685 p += bit_type_32_sz; 12686 } 12687 12688 *p++ = '\n'; 12689 } 12690 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12691 for (j = 0; j < sdma_engines; j++) { 12692 snprintf(name, C_MAX_NAME, "%s%d", 12693 dev_cntrs[i].name, j); 12694 memcpy(p, name, strlen(name)); 12695 p += strlen(name); 12696 12697 /* Counter is 32 bits */ 12698 if (dev_cntrs[i].flags & CNTR_32BIT) { 12699 memcpy(p, bit_type_32, bit_type_32_sz); 12700 p += bit_type_32_sz; 12701 } 12702 12703 *p++ = '\n'; 12704 } 12705 } else { 12706 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name)); 12707 p += strlen(dev_cntrs[i].name); 12708 12709 /* Counter is 32 bits */ 12710 if (dev_cntrs[i].flags & CNTR_32BIT) { 12711 memcpy(p, bit_type_32, bit_type_32_sz); 12712 p += bit_type_32_sz; 12713 } 12714 12715 *p++ = '\n'; 12716 } 12717 } 12718 12719 /*********************/ 12720 /* per port counters */ 12721 /*********************/ 12722 12723 /* 12724 * Go through the counters for the overflows and disable the ones we 12725 * don't need. This varies based on platform so we need to do it 12726 * dynamically here. 12727 */ 12728 rcv_ctxts = dd->num_rcv_contexts; 12729 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts; 12730 i <= C_RCV_HDR_OVF_LAST; i++) { 12731 port_cntrs[i].flags |= CNTR_DISABLED; 12732 } 12733 12734 /* size port counter names and determine how many we have*/ 12735 sz = 0; 12736 dd->nportcntrs = 0; 12737 for (i = 0; i < PORT_CNTR_LAST; i++) { 12738 if (port_cntrs[i].flags & CNTR_DISABLED) { 12739 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name); 12740 continue; 12741 } 12742 12743 if (port_cntrs[i].flags & CNTR_VL) { 12744 port_cntrs[i].offset = dd->nportcntrs; 12745 for (j = 0; j < C_VL_COUNT; j++) { 12746 snprintf(name, C_MAX_NAME, "%s%d", 12747 port_cntrs[i].name, vl_from_idx(j)); 12748 sz += strlen(name); 12749 /* Add ",32" for 32-bit counters */ 12750 if (port_cntrs[i].flags & CNTR_32BIT) 12751 sz += bit_type_32_sz; 12752 sz++; 12753 dd->nportcntrs++; 12754 } 12755 } else { 12756 /* +1 for newline */ 12757 sz += strlen(port_cntrs[i].name) + 1; 12758 /* Add ",32" for 32-bit counters */ 12759 if (port_cntrs[i].flags & CNTR_32BIT) 12760 sz += bit_type_32_sz; 12761 port_cntrs[i].offset = dd->nportcntrs; 12762 dd->nportcntrs++; 12763 } 12764 } 12765 12766 /* allocate space for the counter names */ 12767 dd->portcntrnameslen = sz; 12768 dd->portcntrnames = kmalloc(sz, GFP_KERNEL); 12769 if (!dd->portcntrnames) 12770 goto bail; 12771 12772 /* fill in port cntr names */ 12773 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) { 12774 if (port_cntrs[i].flags & CNTR_DISABLED) 12775 continue; 12776 12777 if (port_cntrs[i].flags & CNTR_VL) { 12778 for (j = 0; j < C_VL_COUNT; j++) { 12779 snprintf(name, C_MAX_NAME, "%s%d", 12780 port_cntrs[i].name, vl_from_idx(j)); 12781 memcpy(p, name, strlen(name)); 12782 p += strlen(name); 12783 12784 /* Counter is 32 bits */ 12785 if (port_cntrs[i].flags & CNTR_32BIT) { 12786 memcpy(p, bit_type_32, bit_type_32_sz); 12787 p += bit_type_32_sz; 12788 } 12789 12790 *p++ = '\n'; 12791 } 12792 } else { 12793 memcpy(p, port_cntrs[i].name, 12794 strlen(port_cntrs[i].name)); 12795 p += strlen(port_cntrs[i].name); 12796 12797 /* Counter is 32 bits */ 12798 if (port_cntrs[i].flags & CNTR_32BIT) { 12799 memcpy(p, bit_type_32, bit_type_32_sz); 12800 p += bit_type_32_sz; 12801 } 12802 12803 *p++ = '\n'; 12804 } 12805 } 12806 12807 /* allocate per port storage for counter values */ 12808 ppd = (struct hfi1_pportdata *)(dd + 1); 12809 for (i = 0; i < dd->num_pports; i++, ppd++) { 12810 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12811 if (!ppd->cntrs) 12812 goto bail; 12813 12814 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12815 if (!ppd->scntrs) 12816 goto bail; 12817 } 12818 12819 /* CPU counters need to be allocated and zeroed */ 12820 if (init_cpu_counters(dd)) 12821 goto bail; 12822 12823 dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d", 12824 WQ_MEM_RECLAIM, dd->unit); 12825 if (!dd->update_cntr_wq) 12826 goto bail; 12827 12828 INIT_WORK(&dd->update_cntr_work, do_update_synth_timer); 12829 12830 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12831 return 0; 12832 bail: 12833 free_cntrs(dd); 12834 return -ENOMEM; 12835 } 12836 12837 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) 12838 { 12839 switch (chip_lstate) { 12840 case LSTATE_DOWN: 12841 return IB_PORT_DOWN; 12842 case LSTATE_INIT: 12843 return IB_PORT_INIT; 12844 case LSTATE_ARMED: 12845 return IB_PORT_ARMED; 12846 case LSTATE_ACTIVE: 12847 return IB_PORT_ACTIVE; 12848 default: 12849 dd_dev_err(dd, 12850 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n", 12851 chip_lstate); 12852 return IB_PORT_DOWN; 12853 } 12854 } 12855 12856 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) 12857 { 12858 /* look at the HFI meta-states only */ 12859 switch (chip_pstate & 0xf0) { 12860 case PLS_DISABLED: 12861 return IB_PORTPHYSSTATE_DISABLED; 12862 case PLS_OFFLINE: 12863 return OPA_PORTPHYSSTATE_OFFLINE; 12864 case PLS_POLLING: 12865 return IB_PORTPHYSSTATE_POLLING; 12866 case PLS_CONFIGPHY: 12867 return IB_PORTPHYSSTATE_TRAINING; 12868 case PLS_LINKUP: 12869 return IB_PORTPHYSSTATE_LINKUP; 12870 case PLS_PHYTEST: 12871 return IB_PORTPHYSSTATE_PHY_TEST; 12872 default: 12873 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n", 12874 chip_pstate); 12875 return IB_PORTPHYSSTATE_DISABLED; 12876 } 12877 } 12878 12879 /* return the OPA port logical state name */ 12880 const char *opa_lstate_name(u32 lstate) 12881 { 12882 static const char * const port_logical_names[] = { 12883 "PORT_NOP", 12884 "PORT_DOWN", 12885 "PORT_INIT", 12886 "PORT_ARMED", 12887 "PORT_ACTIVE", 12888 "PORT_ACTIVE_DEFER", 12889 }; 12890 if (lstate < ARRAY_SIZE(port_logical_names)) 12891 return port_logical_names[lstate]; 12892 return "unknown"; 12893 } 12894 12895 /* return the OPA port physical state name */ 12896 const char *opa_pstate_name(u32 pstate) 12897 { 12898 static const char * const port_physical_names[] = { 12899 "PHYS_NOP", 12900 "reserved1", 12901 "PHYS_POLL", 12902 "PHYS_DISABLED", 12903 "PHYS_TRAINING", 12904 "PHYS_LINKUP", 12905 "PHYS_LINK_ERR_RECOVER", 12906 "PHYS_PHY_TEST", 12907 "reserved8", 12908 "PHYS_OFFLINE", 12909 "PHYS_GANGED", 12910 "PHYS_TEST", 12911 }; 12912 if (pstate < ARRAY_SIZE(port_physical_names)) 12913 return port_physical_names[pstate]; 12914 return "unknown"; 12915 } 12916 12917 /** 12918 * update_statusp - Update userspace status flag 12919 * @ppd: Port data structure 12920 * @state: port state information 12921 * 12922 * Actual port status is determined by the host_link_state value 12923 * in the ppd. 12924 * 12925 * host_link_state MUST be updated before updating the user space 12926 * statusp. 12927 */ 12928 static void update_statusp(struct hfi1_pportdata *ppd, u32 state) 12929 { 12930 /* 12931 * Set port status flags in the page mapped into userspace 12932 * memory. Do it here to ensure a reliable state - this is 12933 * the only function called by all state handling code. 12934 * Always set the flags due to the fact that the cache value 12935 * might have been changed explicitly outside of this 12936 * function. 12937 */ 12938 if (ppd->statusp) { 12939 switch (state) { 12940 case IB_PORT_DOWN: 12941 case IB_PORT_INIT: 12942 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF | 12943 HFI1_STATUS_IB_READY); 12944 break; 12945 case IB_PORT_ARMED: 12946 *ppd->statusp |= HFI1_STATUS_IB_CONF; 12947 break; 12948 case IB_PORT_ACTIVE: 12949 *ppd->statusp |= HFI1_STATUS_IB_READY; 12950 break; 12951 } 12952 } 12953 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n", 12954 opa_lstate_name(state), state); 12955 } 12956 12957 /** 12958 * wait_logical_linkstate - wait for an IB link state change to occur 12959 * @ppd: port device 12960 * @state: the state to wait for 12961 * @msecs: the number of milliseconds to wait 12962 * 12963 * Wait up to msecs milliseconds for IB link state change to occur. 12964 * For now, take the easy polling route. 12965 * Returns 0 if state reached, otherwise -ETIMEDOUT. 12966 */ 12967 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 12968 int msecs) 12969 { 12970 unsigned long timeout; 12971 u32 new_state; 12972 12973 timeout = jiffies + msecs_to_jiffies(msecs); 12974 while (1) { 12975 new_state = chip_to_opa_lstate(ppd->dd, 12976 read_logical_state(ppd->dd)); 12977 if (new_state == state) 12978 break; 12979 if (time_after(jiffies, timeout)) { 12980 dd_dev_err(ppd->dd, 12981 "timeout waiting for link state 0x%x\n", 12982 state); 12983 return -ETIMEDOUT; 12984 } 12985 msleep(20); 12986 } 12987 12988 return 0; 12989 } 12990 12991 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state) 12992 { 12993 u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state); 12994 12995 dd_dev_info(ppd->dd, 12996 "physical state changed to %s (0x%x), phy 0x%x\n", 12997 opa_pstate_name(ib_pstate), ib_pstate, state); 12998 } 12999 13000 /* 13001 * Read the physical hardware link state and check if it matches host 13002 * drivers anticipated state. 13003 */ 13004 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state) 13005 { 13006 u32 read_state = read_physical_state(ppd->dd); 13007 13008 if (read_state == state) { 13009 log_state_transition(ppd, state); 13010 } else { 13011 dd_dev_err(ppd->dd, 13012 "anticipated phy link state 0x%x, read 0x%x\n", 13013 state, read_state); 13014 } 13015 } 13016 13017 /* 13018 * wait_physical_linkstate - wait for an physical link state change to occur 13019 * @ppd: port device 13020 * @state: the state to wait for 13021 * @msecs: the number of milliseconds to wait 13022 * 13023 * Wait up to msecs milliseconds for physical link state change to occur. 13024 * Returns 0 if state reached, otherwise -ETIMEDOUT. 13025 */ 13026 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, 13027 int msecs) 13028 { 13029 u32 read_state; 13030 unsigned long timeout; 13031 13032 timeout = jiffies + msecs_to_jiffies(msecs); 13033 while (1) { 13034 read_state = read_physical_state(ppd->dd); 13035 if (read_state == state) 13036 break; 13037 if (time_after(jiffies, timeout)) { 13038 dd_dev_err(ppd->dd, 13039 "timeout waiting for phy link state 0x%x\n", 13040 state); 13041 return -ETIMEDOUT; 13042 } 13043 usleep_range(1950, 2050); /* sleep 2ms-ish */ 13044 } 13045 13046 log_state_transition(ppd, state); 13047 return 0; 13048 } 13049 13050 /* 13051 * wait_phys_link_offline_quiet_substates - wait for any offline substate 13052 * @ppd: port device 13053 * @msecs: the number of milliseconds to wait 13054 * 13055 * Wait up to msecs milliseconds for any offline physical link 13056 * state change to occur. 13057 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. 13058 */ 13059 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, 13060 int msecs) 13061 { 13062 u32 read_state; 13063 unsigned long timeout; 13064 13065 timeout = jiffies + msecs_to_jiffies(msecs); 13066 while (1) { 13067 read_state = read_physical_state(ppd->dd); 13068 if ((read_state & 0xF0) == PLS_OFFLINE) 13069 break; 13070 if (time_after(jiffies, timeout)) { 13071 dd_dev_err(ppd->dd, 13072 "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n", 13073 read_state, msecs); 13074 return -ETIMEDOUT; 13075 } 13076 usleep_range(1950, 2050); /* sleep 2ms-ish */ 13077 } 13078 13079 log_state_transition(ppd, read_state); 13080 return read_state; 13081 } 13082 13083 /* 13084 * wait_phys_link_out_of_offline - wait for any out of offline state 13085 * @ppd: port device 13086 * @msecs: the number of milliseconds to wait 13087 * 13088 * Wait up to msecs milliseconds for any out of offline physical link 13089 * state change to occur. 13090 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. 13091 */ 13092 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd, 13093 int msecs) 13094 { 13095 u32 read_state; 13096 unsigned long timeout; 13097 13098 timeout = jiffies + msecs_to_jiffies(msecs); 13099 while (1) { 13100 read_state = read_physical_state(ppd->dd); 13101 if ((read_state & 0xF0) != PLS_OFFLINE) 13102 break; 13103 if (time_after(jiffies, timeout)) { 13104 dd_dev_err(ppd->dd, 13105 "timeout waiting for phy link out of offline. Read state 0x%x, %dms\n", 13106 read_state, msecs); 13107 return -ETIMEDOUT; 13108 } 13109 usleep_range(1950, 2050); /* sleep 2ms-ish */ 13110 } 13111 13112 log_state_transition(ppd, read_state); 13113 return read_state; 13114 } 13115 13116 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \ 13117 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 13118 13119 #define SET_STATIC_RATE_CONTROL_SMASK(r) \ 13120 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 13121 13122 void hfi1_init_ctxt(struct send_context *sc) 13123 { 13124 if (sc) { 13125 struct hfi1_devdata *dd = sc->dd; 13126 u64 reg; 13127 u8 set = (sc->type == SC_USER ? 13128 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) : 13129 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL)); 13130 reg = read_kctxt_csr(dd, sc->hw_context, 13131 SEND_CTXT_CHECK_ENABLE); 13132 if (set) 13133 CLEAR_STATIC_RATE_CONTROL_SMASK(reg); 13134 else 13135 SET_STATIC_RATE_CONTROL_SMASK(reg); 13136 write_kctxt_csr(dd, sc->hw_context, 13137 SEND_CTXT_CHECK_ENABLE, reg); 13138 } 13139 } 13140 13141 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp) 13142 { 13143 int ret = 0; 13144 u64 reg; 13145 13146 if (dd->icode != ICODE_RTL_SILICON) { 13147 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 13148 dd_dev_info(dd, "%s: tempsense not supported by HW\n", 13149 __func__); 13150 return -EINVAL; 13151 } 13152 reg = read_csr(dd, ASIC_STS_THERM); 13153 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) & 13154 ASIC_STS_THERM_CURR_TEMP_MASK); 13155 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) & 13156 ASIC_STS_THERM_LO_TEMP_MASK); 13157 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) & 13158 ASIC_STS_THERM_HI_TEMP_MASK); 13159 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) & 13160 ASIC_STS_THERM_CRIT_TEMP_MASK); 13161 /* triggers is a 3-bit value - 1 bit per trigger. */ 13162 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7); 13163 13164 return ret; 13165 } 13166 13167 /* ========================================================================= */ 13168 13169 /** 13170 * read_mod_write() - Calculate the IRQ register index and set/clear the bits 13171 * @dd: valid devdata 13172 * @src: IRQ source to determine register index from 13173 * @bits: the bits to set or clear 13174 * @set: true == set the bits, false == clear the bits 13175 * 13176 */ 13177 static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits, 13178 bool set) 13179 { 13180 u64 reg; 13181 u16 idx = src / BITS_PER_REGISTER; 13182 13183 spin_lock(&dd->irq_src_lock); 13184 reg = read_csr(dd, CCE_INT_MASK + (8 * idx)); 13185 if (set) 13186 reg |= bits; 13187 else 13188 reg &= ~bits; 13189 write_csr(dd, CCE_INT_MASK + (8 * idx), reg); 13190 spin_unlock(&dd->irq_src_lock); 13191 } 13192 13193 /** 13194 * set_intr_bits() - Enable/disable a range (one or more) IRQ sources 13195 * @dd: valid devdata 13196 * @first: first IRQ source to set/clear 13197 * @last: last IRQ source (inclusive) to set/clear 13198 * @set: true == set the bits, false == clear the bits 13199 * 13200 * If first == last, set the exact source. 13201 */ 13202 int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set) 13203 { 13204 u64 bits = 0; 13205 u64 bit; 13206 u16 src; 13207 13208 if (first > NUM_INTERRUPT_SOURCES || last > NUM_INTERRUPT_SOURCES) 13209 return -EINVAL; 13210 13211 if (last < first) 13212 return -ERANGE; 13213 13214 for (src = first; src <= last; src++) { 13215 bit = src % BITS_PER_REGISTER; 13216 /* wrapped to next register? */ 13217 if (!bit && bits) { 13218 read_mod_write(dd, src - 1, bits, set); 13219 bits = 0; 13220 } 13221 bits |= BIT_ULL(bit); 13222 } 13223 read_mod_write(dd, last, bits, set); 13224 13225 return 0; 13226 } 13227 13228 /* 13229 * Clear all interrupt sources on the chip. 13230 */ 13231 void clear_all_interrupts(struct hfi1_devdata *dd) 13232 { 13233 int i; 13234 13235 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 13236 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0); 13237 13238 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0); 13239 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0); 13240 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0); 13241 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0); 13242 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0); 13243 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0); 13244 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0); 13245 for (i = 0; i < chip_send_contexts(dd); i++) 13246 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0); 13247 for (i = 0; i < chip_sdma_engines(dd); i++) 13248 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0); 13249 13250 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0); 13251 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0); 13252 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0); 13253 } 13254 13255 /* 13256 * Remap the interrupt source from the general handler to the given MSI-X 13257 * interrupt. 13258 */ 13259 void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr) 13260 { 13261 u64 reg; 13262 int m, n; 13263 13264 /* clear from the handled mask of the general interrupt */ 13265 m = isrc / 64; 13266 n = isrc % 64; 13267 if (likely(m < CCE_NUM_INT_CSRS)) { 13268 dd->gi_mask[m] &= ~((u64)1 << n); 13269 } else { 13270 dd_dev_err(dd, "remap interrupt err\n"); 13271 return; 13272 } 13273 13274 /* direct the chip source to the given MSI-X interrupt */ 13275 m = isrc / 8; 13276 n = isrc % 8; 13277 reg = read_csr(dd, CCE_INT_MAP + (8 * m)); 13278 reg &= ~((u64)0xff << (8 * n)); 13279 reg |= ((u64)msix_intr & 0xff) << (8 * n); 13280 write_csr(dd, CCE_INT_MAP + (8 * m), reg); 13281 } 13282 13283 void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr) 13284 { 13285 /* 13286 * SDMA engine interrupt sources grouped by type, rather than 13287 * engine. Per-engine interrupts are as follows: 13288 * SDMA 13289 * SDMAProgress 13290 * SDMAIdle 13291 */ 13292 remap_intr(dd, IS_SDMA_START + engine, msix_intr); 13293 remap_intr(dd, IS_SDMA_PROGRESS_START + engine, msix_intr); 13294 remap_intr(dd, IS_SDMA_IDLE_START + engine, msix_intr); 13295 } 13296 13297 /* 13298 * Set the general handler to accept all interrupts, remap all 13299 * chip interrupts back to MSI-X 0. 13300 */ 13301 void reset_interrupts(struct hfi1_devdata *dd) 13302 { 13303 int i; 13304 13305 /* all interrupts handled by the general handler */ 13306 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 13307 dd->gi_mask[i] = ~(u64)0; 13308 13309 /* all chip interrupts map to MSI-X 0 */ 13310 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13311 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 13312 } 13313 13314 /** 13315 * set_up_interrupts() - Initialize the IRQ resources and state 13316 * @dd: valid devdata 13317 * 13318 */ 13319 static int set_up_interrupts(struct hfi1_devdata *dd) 13320 { 13321 int ret; 13322 13323 /* mask all interrupts */ 13324 set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false); 13325 13326 /* clear all pending interrupts */ 13327 clear_all_interrupts(dd); 13328 13329 /* reset general handler mask, chip MSI-X mappings */ 13330 reset_interrupts(dd); 13331 13332 /* ask for MSI-X interrupts */ 13333 ret = msix_initialize(dd); 13334 if (ret) 13335 return ret; 13336 13337 ret = msix_request_irqs(dd); 13338 if (ret) 13339 msix_clean_up_interrupts(dd); 13340 13341 return ret; 13342 } 13343 13344 /* 13345 * Set up context values in dd. Sets: 13346 * 13347 * num_rcv_contexts - number of contexts being used 13348 * n_krcv_queues - number of kernel contexts 13349 * first_dyn_alloc_ctxt - first dynamically allocated context 13350 * in array of contexts 13351 * freectxts - number of free user contexts 13352 * num_send_contexts - number of PIO send contexts being used 13353 * num_netdev_contexts - number of contexts reserved for netdev 13354 */ 13355 static int set_up_context_variables(struct hfi1_devdata *dd) 13356 { 13357 unsigned long num_kernel_contexts; 13358 u16 num_netdev_contexts; 13359 int ret; 13360 unsigned ngroups; 13361 int rmt_count; 13362 int user_rmt_reduced; 13363 u32 n_usr_ctxts; 13364 u32 send_contexts = chip_send_contexts(dd); 13365 u32 rcv_contexts = chip_rcv_contexts(dd); 13366 13367 /* 13368 * Kernel receive contexts: 13369 * - Context 0 - control context (VL15/multicast/error) 13370 * - Context 1 - first kernel context 13371 * - Context 2 - second kernel context 13372 * ... 13373 */ 13374 if (n_krcvqs) 13375 /* 13376 * n_krcvqs is the sum of module parameter kernel receive 13377 * contexts, krcvqs[]. It does not include the control 13378 * context, so add that. 13379 */ 13380 num_kernel_contexts = n_krcvqs + 1; 13381 else 13382 num_kernel_contexts = DEFAULT_KRCVQS + 1; 13383 /* 13384 * Every kernel receive context needs an ACK send context. 13385 * one send context is allocated for each VL{0-7} and VL15 13386 */ 13387 if (num_kernel_contexts > (send_contexts - num_vls - 1)) { 13388 dd_dev_err(dd, 13389 "Reducing # kernel rcv contexts to: %d, from %lu\n", 13390 send_contexts - num_vls - 1, 13391 num_kernel_contexts); 13392 num_kernel_contexts = send_contexts - num_vls - 1; 13393 } 13394 13395 /* 13396 * User contexts: 13397 * - default to 1 user context per real (non-HT) CPU core if 13398 * num_user_contexts is negative 13399 */ 13400 if (num_user_contexts < 0) 13401 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask); 13402 else 13403 n_usr_ctxts = num_user_contexts; 13404 /* 13405 * Adjust the counts given a global max. 13406 */ 13407 if (num_kernel_contexts + n_usr_ctxts > rcv_contexts) { 13408 dd_dev_err(dd, 13409 "Reducing # user receive contexts to: %u, from %u\n", 13410 (u32)(rcv_contexts - num_kernel_contexts), 13411 n_usr_ctxts); 13412 /* recalculate */ 13413 n_usr_ctxts = rcv_contexts - num_kernel_contexts; 13414 } 13415 13416 num_netdev_contexts = 13417 hfi1_num_netdev_contexts(dd, rcv_contexts - 13418 (num_kernel_contexts + n_usr_ctxts), 13419 &node_affinity.real_cpu_mask); 13420 /* 13421 * The RMT entries are currently allocated as shown below: 13422 * 1. QOS (0 to 128 entries); 13423 * 2. FECN (num_kernel_context - 1 + num_user_contexts + 13424 * num_netdev_contexts); 13425 * 3. netdev (num_netdev_contexts). 13426 * It should be noted that FECN oversubscribe num_netdev_contexts 13427 * entries of RMT because both netdev and PSM could allocate any receive 13428 * context between dd->first_dyn_alloc_text and dd->num_rcv_contexts, 13429 * and PSM FECN must reserve an RMT entry for each possible PSM receive 13430 * context. 13431 */ 13432 rmt_count = qos_rmt_entries(dd, NULL, NULL) + (num_netdev_contexts * 2); 13433 if (HFI1_CAP_IS_KSET(TID_RDMA)) 13434 rmt_count += num_kernel_contexts - 1; 13435 if (rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) { 13436 user_rmt_reduced = NUM_MAP_ENTRIES - rmt_count; 13437 dd_dev_err(dd, 13438 "RMT size is reducing the number of user receive contexts from %u to %d\n", 13439 n_usr_ctxts, 13440 user_rmt_reduced); 13441 /* recalculate */ 13442 n_usr_ctxts = user_rmt_reduced; 13443 } 13444 13445 /* the first N are kernel contexts, the rest are user/netdev contexts */ 13446 dd->num_rcv_contexts = 13447 num_kernel_contexts + n_usr_ctxts + num_netdev_contexts; 13448 dd->n_krcv_queues = num_kernel_contexts; 13449 dd->first_dyn_alloc_ctxt = num_kernel_contexts; 13450 dd->num_netdev_contexts = num_netdev_contexts; 13451 dd->num_user_contexts = n_usr_ctxts; 13452 dd->freectxts = n_usr_ctxts; 13453 dd_dev_info(dd, 13454 "rcv contexts: chip %d, used %d (kernel %d, netdev %u, user %u)\n", 13455 rcv_contexts, 13456 (int)dd->num_rcv_contexts, 13457 (int)dd->n_krcv_queues, 13458 dd->num_netdev_contexts, 13459 dd->num_user_contexts); 13460 13461 /* 13462 * Receive array allocation: 13463 * All RcvArray entries are divided into groups of 8. This 13464 * is required by the hardware and will speed up writes to 13465 * consecutive entries by using write-combining of the entire 13466 * cacheline. 13467 * 13468 * The number of groups are evenly divided among all contexts. 13469 * any left over groups will be given to the first N user 13470 * contexts. 13471 */ 13472 dd->rcv_entries.group_size = RCV_INCREMENT; 13473 ngroups = chip_rcv_array_count(dd) / dd->rcv_entries.group_size; 13474 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts; 13475 dd->rcv_entries.nctxt_extra = ngroups - 13476 (dd->num_rcv_contexts * dd->rcv_entries.ngroups); 13477 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n", 13478 dd->rcv_entries.ngroups, 13479 dd->rcv_entries.nctxt_extra); 13480 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size > 13481 MAX_EAGER_ENTRIES * 2) { 13482 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) / 13483 dd->rcv_entries.group_size; 13484 dd_dev_info(dd, 13485 "RcvArray group count too high, change to %u\n", 13486 dd->rcv_entries.ngroups); 13487 dd->rcv_entries.nctxt_extra = 0; 13488 } 13489 /* 13490 * PIO send contexts 13491 */ 13492 ret = init_sc_pools_and_sizes(dd); 13493 if (ret >= 0) { /* success */ 13494 dd->num_send_contexts = ret; 13495 dd_dev_info( 13496 dd, 13497 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n", 13498 send_contexts, 13499 dd->num_send_contexts, 13500 dd->sc_sizes[SC_KERNEL].count, 13501 dd->sc_sizes[SC_ACK].count, 13502 dd->sc_sizes[SC_USER].count, 13503 dd->sc_sizes[SC_VL15].count); 13504 ret = 0; /* success */ 13505 } 13506 13507 return ret; 13508 } 13509 13510 /* 13511 * Set the device/port partition key table. The MAD code 13512 * will ensure that, at least, the partial management 13513 * partition key is present in the table. 13514 */ 13515 static void set_partition_keys(struct hfi1_pportdata *ppd) 13516 { 13517 struct hfi1_devdata *dd = ppd->dd; 13518 u64 reg = 0; 13519 int i; 13520 13521 dd_dev_info(dd, "Setting partition keys\n"); 13522 for (i = 0; i < hfi1_get_npkeys(dd); i++) { 13523 reg |= (ppd->pkeys[i] & 13524 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) << 13525 ((i % 4) * 13526 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT); 13527 /* Each register holds 4 PKey values. */ 13528 if ((i % 4) == 3) { 13529 write_csr(dd, RCV_PARTITION_KEY + 13530 ((i - 3) * 2), reg); 13531 reg = 0; 13532 } 13533 } 13534 13535 /* Always enable HW pkeys check when pkeys table is set */ 13536 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK); 13537 } 13538 13539 /* 13540 * These CSRs and memories are uninitialized on reset and must be 13541 * written before reading to set the ECC/parity bits. 13542 * 13543 * NOTE: All user context CSRs that are not mmaped write-only 13544 * (e.g. the TID flows) must be initialized even if the driver never 13545 * reads them. 13546 */ 13547 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd) 13548 { 13549 int i, j; 13550 13551 /* CceIntMap */ 13552 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13553 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 13554 13555 /* SendCtxtCreditReturnAddr */ 13556 for (i = 0; i < chip_send_contexts(dd); i++) 13557 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13558 13559 /* PIO Send buffers */ 13560 /* SDMA Send buffers */ 13561 /* 13562 * These are not normally read, and (presently) have no method 13563 * to be read, so are not pre-initialized 13564 */ 13565 13566 /* RcvHdrAddr */ 13567 /* RcvHdrTailAddr */ 13568 /* RcvTidFlowTable */ 13569 for (i = 0; i < chip_rcv_contexts(dd); i++) { 13570 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13571 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13572 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) 13573 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0); 13574 } 13575 13576 /* RcvArray */ 13577 for (i = 0; i < chip_rcv_array_count(dd); i++) 13578 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0); 13579 13580 /* RcvQPMapTable */ 13581 for (i = 0; i < 32; i++) 13582 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13583 } 13584 13585 /* 13586 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus. 13587 */ 13588 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits, 13589 u64 ctrl_bits) 13590 { 13591 unsigned long timeout; 13592 u64 reg; 13593 13594 /* is the condition present? */ 13595 reg = read_csr(dd, CCE_STATUS); 13596 if ((reg & status_bits) == 0) 13597 return; 13598 13599 /* clear the condition */ 13600 write_csr(dd, CCE_CTRL, ctrl_bits); 13601 13602 /* wait for the condition to clear */ 13603 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT); 13604 while (1) { 13605 reg = read_csr(dd, CCE_STATUS); 13606 if ((reg & status_bits) == 0) 13607 return; 13608 if (time_after(jiffies, timeout)) { 13609 dd_dev_err(dd, 13610 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n", 13611 status_bits, reg & status_bits); 13612 return; 13613 } 13614 udelay(1); 13615 } 13616 } 13617 13618 /* set CCE CSRs to chip reset defaults */ 13619 static void reset_cce_csrs(struct hfi1_devdata *dd) 13620 { 13621 int i; 13622 13623 /* CCE_REVISION read-only */ 13624 /* CCE_REVISION2 read-only */ 13625 /* CCE_CTRL - bits clear automatically */ 13626 /* CCE_STATUS read-only, use CceCtrl to clear */ 13627 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK); 13628 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK); 13629 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK); 13630 for (i = 0; i < CCE_NUM_SCRATCH; i++) 13631 write_csr(dd, CCE_SCRATCH + (8 * i), 0); 13632 /* CCE_ERR_STATUS read-only */ 13633 write_csr(dd, CCE_ERR_MASK, 0); 13634 write_csr(dd, CCE_ERR_CLEAR, ~0ull); 13635 /* CCE_ERR_FORCE leave alone */ 13636 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++) 13637 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0); 13638 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR); 13639 /* CCE_PCIE_CTRL leave alone */ 13640 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) { 13641 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0); 13642 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i), 13643 CCE_MSIX_TABLE_UPPER_RESETCSR); 13644 } 13645 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) { 13646 /* CCE_MSIX_PBA read-only */ 13647 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull); 13648 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull); 13649 } 13650 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13651 write_csr(dd, CCE_INT_MAP, 0); 13652 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 13653 /* CCE_INT_STATUS read-only */ 13654 write_csr(dd, CCE_INT_MASK + (8 * i), 0); 13655 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull); 13656 /* CCE_INT_FORCE leave alone */ 13657 /* CCE_INT_BLOCKED read-only */ 13658 } 13659 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++) 13660 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0); 13661 } 13662 13663 /* set MISC CSRs to chip reset defaults */ 13664 static void reset_misc_csrs(struct hfi1_devdata *dd) 13665 { 13666 int i; 13667 13668 for (i = 0; i < 32; i++) { 13669 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0); 13670 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0); 13671 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0); 13672 } 13673 /* 13674 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can 13675 * only be written 128-byte chunks 13676 */ 13677 /* init RSA engine to clear lingering errors */ 13678 write_csr(dd, MISC_CFG_RSA_CMD, 1); 13679 write_csr(dd, MISC_CFG_RSA_MU, 0); 13680 write_csr(dd, MISC_CFG_FW_CTRL, 0); 13681 /* MISC_STS_8051_DIGEST read-only */ 13682 /* MISC_STS_SBM_DIGEST read-only */ 13683 /* MISC_STS_PCIE_DIGEST read-only */ 13684 /* MISC_STS_FAB_DIGEST read-only */ 13685 /* MISC_ERR_STATUS read-only */ 13686 write_csr(dd, MISC_ERR_MASK, 0); 13687 write_csr(dd, MISC_ERR_CLEAR, ~0ull); 13688 /* MISC_ERR_FORCE leave alone */ 13689 } 13690 13691 /* set TXE CSRs to chip reset defaults */ 13692 static void reset_txe_csrs(struct hfi1_devdata *dd) 13693 { 13694 int i; 13695 13696 /* 13697 * TXE Kernel CSRs 13698 */ 13699 write_csr(dd, SEND_CTRL, 0); 13700 __cm_reset(dd, 0); /* reset CM internal state */ 13701 /* SEND_CONTEXTS read-only */ 13702 /* SEND_DMA_ENGINES read-only */ 13703 /* SEND_PIO_MEM_SIZE read-only */ 13704 /* SEND_DMA_MEM_SIZE read-only */ 13705 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0); 13706 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */ 13707 /* SEND_PIO_ERR_STATUS read-only */ 13708 write_csr(dd, SEND_PIO_ERR_MASK, 0); 13709 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull); 13710 /* SEND_PIO_ERR_FORCE leave alone */ 13711 /* SEND_DMA_ERR_STATUS read-only */ 13712 write_csr(dd, SEND_DMA_ERR_MASK, 0); 13713 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull); 13714 /* SEND_DMA_ERR_FORCE leave alone */ 13715 /* SEND_EGRESS_ERR_STATUS read-only */ 13716 write_csr(dd, SEND_EGRESS_ERR_MASK, 0); 13717 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull); 13718 /* SEND_EGRESS_ERR_FORCE leave alone */ 13719 write_csr(dd, SEND_BTH_QP, 0); 13720 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0); 13721 write_csr(dd, SEND_SC2VLT0, 0); 13722 write_csr(dd, SEND_SC2VLT1, 0); 13723 write_csr(dd, SEND_SC2VLT2, 0); 13724 write_csr(dd, SEND_SC2VLT3, 0); 13725 write_csr(dd, SEND_LEN_CHECK0, 0); 13726 write_csr(dd, SEND_LEN_CHECK1, 0); 13727 /* SEND_ERR_STATUS read-only */ 13728 write_csr(dd, SEND_ERR_MASK, 0); 13729 write_csr(dd, SEND_ERR_CLEAR, ~0ull); 13730 /* SEND_ERR_FORCE read-only */ 13731 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++) 13732 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0); 13733 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++) 13734 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0); 13735 for (i = 0; i < chip_send_contexts(dd) / NUM_CONTEXTS_PER_SET; i++) 13736 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0); 13737 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++) 13738 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0); 13739 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++) 13740 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0); 13741 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR); 13742 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR); 13743 /* SEND_CM_CREDIT_USED_STATUS read-only */ 13744 write_csr(dd, SEND_CM_TIMER_CTRL, 0); 13745 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0); 13746 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0); 13747 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0); 13748 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0); 13749 for (i = 0; i < TXE_NUM_DATA_VL; i++) 13750 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 13751 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 13752 /* SEND_CM_CREDIT_USED_VL read-only */ 13753 /* SEND_CM_CREDIT_USED_VL15 read-only */ 13754 /* SEND_EGRESS_CTXT_STATUS read-only */ 13755 /* SEND_EGRESS_SEND_DMA_STATUS read-only */ 13756 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull); 13757 /* SEND_EGRESS_ERR_INFO read-only */ 13758 /* SEND_EGRESS_ERR_SOURCE read-only */ 13759 13760 /* 13761 * TXE Per-Context CSRs 13762 */ 13763 for (i = 0; i < chip_send_contexts(dd); i++) { 13764 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 13765 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0); 13766 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13767 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0); 13768 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0); 13769 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull); 13770 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0); 13771 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0); 13772 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0); 13773 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0); 13774 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0); 13775 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0); 13776 } 13777 13778 /* 13779 * TXE Per-SDMA CSRs 13780 */ 13781 for (i = 0; i < chip_sdma_engines(dd); i++) { 13782 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 13783 /* SEND_DMA_STATUS read-only */ 13784 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0); 13785 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0); 13786 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0); 13787 /* SEND_DMA_HEAD read-only */ 13788 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0); 13789 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0); 13790 /* SEND_DMA_IDLE_CNT read-only */ 13791 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0); 13792 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0); 13793 /* SEND_DMA_DESC_FETCHED_CNT read-only */ 13794 /* SEND_DMA_ENG_ERR_STATUS read-only */ 13795 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0); 13796 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull); 13797 /* SEND_DMA_ENG_ERR_FORCE leave alone */ 13798 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0); 13799 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0); 13800 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0); 13801 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0); 13802 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0); 13803 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0); 13804 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0); 13805 } 13806 } 13807 13808 /* 13809 * Expect on entry: 13810 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0 13811 */ 13812 static void init_rbufs(struct hfi1_devdata *dd) 13813 { 13814 u64 reg; 13815 int count; 13816 13817 /* 13818 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are 13819 * clear. 13820 */ 13821 count = 0; 13822 while (1) { 13823 reg = read_csr(dd, RCV_STATUS); 13824 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK 13825 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0) 13826 break; 13827 /* 13828 * Give up after 1ms - maximum wait time. 13829 * 13830 * RBuf size is 136KiB. Slowest possible is PCIe Gen1 x1 at 13831 * 250MB/s bandwidth. Lower rate to 66% for overhead to get: 13832 * 136 KB / (66% * 250MB/s) = 844us 13833 */ 13834 if (count++ > 500) { 13835 dd_dev_err(dd, 13836 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n", 13837 __func__, reg); 13838 break; 13839 } 13840 udelay(2); /* do not busy-wait the CSR */ 13841 } 13842 13843 /* start the init - expect RcvCtrl to be 0 */ 13844 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK); 13845 13846 /* 13847 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief 13848 * period after the write before RcvStatus.RxRbufInitDone is valid. 13849 * The delay in the first run through the loop below is sufficient and 13850 * required before the first read of RcvStatus.RxRbufInintDone. 13851 */ 13852 read_csr(dd, RCV_CTRL); 13853 13854 /* wait for the init to finish */ 13855 count = 0; 13856 while (1) { 13857 /* delay is required first time through - see above */ 13858 udelay(2); /* do not busy-wait the CSR */ 13859 reg = read_csr(dd, RCV_STATUS); 13860 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK)) 13861 break; 13862 13863 /* give up after 100us - slowest possible at 33MHz is 73us */ 13864 if (count++ > 50) { 13865 dd_dev_err(dd, 13866 "%s: RcvStatus.RxRbufInit not set, continuing\n", 13867 __func__); 13868 break; 13869 } 13870 } 13871 } 13872 13873 /* set RXE CSRs to chip reset defaults */ 13874 static void reset_rxe_csrs(struct hfi1_devdata *dd) 13875 { 13876 int i, j; 13877 13878 /* 13879 * RXE Kernel CSRs 13880 */ 13881 write_csr(dd, RCV_CTRL, 0); 13882 init_rbufs(dd); 13883 /* RCV_STATUS read-only */ 13884 /* RCV_CONTEXTS read-only */ 13885 /* RCV_ARRAY_CNT read-only */ 13886 /* RCV_BUF_SIZE read-only */ 13887 write_csr(dd, RCV_BTH_QP, 0); 13888 write_csr(dd, RCV_MULTICAST, 0); 13889 write_csr(dd, RCV_BYPASS, 0); 13890 write_csr(dd, RCV_VL15, 0); 13891 /* this is a clear-down */ 13892 write_csr(dd, RCV_ERR_INFO, 13893 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK); 13894 /* RCV_ERR_STATUS read-only */ 13895 write_csr(dd, RCV_ERR_MASK, 0); 13896 write_csr(dd, RCV_ERR_CLEAR, ~0ull); 13897 /* RCV_ERR_FORCE leave alone */ 13898 for (i = 0; i < 32; i++) 13899 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13900 for (i = 0; i < 4; i++) 13901 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0); 13902 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++) 13903 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0); 13904 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++) 13905 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0); 13906 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) 13907 clear_rsm_rule(dd, i); 13908 for (i = 0; i < 32; i++) 13909 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0); 13910 13911 /* 13912 * RXE Kernel and User Per-Context CSRs 13913 */ 13914 for (i = 0; i < chip_rcv_contexts(dd); i++) { 13915 /* kernel */ 13916 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0); 13917 /* RCV_CTXT_STATUS read-only */ 13918 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0); 13919 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0); 13920 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0); 13921 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13922 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0); 13923 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0); 13924 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0); 13925 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13926 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0); 13927 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0); 13928 13929 /* user */ 13930 /* RCV_HDR_TAIL read-only */ 13931 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0); 13932 /* RCV_EGR_INDEX_TAIL read-only */ 13933 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0); 13934 /* RCV_EGR_OFFSET_TAIL read-only */ 13935 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) { 13936 write_uctxt_csr(dd, i, 13937 RCV_TID_FLOW_TABLE + (8 * j), 0); 13938 } 13939 } 13940 } 13941 13942 /* 13943 * Set sc2vl tables. 13944 * 13945 * They power on to zeros, so to avoid send context errors 13946 * they need to be set: 13947 * 13948 * SC 0-7 -> VL 0-7 (respectively) 13949 * SC 15 -> VL 15 13950 * otherwise 13951 * -> VL 0 13952 */ 13953 static void init_sc2vl_tables(struct hfi1_devdata *dd) 13954 { 13955 int i; 13956 /* init per architecture spec, constrained by hardware capability */ 13957 13958 /* HFI maps sent packets */ 13959 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL( 13960 0, 13961 0, 0, 1, 1, 13962 2, 2, 3, 3, 13963 4, 4, 5, 5, 13964 6, 6, 7, 7)); 13965 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL( 13966 1, 13967 8, 0, 9, 0, 13968 10, 0, 11, 0, 13969 12, 0, 13, 0, 13970 14, 0, 15, 15)); 13971 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL( 13972 2, 13973 16, 0, 17, 0, 13974 18, 0, 19, 0, 13975 20, 0, 21, 0, 13976 22, 0, 23, 0)); 13977 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL( 13978 3, 13979 24, 0, 25, 0, 13980 26, 0, 27, 0, 13981 28, 0, 29, 0, 13982 30, 0, 31, 0)); 13983 13984 /* DC maps received packets */ 13985 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL( 13986 15_0, 13987 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 13988 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15)); 13989 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL( 13990 31_16, 13991 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 13992 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0)); 13993 13994 /* initialize the cached sc2vl values consistently with h/w */ 13995 for (i = 0; i < 32; i++) { 13996 if (i < 8 || i == 15) 13997 *((u8 *)(dd->sc2vl) + i) = (u8)i; 13998 else 13999 *((u8 *)(dd->sc2vl) + i) = 0; 14000 } 14001 } 14002 14003 /* 14004 * Read chip sizes and then reset parts to sane, disabled, values. We cannot 14005 * depend on the chip going through a power-on reset - a driver may be loaded 14006 * and unloaded many times. 14007 * 14008 * Do not write any CSR values to the chip in this routine - there may be 14009 * a reset following the (possible) FLR in this routine. 14010 * 14011 */ 14012 static int init_chip(struct hfi1_devdata *dd) 14013 { 14014 int i; 14015 int ret = 0; 14016 14017 /* 14018 * Put the HFI CSRs in a known state. 14019 * Combine this with a DC reset. 14020 * 14021 * Stop the device from doing anything while we do a 14022 * reset. We know there are no other active users of 14023 * the device since we are now in charge. Turn off 14024 * off all outbound and inbound traffic and make sure 14025 * the device does not generate any interrupts. 14026 */ 14027 14028 /* disable send contexts and SDMA engines */ 14029 write_csr(dd, SEND_CTRL, 0); 14030 for (i = 0; i < chip_send_contexts(dd); i++) 14031 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 14032 for (i = 0; i < chip_sdma_engines(dd); i++) 14033 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 14034 /* disable port (turn off RXE inbound traffic) and contexts */ 14035 write_csr(dd, RCV_CTRL, 0); 14036 for (i = 0; i < chip_rcv_contexts(dd); i++) 14037 write_csr(dd, RCV_CTXT_CTRL, 0); 14038 /* mask all interrupt sources */ 14039 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 14040 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull); 14041 14042 /* 14043 * DC Reset: do a full DC reset before the register clear. 14044 * A recommended length of time to hold is one CSR read, 14045 * so reread the CceDcCtrl. Then, hold the DC in reset 14046 * across the clear. 14047 */ 14048 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK); 14049 (void)read_csr(dd, CCE_DC_CTRL); 14050 14051 if (use_flr) { 14052 /* 14053 * A FLR will reset the SPC core and part of the PCIe. 14054 * The parts that need to be restored have already been 14055 * saved. 14056 */ 14057 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 14058 14059 /* do the FLR, the DC reset will remain */ 14060 pcie_flr(dd->pcidev); 14061 14062 /* restore command and BARs */ 14063 ret = restore_pci_variables(dd); 14064 if (ret) { 14065 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 14066 __func__); 14067 return ret; 14068 } 14069 14070 if (is_ax(dd)) { 14071 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 14072 pcie_flr(dd->pcidev); 14073 ret = restore_pci_variables(dd); 14074 if (ret) { 14075 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 14076 __func__); 14077 return ret; 14078 } 14079 } 14080 } else { 14081 dd_dev_info(dd, "Resetting CSRs with writes\n"); 14082 reset_cce_csrs(dd); 14083 reset_txe_csrs(dd); 14084 reset_rxe_csrs(dd); 14085 reset_misc_csrs(dd); 14086 } 14087 /* clear the DC reset */ 14088 write_csr(dd, CCE_DC_CTRL, 0); 14089 14090 /* Set the LED off */ 14091 setextled(dd, 0); 14092 14093 /* 14094 * Clear the QSFP reset. 14095 * An FLR enforces a 0 on all out pins. The driver does not touch 14096 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and 14097 * anything plugged constantly in reset, if it pays attention 14098 * to RESET_N. 14099 * Prime examples of this are optical cables. Set all pins high. 14100 * I2CCLK and I2CDAT will change per direction, and INT_N and 14101 * MODPRS_N are input only and their value is ignored. 14102 */ 14103 write_csr(dd, ASIC_QSFP1_OUT, 0x1f); 14104 write_csr(dd, ASIC_QSFP2_OUT, 0x1f); 14105 init_chip_resources(dd); 14106 return ret; 14107 } 14108 14109 static void init_early_variables(struct hfi1_devdata *dd) 14110 { 14111 int i; 14112 14113 /* assign link credit variables */ 14114 dd->vau = CM_VAU; 14115 dd->link_credits = CM_GLOBAL_CREDITS; 14116 if (is_ax(dd)) 14117 dd->link_credits--; 14118 dd->vcu = cu_to_vcu(hfi1_cu); 14119 /* enough room for 8 MAD packets plus header - 17K */ 14120 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau); 14121 if (dd->vl15_init > dd->link_credits) 14122 dd->vl15_init = dd->link_credits; 14123 14124 write_uninitialized_csrs_and_memories(dd); 14125 14126 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 14127 for (i = 0; i < dd->num_pports; i++) { 14128 struct hfi1_pportdata *ppd = &dd->pport[i]; 14129 14130 set_partition_keys(ppd); 14131 } 14132 init_sc2vl_tables(dd); 14133 } 14134 14135 static void init_kdeth_qp(struct hfi1_devdata *dd) 14136 { 14137 write_csr(dd, SEND_BTH_QP, 14138 (RVT_KDETH_QP_PREFIX & SEND_BTH_QP_KDETH_QP_MASK) << 14139 SEND_BTH_QP_KDETH_QP_SHIFT); 14140 14141 write_csr(dd, RCV_BTH_QP, 14142 (RVT_KDETH_QP_PREFIX & RCV_BTH_QP_KDETH_QP_MASK) << 14143 RCV_BTH_QP_KDETH_QP_SHIFT); 14144 } 14145 14146 /** 14147 * hfi1_get_qp_map - get qp map 14148 * @dd: device data 14149 * @idx: index to read 14150 */ 14151 u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx) 14152 { 14153 u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8); 14154 14155 reg >>= (idx % 8) * 8; 14156 return reg; 14157 } 14158 14159 /** 14160 * init_qpmap_table - init qp map 14161 * @dd: device data 14162 * @first_ctxt: first context 14163 * @last_ctxt: first context 14164 * 14165 * This return sets the qpn mapping table that 14166 * is indexed by qpn[8:1]. 14167 * 14168 * The routine will round robin the 256 settings 14169 * from first_ctxt to last_ctxt. 14170 * 14171 * The first/last looks ahead to having specialized 14172 * receive contexts for mgmt and bypass. Normal 14173 * verbs traffic will assumed to be on a range 14174 * of receive contexts. 14175 */ 14176 static void init_qpmap_table(struct hfi1_devdata *dd, 14177 u32 first_ctxt, 14178 u32 last_ctxt) 14179 { 14180 u64 reg = 0; 14181 u64 regno = RCV_QP_MAP_TABLE; 14182 int i; 14183 u64 ctxt = first_ctxt; 14184 14185 for (i = 0; i < 256; i++) { 14186 reg |= ctxt << (8 * (i % 8)); 14187 ctxt++; 14188 if (ctxt > last_ctxt) 14189 ctxt = first_ctxt; 14190 if (i % 8 == 7) { 14191 write_csr(dd, regno, reg); 14192 reg = 0; 14193 regno += 8; 14194 } 14195 } 14196 14197 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK 14198 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK); 14199 } 14200 14201 struct rsm_map_table { 14202 u64 map[NUM_MAP_REGS]; 14203 unsigned int used; 14204 }; 14205 14206 struct rsm_rule_data { 14207 u8 offset; 14208 u8 pkt_type; 14209 u32 field1_off; 14210 u32 field2_off; 14211 u32 index1_off; 14212 u32 index1_width; 14213 u32 index2_off; 14214 u32 index2_width; 14215 u32 mask1; 14216 u32 value1; 14217 u32 mask2; 14218 u32 value2; 14219 }; 14220 14221 /* 14222 * Return an initialized RMT map table for users to fill in. OK if it 14223 * returns NULL, indicating no table. 14224 */ 14225 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd) 14226 { 14227 struct rsm_map_table *rmt; 14228 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */ 14229 14230 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL); 14231 if (rmt) { 14232 memset(rmt->map, rxcontext, sizeof(rmt->map)); 14233 rmt->used = 0; 14234 } 14235 14236 return rmt; 14237 } 14238 14239 /* 14240 * Write the final RMT map table to the chip and free the table. OK if 14241 * table is NULL. 14242 */ 14243 static void complete_rsm_map_table(struct hfi1_devdata *dd, 14244 struct rsm_map_table *rmt) 14245 { 14246 int i; 14247 14248 if (rmt) { 14249 /* write table to chip */ 14250 for (i = 0; i < NUM_MAP_REGS; i++) 14251 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]); 14252 14253 /* enable RSM */ 14254 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14255 } 14256 } 14257 14258 /* Is a receive side mapping rule */ 14259 static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) 14260 { 14261 return read_csr(dd, RCV_RSM_CFG + (8 * rule_index)) != 0; 14262 } 14263 14264 /* 14265 * Add a receive side mapping rule. 14266 */ 14267 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index, 14268 struct rsm_rule_data *rrd) 14269 { 14270 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 14271 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT | 14272 1ull << rule_index | /* enable bit */ 14273 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT); 14274 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 14275 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT | 14276 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT | 14277 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT | 14278 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT | 14279 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT | 14280 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT); 14281 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 14282 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT | 14283 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT | 14284 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT | 14285 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT); 14286 } 14287 14288 /* 14289 * Clear a receive side mapping rule. 14290 */ 14291 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) 14292 { 14293 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0); 14294 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0); 14295 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0); 14296 } 14297 14298 /* return the number of RSM map table entries that will be used for QOS */ 14299 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 14300 unsigned int *np) 14301 { 14302 int i; 14303 unsigned int m, n; 14304 u8 max_by_vl = 0; 14305 14306 /* is QOS active at all? */ 14307 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS || 14308 num_vls == 1 || 14309 krcvqsset <= 1) 14310 goto no_qos; 14311 14312 /* determine bits for qpn */ 14313 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++) 14314 if (krcvqs[i] > max_by_vl) 14315 max_by_vl = krcvqs[i]; 14316 if (max_by_vl > 32) 14317 goto no_qos; 14318 m = ilog2(__roundup_pow_of_two(max_by_vl)); 14319 14320 /* determine bits for vl */ 14321 n = ilog2(__roundup_pow_of_two(num_vls)); 14322 14323 /* reject if too much is used */ 14324 if ((m + n) > 7) 14325 goto no_qos; 14326 14327 if (mp) 14328 *mp = m; 14329 if (np) 14330 *np = n; 14331 14332 return 1 << (m + n); 14333 14334 no_qos: 14335 if (mp) 14336 *mp = 0; 14337 if (np) 14338 *np = 0; 14339 return 0; 14340 } 14341 14342 /** 14343 * init_qos - init RX qos 14344 * @dd: device data 14345 * @rmt: RSM map table 14346 * 14347 * This routine initializes Rule 0 and the RSM map table to implement 14348 * quality of service (qos). 14349 * 14350 * If all of the limit tests succeed, qos is applied based on the array 14351 * interpretation of krcvqs where entry 0 is VL0. 14352 * 14353 * The number of vl bits (n) and the number of qpn bits (m) are computed to 14354 * feed both the RSM map table and the single rule. 14355 */ 14356 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt) 14357 { 14358 struct rsm_rule_data rrd; 14359 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m; 14360 unsigned int rmt_entries; 14361 u64 reg; 14362 14363 if (!rmt) 14364 goto bail; 14365 rmt_entries = qos_rmt_entries(dd, &m, &n); 14366 if (rmt_entries == 0) 14367 goto bail; 14368 qpns_per_vl = 1 << m; 14369 14370 /* enough room in the map table? */ 14371 rmt_entries = 1 << (m + n); 14372 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES) 14373 goto bail; 14374 14375 /* add qos entries to the RSM map table */ 14376 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) { 14377 unsigned tctxt; 14378 14379 for (qpn = 0, tctxt = ctxt; 14380 krcvqs[i] && qpn < qpns_per_vl; qpn++) { 14381 unsigned idx, regoff, regidx; 14382 14383 /* generate the index the hardware will produce */ 14384 idx = rmt->used + ((qpn << n) ^ i); 14385 regoff = (idx % 8) * 8; 14386 regidx = idx / 8; 14387 /* replace default with context number */ 14388 reg = rmt->map[regidx]; 14389 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK 14390 << regoff); 14391 reg |= (u64)(tctxt++) << regoff; 14392 rmt->map[regidx] = reg; 14393 if (tctxt == ctxt + krcvqs[i]) 14394 tctxt = ctxt; 14395 } 14396 ctxt += krcvqs[i]; 14397 } 14398 14399 rrd.offset = rmt->used; 14400 rrd.pkt_type = 2; 14401 rrd.field1_off = LRH_BTH_MATCH_OFFSET; 14402 rrd.field2_off = LRH_SC_MATCH_OFFSET; 14403 rrd.index1_off = LRH_SC_SELECT_OFFSET; 14404 rrd.index1_width = n; 14405 rrd.index2_off = QPN_SELECT_OFFSET; 14406 rrd.index2_width = m + n; 14407 rrd.mask1 = LRH_BTH_MASK; 14408 rrd.value1 = LRH_BTH_VALUE; 14409 rrd.mask2 = LRH_SC_MASK; 14410 rrd.value2 = LRH_SC_VALUE; 14411 14412 /* add rule 0 */ 14413 add_rsm_rule(dd, RSM_INS_VERBS, &rrd); 14414 14415 /* mark RSM map entries as used */ 14416 rmt->used += rmt_entries; 14417 /* map everything else to the mcast/err/vl15 context */ 14418 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT); 14419 dd->qos_shift = n + 1; 14420 return; 14421 bail: 14422 dd->qos_shift = 1; 14423 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1); 14424 } 14425 14426 static void init_fecn_handling(struct hfi1_devdata *dd, 14427 struct rsm_map_table *rmt) 14428 { 14429 struct rsm_rule_data rrd; 14430 u64 reg; 14431 int i, idx, regoff, regidx, start; 14432 u8 offset; 14433 u32 total_cnt; 14434 14435 if (HFI1_CAP_IS_KSET(TID_RDMA)) 14436 /* Exclude context 0 */ 14437 start = 1; 14438 else 14439 start = dd->first_dyn_alloc_ctxt; 14440 14441 total_cnt = dd->num_rcv_contexts - start; 14442 14443 /* there needs to be enough room in the map table */ 14444 if (rmt->used + total_cnt >= NUM_MAP_ENTRIES) { 14445 dd_dev_err(dd, "FECN handling disabled - too many contexts allocated\n"); 14446 return; 14447 } 14448 14449 /* 14450 * RSM will extract the destination context as an index into the 14451 * map table. The destination contexts are a sequential block 14452 * in the range start...num_rcv_contexts-1 (inclusive). 14453 * Map entries are accessed as offset + extracted value. Adjust 14454 * the added offset so this sequence can be placed anywhere in 14455 * the table - as long as the entries themselves do not wrap. 14456 * There are only enough bits in offset for the table size, so 14457 * start with that to allow for a "negative" offset. 14458 */ 14459 offset = (u8)(NUM_MAP_ENTRIES + rmt->used - start); 14460 14461 for (i = start, idx = rmt->used; i < dd->num_rcv_contexts; 14462 i++, idx++) { 14463 /* replace with identity mapping */ 14464 regoff = (idx % 8) * 8; 14465 regidx = idx / 8; 14466 reg = rmt->map[regidx]; 14467 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff); 14468 reg |= (u64)i << regoff; 14469 rmt->map[regidx] = reg; 14470 } 14471 14472 /* 14473 * For RSM intercept of Expected FECN packets: 14474 * o packet type 0 - expected 14475 * o match on F (bit 95), using select/match 1, and 14476 * o match on SH (bit 133), using select/match 2. 14477 * 14478 * Use index 1 to extract the 8-bit receive context from DestQP 14479 * (start at bit 64). Use that as the RSM map table index. 14480 */ 14481 rrd.offset = offset; 14482 rrd.pkt_type = 0; 14483 rrd.field1_off = 95; 14484 rrd.field2_off = 133; 14485 rrd.index1_off = 64; 14486 rrd.index1_width = 8; 14487 rrd.index2_off = 0; 14488 rrd.index2_width = 0; 14489 rrd.mask1 = 1; 14490 rrd.value1 = 1; 14491 rrd.mask2 = 1; 14492 rrd.value2 = 1; 14493 14494 /* add rule 1 */ 14495 add_rsm_rule(dd, RSM_INS_FECN, &rrd); 14496 14497 rmt->used += total_cnt; 14498 } 14499 14500 static inline bool hfi1_is_rmt_full(int start, int spare) 14501 { 14502 return (start + spare) > NUM_MAP_ENTRIES; 14503 } 14504 14505 static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd) 14506 { 14507 u8 i, j; 14508 u8 ctx_id = 0; 14509 u64 reg; 14510 u32 regoff; 14511 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd); 14512 int ctxt_count = hfi1_netdev_ctxt_count(dd); 14513 14514 /* We already have contexts mapped in RMT */ 14515 if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) { 14516 dd_dev_info(dd, "Contexts are already mapped in RMT\n"); 14517 return true; 14518 } 14519 14520 if (hfi1_is_rmt_full(rmt_start, NUM_NETDEV_MAP_ENTRIES)) { 14521 dd_dev_err(dd, "Not enough RMT entries used = %d\n", 14522 rmt_start); 14523 return false; 14524 } 14525 14526 dev_dbg(&(dd)->pcidev->dev, "RMT start = %d, end %d\n", 14527 rmt_start, 14528 rmt_start + NUM_NETDEV_MAP_ENTRIES); 14529 14530 /* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */ 14531 regoff = RCV_RSM_MAP_TABLE + (rmt_start / 8) * 8; 14532 reg = read_csr(dd, regoff); 14533 for (i = 0; i < NUM_NETDEV_MAP_ENTRIES; i++) { 14534 /* Update map register with netdev context */ 14535 j = (rmt_start + i) % 8; 14536 reg &= ~(0xffllu << (j * 8)); 14537 reg |= (u64)hfi1_netdev_get_ctxt(dd, ctx_id++)->ctxt << (j * 8); 14538 /* Wrap up netdev ctx index */ 14539 ctx_id %= ctxt_count; 14540 /* Write back map register */ 14541 if (j == 7 || ((i + 1) == NUM_NETDEV_MAP_ENTRIES)) { 14542 dev_dbg(&(dd)->pcidev->dev, 14543 "RMT[%d] =0x%llx\n", 14544 regoff - RCV_RSM_MAP_TABLE, reg); 14545 14546 write_csr(dd, regoff, reg); 14547 regoff += 8; 14548 if (i < (NUM_NETDEV_MAP_ENTRIES - 1)) 14549 reg = read_csr(dd, regoff); 14550 } 14551 } 14552 14553 return true; 14554 } 14555 14556 static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd, 14557 int rule, struct rsm_rule_data *rrd) 14558 { 14559 if (!hfi1_netdev_update_rmt(dd)) { 14560 dd_dev_err(dd, "Failed to update RMT for RSM%d rule\n", rule); 14561 return; 14562 } 14563 14564 add_rsm_rule(dd, rule, rrd); 14565 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14566 } 14567 14568 void hfi1_init_aip_rsm(struct hfi1_devdata *dd) 14569 { 14570 /* 14571 * go through with the initialisation only if this rule actually doesn't 14572 * exist yet 14573 */ 14574 if (atomic_fetch_inc(&dd->ipoib_rsm_usr_num) == 0) { 14575 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd); 14576 struct rsm_rule_data rrd = { 14577 .offset = rmt_start, 14578 .pkt_type = IB_PACKET_TYPE, 14579 .field1_off = LRH_BTH_MATCH_OFFSET, 14580 .mask1 = LRH_BTH_MASK, 14581 .value1 = LRH_BTH_VALUE, 14582 .field2_off = BTH_DESTQP_MATCH_OFFSET, 14583 .mask2 = BTH_DESTQP_MASK, 14584 .value2 = BTH_DESTQP_VALUE, 14585 .index1_off = DETH_AIP_SQPN_SELECT_OFFSET + 14586 ilog2(NUM_NETDEV_MAP_ENTRIES), 14587 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES), 14588 .index2_off = DETH_AIP_SQPN_SELECT_OFFSET, 14589 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES) 14590 }; 14591 14592 hfi1_enable_rsm_rule(dd, RSM_INS_AIP, &rrd); 14593 } 14594 } 14595 14596 /* Initialize RSM for VNIC */ 14597 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd) 14598 { 14599 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd); 14600 struct rsm_rule_data rrd = { 14601 /* Add rule for vnic */ 14602 .offset = rmt_start, 14603 .pkt_type = 4, 14604 /* Match 16B packets */ 14605 .field1_off = L2_TYPE_MATCH_OFFSET, 14606 .mask1 = L2_TYPE_MASK, 14607 .value1 = L2_16B_VALUE, 14608 /* Match ETH L4 packets */ 14609 .field2_off = L4_TYPE_MATCH_OFFSET, 14610 .mask2 = L4_16B_TYPE_MASK, 14611 .value2 = L4_16B_ETH_VALUE, 14612 /* Calc context from veswid and entropy */ 14613 .index1_off = L4_16B_HDR_VESWID_OFFSET, 14614 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES), 14615 .index2_off = L2_16B_ENTROPY_OFFSET, 14616 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES) 14617 }; 14618 14619 hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &rrd); 14620 } 14621 14622 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd) 14623 { 14624 clear_rsm_rule(dd, RSM_INS_VNIC); 14625 } 14626 14627 void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd) 14628 { 14629 /* only actually clear the rule if it's the last user asking to do so */ 14630 if (atomic_fetch_add_unless(&dd->ipoib_rsm_usr_num, -1, 0) == 1) 14631 clear_rsm_rule(dd, RSM_INS_AIP); 14632 } 14633 14634 static int init_rxe(struct hfi1_devdata *dd) 14635 { 14636 struct rsm_map_table *rmt; 14637 u64 val; 14638 14639 /* enable all receive errors */ 14640 write_csr(dd, RCV_ERR_MASK, ~0ull); 14641 14642 rmt = alloc_rsm_map_table(dd); 14643 if (!rmt) 14644 return -ENOMEM; 14645 14646 /* set up QOS, including the QPN map table */ 14647 init_qos(dd, rmt); 14648 init_fecn_handling(dd, rmt); 14649 complete_rsm_map_table(dd, rmt); 14650 /* record number of used rsm map entries for netdev */ 14651 hfi1_netdev_set_free_rmt_idx(dd, rmt->used); 14652 kfree(rmt); 14653 14654 /* 14655 * make sure RcvCtrl.RcvWcb <= PCIe Device Control 14656 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config 14657 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one 14658 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and 14659 * Max_PayLoad_Size set to its minimum of 128. 14660 * 14661 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0 14662 * (64 bytes). Max_Payload_Size is possibly modified upward in 14663 * tune_pcie_caps() which is called after this routine. 14664 */ 14665 14666 /* Have 16 bytes (4DW) of bypass header available in header queue */ 14667 val = read_csr(dd, RCV_BYPASS); 14668 val &= ~RCV_BYPASS_HDR_SIZE_SMASK; 14669 val |= ((4ull & RCV_BYPASS_HDR_SIZE_MASK) << 14670 RCV_BYPASS_HDR_SIZE_SHIFT); 14671 write_csr(dd, RCV_BYPASS, val); 14672 return 0; 14673 } 14674 14675 static void init_other(struct hfi1_devdata *dd) 14676 { 14677 /* enable all CCE errors */ 14678 write_csr(dd, CCE_ERR_MASK, ~0ull); 14679 /* enable *some* Misc errors */ 14680 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK); 14681 /* enable all DC errors, except LCB */ 14682 write_csr(dd, DCC_ERR_FLG_EN, ~0ull); 14683 write_csr(dd, DC_DC8051_ERR_EN, ~0ull); 14684 } 14685 14686 /* 14687 * Fill out the given AU table using the given CU. A CU is defined in terms 14688 * AUs. The table is a an encoding: given the index, how many AUs does that 14689 * represent? 14690 * 14691 * NOTE: Assumes that the register layout is the same for the 14692 * local and remote tables. 14693 */ 14694 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu, 14695 u32 csr0to3, u32 csr4to7) 14696 { 14697 write_csr(dd, csr0to3, 14698 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT | 14699 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT | 14700 2ull * cu << 14701 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT | 14702 4ull * cu << 14703 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT); 14704 write_csr(dd, csr4to7, 14705 8ull * cu << 14706 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT | 14707 16ull * cu << 14708 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT | 14709 32ull * cu << 14710 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT | 14711 64ull * cu << 14712 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT); 14713 } 14714 14715 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 14716 { 14717 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3, 14718 SEND_CM_LOCAL_AU_TABLE4_TO7); 14719 } 14720 14721 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 14722 { 14723 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3, 14724 SEND_CM_REMOTE_AU_TABLE4_TO7); 14725 } 14726 14727 static void init_txe(struct hfi1_devdata *dd) 14728 { 14729 int i; 14730 14731 /* enable all PIO, SDMA, general, and Egress errors */ 14732 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull); 14733 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull); 14734 write_csr(dd, SEND_ERR_MASK, ~0ull); 14735 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull); 14736 14737 /* enable all per-context and per-SDMA engine errors */ 14738 for (i = 0; i < chip_send_contexts(dd); i++) 14739 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull); 14740 for (i = 0; i < chip_sdma_engines(dd); i++) 14741 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull); 14742 14743 /* set the local CU to AU mapping */ 14744 assign_local_cm_au_table(dd, dd->vcu); 14745 14746 /* 14747 * Set reasonable default for Credit Return Timer 14748 * Don't set on Simulator - causes it to choke. 14749 */ 14750 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR) 14751 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE); 14752 } 14753 14754 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, 14755 u16 jkey) 14756 { 14757 u8 hw_ctxt; 14758 u64 reg; 14759 14760 if (!rcd || !rcd->sc) 14761 return -EINVAL; 14762 14763 hw_ctxt = rcd->sc->hw_context; 14764 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */ 14765 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) << 14766 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT); 14767 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */ 14768 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY)) 14769 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK; 14770 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg); 14771 /* 14772 * Enable send-side J_KEY integrity check, unless this is A0 h/w 14773 */ 14774 if (!is_ax(dd)) { 14775 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14776 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14777 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14778 } 14779 14780 /* Enable J_KEY check on receive context. */ 14781 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK | 14782 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) << 14783 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT); 14784 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg); 14785 14786 return 0; 14787 } 14788 14789 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) 14790 { 14791 u8 hw_ctxt; 14792 u64 reg; 14793 14794 if (!rcd || !rcd->sc) 14795 return -EINVAL; 14796 14797 hw_ctxt = rcd->sc->hw_context; 14798 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0); 14799 /* 14800 * Disable send-side J_KEY integrity check, unless this is A0 h/w. 14801 * This check would not have been enabled for A0 h/w, see 14802 * set_ctxt_jkey(). 14803 */ 14804 if (!is_ax(dd)) { 14805 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14806 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14807 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14808 } 14809 /* Turn off the J_KEY on the receive side */ 14810 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0); 14811 14812 return 0; 14813 } 14814 14815 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, 14816 u16 pkey) 14817 { 14818 u8 hw_ctxt; 14819 u64 reg; 14820 14821 if (!rcd || !rcd->sc) 14822 return -EINVAL; 14823 14824 hw_ctxt = rcd->sc->hw_context; 14825 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) << 14826 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT; 14827 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg); 14828 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14829 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14830 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK; 14831 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14832 14833 return 0; 14834 } 14835 14836 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt) 14837 { 14838 u8 hw_ctxt; 14839 u64 reg; 14840 14841 if (!ctxt || !ctxt->sc) 14842 return -EINVAL; 14843 14844 hw_ctxt = ctxt->sc->hw_context; 14845 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14846 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14847 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14848 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0); 14849 14850 return 0; 14851 } 14852 14853 /* 14854 * Start doing the clean up the chip. Our clean up happens in multiple 14855 * stages and this is just the first. 14856 */ 14857 void hfi1_start_cleanup(struct hfi1_devdata *dd) 14858 { 14859 aspm_exit(dd); 14860 free_cntrs(dd); 14861 free_rcverr(dd); 14862 finish_chip_resources(dd); 14863 } 14864 14865 #define HFI_BASE_GUID(dev) \ 14866 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT)) 14867 14868 /* 14869 * Information can be shared between the two HFIs on the same ASIC 14870 * in the same OS. This function finds the peer device and sets 14871 * up a shared structure. 14872 */ 14873 static int init_asic_data(struct hfi1_devdata *dd) 14874 { 14875 unsigned long index; 14876 struct hfi1_devdata *peer; 14877 struct hfi1_asic_data *asic_data; 14878 int ret = 0; 14879 14880 /* pre-allocate the asic structure in case we are the first device */ 14881 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL); 14882 if (!asic_data) 14883 return -ENOMEM; 14884 14885 xa_lock_irq(&hfi1_dev_table); 14886 /* Find our peer device */ 14887 xa_for_each(&hfi1_dev_table, index, peer) { 14888 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(peer)) && 14889 dd->unit != peer->unit) 14890 break; 14891 } 14892 14893 if (peer) { 14894 /* use already allocated structure */ 14895 dd->asic_data = peer->asic_data; 14896 kfree(asic_data); 14897 } else { 14898 dd->asic_data = asic_data; 14899 mutex_init(&dd->asic_data->asic_resource_mutex); 14900 } 14901 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */ 14902 xa_unlock_irq(&hfi1_dev_table); 14903 14904 /* first one through - set up i2c devices */ 14905 if (!peer) 14906 ret = set_up_i2c(dd, dd->asic_data); 14907 14908 return ret; 14909 } 14910 14911 /* 14912 * Set dd->boardname. Use a generic name if a name is not returned from 14913 * EFI variable space. 14914 * 14915 * Return 0 on success, -ENOMEM if space could not be allocated. 14916 */ 14917 static int obtain_boardname(struct hfi1_devdata *dd) 14918 { 14919 /* generic board description */ 14920 const char generic[] = 14921 "Intel Omni-Path Host Fabric Interface Adapter 100 Series"; 14922 unsigned long size; 14923 int ret; 14924 14925 ret = read_hfi1_efi_var(dd, "description", &size, 14926 (void **)&dd->boardname); 14927 if (ret) { 14928 dd_dev_info(dd, "Board description not found\n"); 14929 /* use generic description */ 14930 dd->boardname = kstrdup(generic, GFP_KERNEL); 14931 if (!dd->boardname) 14932 return -ENOMEM; 14933 } 14934 return 0; 14935 } 14936 14937 /* 14938 * Check the interrupt registers to make sure that they are mapped correctly. 14939 * It is intended to help user identify any mismapping by VMM when the driver 14940 * is running in a VM. This function should only be called before interrupt 14941 * is set up properly. 14942 * 14943 * Return 0 on success, -EINVAL on failure. 14944 */ 14945 static int check_int_registers(struct hfi1_devdata *dd) 14946 { 14947 u64 reg; 14948 u64 all_bits = ~(u64)0; 14949 u64 mask; 14950 14951 /* Clear CceIntMask[0] to avoid raising any interrupts */ 14952 mask = read_csr(dd, CCE_INT_MASK); 14953 write_csr(dd, CCE_INT_MASK, 0ull); 14954 reg = read_csr(dd, CCE_INT_MASK); 14955 if (reg) 14956 goto err_exit; 14957 14958 /* Clear all interrupt status bits */ 14959 write_csr(dd, CCE_INT_CLEAR, all_bits); 14960 reg = read_csr(dd, CCE_INT_STATUS); 14961 if (reg) 14962 goto err_exit; 14963 14964 /* Set all interrupt status bits */ 14965 write_csr(dd, CCE_INT_FORCE, all_bits); 14966 reg = read_csr(dd, CCE_INT_STATUS); 14967 if (reg != all_bits) 14968 goto err_exit; 14969 14970 /* Restore the interrupt mask */ 14971 write_csr(dd, CCE_INT_CLEAR, all_bits); 14972 write_csr(dd, CCE_INT_MASK, mask); 14973 14974 return 0; 14975 err_exit: 14976 write_csr(dd, CCE_INT_MASK, mask); 14977 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n"); 14978 return -EINVAL; 14979 } 14980 14981 /** 14982 * hfi1_init_dd() - Initialize most of the dd structure. 14983 * @dd: the dd device 14984 * 14985 * This is global, and is called directly at init to set up the 14986 * chip-specific function pointers for later use. 14987 */ 14988 int hfi1_init_dd(struct hfi1_devdata *dd) 14989 { 14990 struct pci_dev *pdev = dd->pcidev; 14991 struct hfi1_pportdata *ppd; 14992 u64 reg; 14993 int i, ret; 14994 static const char * const inames[] = { /* implementation names */ 14995 "RTL silicon", 14996 "RTL VCS simulation", 14997 "RTL FPGA emulation", 14998 "Functional simulator" 14999 }; 15000 struct pci_dev *parent = pdev->bus->self; 15001 u32 sdma_engines = chip_sdma_engines(dd); 15002 15003 ppd = dd->pport; 15004 for (i = 0; i < dd->num_pports; i++, ppd++) { 15005 int vl; 15006 /* init common fields */ 15007 hfi1_init_pportdata(pdev, ppd, dd, 0, 1); 15008 /* DC supports 4 link widths */ 15009 ppd->link_width_supported = 15010 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X | 15011 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X; 15012 ppd->link_width_downgrade_supported = 15013 ppd->link_width_supported; 15014 /* start out enabling only 4X */ 15015 ppd->link_width_enabled = OPA_LINK_WIDTH_4X; 15016 ppd->link_width_downgrade_enabled = 15017 ppd->link_width_downgrade_supported; 15018 /* link width active is 0 when link is down */ 15019 /* link width downgrade active is 0 when link is down */ 15020 15021 if (num_vls < HFI1_MIN_VLS_SUPPORTED || 15022 num_vls > HFI1_MAX_VLS_SUPPORTED) { 15023 dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n", 15024 num_vls, HFI1_MAX_VLS_SUPPORTED); 15025 num_vls = HFI1_MAX_VLS_SUPPORTED; 15026 } 15027 ppd->vls_supported = num_vls; 15028 ppd->vls_operational = ppd->vls_supported; 15029 /* Set the default MTU. */ 15030 for (vl = 0; vl < num_vls; vl++) 15031 dd->vld[vl].mtu = hfi1_max_mtu; 15032 dd->vld[15].mtu = MAX_MAD_PACKET; 15033 /* 15034 * Set the initial values to reasonable default, will be set 15035 * for real when link is up. 15036 */ 15037 ppd->overrun_threshold = 0x4; 15038 ppd->phy_error_threshold = 0xf; 15039 ppd->port_crc_mode_enabled = link_crc_mask; 15040 /* initialize supported LTP CRC mode */ 15041 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 15042 /* initialize enabled LTP CRC mode */ 15043 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4; 15044 /* start in offline */ 15045 ppd->host_link_state = HLS_DN_OFFLINE; 15046 init_vl_arb_caches(ppd); 15047 } 15048 15049 /* 15050 * Do remaining PCIe setup and save PCIe values in dd. 15051 * Any error printing is already done by the init code. 15052 * On return, we have the chip mapped. 15053 */ 15054 ret = hfi1_pcie_ddinit(dd, pdev); 15055 if (ret < 0) 15056 goto bail_free; 15057 15058 /* Save PCI space registers to rewrite after device reset */ 15059 ret = save_pci_variables(dd); 15060 if (ret < 0) 15061 goto bail_cleanup; 15062 15063 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT) 15064 & CCE_REVISION_CHIP_REV_MAJOR_MASK; 15065 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT) 15066 & CCE_REVISION_CHIP_REV_MINOR_MASK; 15067 15068 /* 15069 * Check interrupt registers mapping if the driver has no access to 15070 * the upstream component. In this case, it is likely that the driver 15071 * is running in a VM. 15072 */ 15073 if (!parent) { 15074 ret = check_int_registers(dd); 15075 if (ret) 15076 goto bail_cleanup; 15077 } 15078 15079 /* 15080 * obtain the hardware ID - NOT related to unit, which is a 15081 * software enumeration 15082 */ 15083 reg = read_csr(dd, CCE_REVISION2); 15084 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT) 15085 & CCE_REVISION2_HFI_ID_MASK; 15086 /* the variable size will remove unwanted bits */ 15087 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT; 15088 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT; 15089 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n", 15090 dd->icode < ARRAY_SIZE(inames) ? 15091 inames[dd->icode] : "unknown", (int)dd->irev); 15092 15093 /* speeds the hardware can support */ 15094 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G; 15095 /* speeds allowed to run at */ 15096 dd->pport->link_speed_enabled = dd->pport->link_speed_supported; 15097 /* give a reasonable active value, will be set on link up */ 15098 dd->pport->link_speed_active = OPA_LINK_SPEED_25G; 15099 15100 /* fix up link widths for emulation _p */ 15101 ppd = dd->pport; 15102 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) { 15103 ppd->link_width_supported = 15104 ppd->link_width_enabled = 15105 ppd->link_width_downgrade_supported = 15106 ppd->link_width_downgrade_enabled = 15107 OPA_LINK_WIDTH_1X; 15108 } 15109 /* insure num_vls isn't larger than number of sdma engines */ 15110 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > sdma_engines) { 15111 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n", 15112 num_vls, sdma_engines); 15113 num_vls = sdma_engines; 15114 ppd->vls_supported = sdma_engines; 15115 ppd->vls_operational = ppd->vls_supported; 15116 } 15117 15118 /* 15119 * Convert the ns parameter to the 64 * cclocks used in the CSR. 15120 * Limit the max if larger than the field holds. If timeout is 15121 * non-zero, then the calculated field will be at least 1. 15122 * 15123 * Must be after icode is set up - the cclock rate depends 15124 * on knowing the hardware being used. 15125 */ 15126 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64; 15127 if (dd->rcv_intr_timeout_csr > 15128 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK) 15129 dd->rcv_intr_timeout_csr = 15130 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK; 15131 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout) 15132 dd->rcv_intr_timeout_csr = 1; 15133 15134 /* needs to be done before we look for the peer device */ 15135 read_guid(dd); 15136 15137 /* set up shared ASIC data with peer device */ 15138 ret = init_asic_data(dd); 15139 if (ret) 15140 goto bail_cleanup; 15141 15142 /* obtain chip sizes, reset chip CSRs */ 15143 ret = init_chip(dd); 15144 if (ret) 15145 goto bail_cleanup; 15146 15147 /* read in the PCIe link speed information */ 15148 ret = pcie_speeds(dd); 15149 if (ret) 15150 goto bail_cleanup; 15151 15152 /* call before get_platform_config(), after init_chip_resources() */ 15153 ret = eprom_init(dd); 15154 if (ret) 15155 goto bail_free_rcverr; 15156 15157 /* Needs to be called before hfi1_firmware_init */ 15158 get_platform_config(dd); 15159 15160 /* read in firmware */ 15161 ret = hfi1_firmware_init(dd); 15162 if (ret) 15163 goto bail_cleanup; 15164 15165 /* 15166 * In general, the PCIe Gen3 transition must occur after the 15167 * chip has been idled (so it won't initiate any PCIe transactions 15168 * e.g. an interrupt) and before the driver changes any registers 15169 * (the transition will reset the registers). 15170 * 15171 * In particular, place this call after: 15172 * - init_chip() - the chip will not initiate any PCIe transactions 15173 * - pcie_speeds() - reads the current link speed 15174 * - hfi1_firmware_init() - the needed firmware is ready to be 15175 * downloaded 15176 */ 15177 ret = do_pcie_gen3_transition(dd); 15178 if (ret) 15179 goto bail_cleanup; 15180 15181 /* 15182 * This should probably occur in hfi1_pcie_init(), but historically 15183 * occurs after the do_pcie_gen3_transition() code. 15184 */ 15185 tune_pcie_caps(dd); 15186 15187 /* start setting dd values and adjusting CSRs */ 15188 init_early_variables(dd); 15189 15190 parse_platform_config(dd); 15191 15192 ret = obtain_boardname(dd); 15193 if (ret) 15194 goto bail_cleanup; 15195 15196 snprintf(dd->boardversion, BOARD_VERS_MAX, 15197 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n", 15198 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN, 15199 (u32)dd->majrev, 15200 (u32)dd->minrev, 15201 (dd->revision >> CCE_REVISION_SW_SHIFT) 15202 & CCE_REVISION_SW_MASK); 15203 15204 /* alloc VNIC/AIP rx data */ 15205 ret = hfi1_alloc_rx(dd); 15206 if (ret) 15207 goto bail_cleanup; 15208 15209 ret = set_up_context_variables(dd); 15210 if (ret) 15211 goto bail_cleanup; 15212 15213 /* set initial RXE CSRs */ 15214 ret = init_rxe(dd); 15215 if (ret) 15216 goto bail_cleanup; 15217 15218 /* set initial TXE CSRs */ 15219 init_txe(dd); 15220 /* set initial non-RXE, non-TXE CSRs */ 15221 init_other(dd); 15222 /* set up KDETH QP prefix in both RX and TX CSRs */ 15223 init_kdeth_qp(dd); 15224 15225 ret = hfi1_dev_affinity_init(dd); 15226 if (ret) 15227 goto bail_cleanup; 15228 15229 /* send contexts must be set up before receive contexts */ 15230 ret = init_send_contexts(dd); 15231 if (ret) 15232 goto bail_cleanup; 15233 15234 ret = hfi1_create_kctxts(dd); 15235 if (ret) 15236 goto bail_cleanup; 15237 15238 /* 15239 * Initialize aspm, to be done after gen3 transition and setting up 15240 * contexts and before enabling interrupts 15241 */ 15242 aspm_init(dd); 15243 15244 ret = init_pervl_scs(dd); 15245 if (ret) 15246 goto bail_cleanup; 15247 15248 /* sdma init */ 15249 for (i = 0; i < dd->num_pports; ++i) { 15250 ret = sdma_init(dd, i); 15251 if (ret) 15252 goto bail_cleanup; 15253 } 15254 15255 /* use contexts created by hfi1_create_kctxts */ 15256 ret = set_up_interrupts(dd); 15257 if (ret) 15258 goto bail_cleanup; 15259 15260 ret = hfi1_comp_vectors_set_up(dd); 15261 if (ret) 15262 goto bail_clear_intr; 15263 15264 /* set up LCB access - must be after set_up_interrupts() */ 15265 init_lcb_access(dd); 15266 15267 /* 15268 * Serial number is created from the base guid: 15269 * [27:24] = base guid [38:35] 15270 * [23: 0] = base guid [23: 0] 15271 */ 15272 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n", 15273 (dd->base_guid & 0xFFFFFF) | 15274 ((dd->base_guid >> 11) & 0xF000000)); 15275 15276 dd->oui1 = dd->base_guid >> 56 & 0xFF; 15277 dd->oui2 = dd->base_guid >> 48 & 0xFF; 15278 dd->oui3 = dd->base_guid >> 40 & 0xFF; 15279 15280 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */ 15281 if (ret) 15282 goto bail_clear_intr; 15283 15284 thermal_init(dd); 15285 15286 ret = init_cntrs(dd); 15287 if (ret) 15288 goto bail_clear_intr; 15289 15290 ret = init_rcverr(dd); 15291 if (ret) 15292 goto bail_free_cntrs; 15293 15294 init_completion(&dd->user_comp); 15295 15296 /* The user refcount starts with one to inidicate an active device */ 15297 refcount_set(&dd->user_refcount, 1); 15298 15299 goto bail; 15300 15301 bail_free_rcverr: 15302 free_rcverr(dd); 15303 bail_free_cntrs: 15304 free_cntrs(dd); 15305 bail_clear_intr: 15306 hfi1_comp_vectors_clean_up(dd); 15307 msix_clean_up_interrupts(dd); 15308 bail_cleanup: 15309 hfi1_free_rx(dd); 15310 hfi1_pcie_ddcleanup(dd); 15311 bail_free: 15312 hfi1_free_devdata(dd); 15313 bail: 15314 return ret; 15315 } 15316 15317 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate, 15318 u32 dw_len) 15319 { 15320 u32 delta_cycles; 15321 u32 current_egress_rate = ppd->current_egress_rate; 15322 /* rates here are in units of 10^6 bits/sec */ 15323 15324 if (desired_egress_rate == -1) 15325 return 0; /* shouldn't happen */ 15326 15327 if (desired_egress_rate >= current_egress_rate) 15328 return 0; /* we can't help go faster, only slower */ 15329 15330 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) - 15331 egress_cycles(dw_len * 4, current_egress_rate); 15332 15333 return (u16)delta_cycles; 15334 } 15335 15336 /** 15337 * create_pbc - build a pbc for transmission 15338 * @ppd: info of physical Hfi port 15339 * @flags: special case flags or-ed in built pbc 15340 * @srate_mbs: static rate 15341 * @vl: vl 15342 * @dw_len: dword length (header words + data words + pbc words) 15343 * 15344 * Create a PBC with the given flags, rate, VL, and length. 15345 * 15346 * NOTE: The PBC created will not insert any HCRC - all callers but one are 15347 * for verbs, which does not use this PSM feature. The lone other caller 15348 * is for the diagnostic interface which calls this if the user does not 15349 * supply their own PBC. 15350 */ 15351 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, 15352 u32 dw_len) 15353 { 15354 u64 pbc, delay = 0; 15355 15356 if (unlikely(srate_mbs)) 15357 delay = delay_cycles(ppd, srate_mbs, dw_len); 15358 15359 pbc = flags 15360 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT) 15361 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT) 15362 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT 15363 | (dw_len & PBC_LENGTH_DWS_MASK) 15364 << PBC_LENGTH_DWS_SHIFT; 15365 15366 return pbc; 15367 } 15368 15369 #define SBUS_THERMAL 0x4f 15370 #define SBUS_THERM_MONITOR_MODE 0x1 15371 15372 #define THERM_FAILURE(dev, ret, reason) \ 15373 dd_dev_err((dd), \ 15374 "Thermal sensor initialization failed: %s (%d)\n", \ 15375 (reason), (ret)) 15376 15377 /* 15378 * Initialize the thermal sensor. 15379 * 15380 * After initialization, enable polling of thermal sensor through 15381 * SBus interface. In order for this to work, the SBus Master 15382 * firmware has to be loaded due to the fact that the HW polling 15383 * logic uses SBus interrupts, which are not supported with 15384 * default firmware. Otherwise, no data will be returned through 15385 * the ASIC_STS_THERM CSR. 15386 */ 15387 static int thermal_init(struct hfi1_devdata *dd) 15388 { 15389 int ret = 0; 15390 15391 if (dd->icode != ICODE_RTL_SILICON || 15392 check_chip_resource(dd, CR_THERM_INIT, NULL)) 15393 return ret; 15394 15395 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT); 15396 if (ret) { 15397 THERM_FAILURE(dd, ret, "Acquire SBus"); 15398 return ret; 15399 } 15400 15401 dd_dev_info(dd, "Initializing thermal sensor\n"); 15402 /* Disable polling of thermal readings */ 15403 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0); 15404 msleep(100); 15405 /* Thermal Sensor Initialization */ 15406 /* Step 1: Reset the Thermal SBus Receiver */ 15407 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15408 RESET_SBUS_RECEIVER, 0); 15409 if (ret) { 15410 THERM_FAILURE(dd, ret, "Bus Reset"); 15411 goto done; 15412 } 15413 /* Step 2: Set Reset bit in Thermal block */ 15414 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15415 WRITE_SBUS_RECEIVER, 0x1); 15416 if (ret) { 15417 THERM_FAILURE(dd, ret, "Therm Block Reset"); 15418 goto done; 15419 } 15420 /* Step 3: Write clock divider value (100MHz -> 2MHz) */ 15421 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1, 15422 WRITE_SBUS_RECEIVER, 0x32); 15423 if (ret) { 15424 THERM_FAILURE(dd, ret, "Write Clock Div"); 15425 goto done; 15426 } 15427 /* Step 4: Select temperature mode */ 15428 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3, 15429 WRITE_SBUS_RECEIVER, 15430 SBUS_THERM_MONITOR_MODE); 15431 if (ret) { 15432 THERM_FAILURE(dd, ret, "Write Mode Sel"); 15433 goto done; 15434 } 15435 /* Step 5: De-assert block reset and start conversion */ 15436 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15437 WRITE_SBUS_RECEIVER, 0x2); 15438 if (ret) { 15439 THERM_FAILURE(dd, ret, "Write Reset Deassert"); 15440 goto done; 15441 } 15442 /* Step 5.1: Wait for first conversion (21.5ms per spec) */ 15443 msleep(22); 15444 15445 /* Enable polling of thermal readings */ 15446 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1); 15447 15448 /* Set initialized flag */ 15449 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0); 15450 if (ret) 15451 THERM_FAILURE(dd, ret, "Unable to set thermal init flag"); 15452 15453 done: 15454 release_chip_resource(dd, CR_SBUS); 15455 return ret; 15456 } 15457 15458 static void handle_temp_err(struct hfi1_devdata *dd) 15459 { 15460 struct hfi1_pportdata *ppd = &dd->pport[0]; 15461 /* 15462 * Thermal Critical Interrupt 15463 * Put the device into forced freeze mode, take link down to 15464 * offline, and put DC into reset. 15465 */ 15466 dd_dev_emerg(dd, 15467 "Critical temperature reached! Forcing device into freeze mode!\n"); 15468 dd->flags |= HFI1_FORCED_FREEZE; 15469 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT); 15470 /* 15471 * Shut DC down as much and as quickly as possible. 15472 * 15473 * Step 1: Take the link down to OFFLINE. This will cause the 15474 * 8051 to put the Serdes in reset. However, we don't want to 15475 * go through the entire link state machine since we want to 15476 * shutdown ASAP. Furthermore, this is not a graceful shutdown 15477 * but rather an attempt to save the chip. 15478 * Code below is almost the same as quiet_serdes() but avoids 15479 * all the extra work and the sleeps. 15480 */ 15481 ppd->driver_link_ready = 0; 15482 ppd->link_enabled = 0; 15483 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) | 15484 PLS_OFFLINE); 15485 /* 15486 * Step 2: Shutdown LCB and 8051 15487 * After shutdown, do not restore DC_CFG_RESET value. 15488 */ 15489 dc_shutdown(dd); 15490 } 15491