1 /* 2 * Copyright(c) 2015 - 2017 Intel Corporation. 3 * 4 * This file is provided under a dual BSD/GPLv2 license. When using or 5 * redistributing this file, you may do so under either license. 6 * 7 * GPL LICENSE SUMMARY 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of version 2 of the GNU General Public License as 11 * published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 * General Public License for more details. 17 * 18 * BSD LICENSE 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * - Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * - Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in 28 * the documentation and/or other materials provided with the 29 * distribution. 30 * - Neither the name of Intel Corporation nor the names of its 31 * contributors may be used to endorse or promote products derived 32 * from this software without specific prior written permission. 33 * 34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45 * 46 */ 47 48 /* 49 * This file contains all of the code that is specific to the HFI chip 50 */ 51 52 #include <linux/pci.h> 53 #include <linux/delay.h> 54 #include <linux/interrupt.h> 55 #include <linux/module.h> 56 57 #include "hfi.h" 58 #include "trace.h" 59 #include "mad.h" 60 #include "pio.h" 61 #include "sdma.h" 62 #include "eprom.h" 63 #include "efivar.h" 64 #include "platform.h" 65 #include "aspm.h" 66 #include "affinity.h" 67 #include "debugfs.h" 68 69 #define NUM_IB_PORTS 1 70 71 uint kdeth_qp; 72 module_param_named(kdeth_qp, kdeth_qp, uint, S_IRUGO); 73 MODULE_PARM_DESC(kdeth_qp, "Set the KDETH queue pair prefix"); 74 75 uint num_vls = HFI1_MAX_VLS_SUPPORTED; 76 module_param(num_vls, uint, S_IRUGO); 77 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)"); 78 79 /* 80 * Default time to aggregate two 10K packets from the idle state 81 * (timer not running). The timer starts at the end of the first packet, 82 * so only the time for one 10K packet and header plus a bit extra is needed. 83 * 10 * 1024 + 64 header byte = 10304 byte 84 * 10304 byte / 12.5 GB/s = 824.32ns 85 */ 86 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */ 87 module_param(rcv_intr_timeout, uint, S_IRUGO); 88 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns"); 89 90 uint rcv_intr_count = 16; /* same as qib */ 91 module_param(rcv_intr_count, uint, S_IRUGO); 92 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count"); 93 94 ushort link_crc_mask = SUPPORTED_CRCS; 95 module_param(link_crc_mask, ushort, S_IRUGO); 96 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link"); 97 98 uint loopback; 99 module_param_named(loopback, loopback, uint, S_IRUGO); 100 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable"); 101 102 /* Other driver tunables */ 103 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/ 104 static ushort crc_14b_sideband = 1; 105 static uint use_flr = 1; 106 uint quick_linkup; /* skip LNI */ 107 108 struct flag_table { 109 u64 flag; /* the flag */ 110 char *str; /* description string */ 111 u16 extra; /* extra information */ 112 u16 unused0; 113 u32 unused1; 114 }; 115 116 /* str must be a string constant */ 117 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra} 118 #define FLAG_ENTRY0(str, flag) {flag, str, 0} 119 120 /* Send Error Consequences */ 121 #define SEC_WRITE_DROPPED 0x1 122 #define SEC_PACKET_DROPPED 0x2 123 #define SEC_SC_HALTED 0x4 /* per-context only */ 124 #define SEC_SPC_FREEZE 0x8 /* per-HFI only */ 125 126 #define DEFAULT_KRCVQS 2 127 #define MIN_KERNEL_KCTXTS 2 128 #define FIRST_KERNEL_KCTXT 1 129 130 /* 131 * RSM instance allocation 132 * 0 - Verbs 133 * 1 - User Fecn Handling 134 * 2 - Vnic 135 */ 136 #define RSM_INS_VERBS 0 137 #define RSM_INS_FECN 1 138 #define RSM_INS_VNIC 2 139 140 /* Bit offset into the GUID which carries HFI id information */ 141 #define GUID_HFI_INDEX_SHIFT 39 142 143 /* extract the emulation revision */ 144 #define emulator_rev(dd) ((dd)->irev >> 8) 145 /* parallel and serial emulation versions are 3 and 4 respectively */ 146 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3) 147 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4) 148 149 /* RSM fields for Verbs */ 150 /* packet type */ 151 #define IB_PACKET_TYPE 2ull 152 #define QW_SHIFT 6ull 153 /* QPN[7..1] */ 154 #define QPN_WIDTH 7ull 155 156 /* LRH.BTH: QW 0, OFFSET 48 - for match */ 157 #define LRH_BTH_QW 0ull 158 #define LRH_BTH_BIT_OFFSET 48ull 159 #define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off)) 160 #define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET) 161 #define LRH_BTH_SELECT 162 #define LRH_BTH_MASK 3ull 163 #define LRH_BTH_VALUE 2ull 164 165 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */ 166 #define LRH_SC_QW 0ull 167 #define LRH_SC_BIT_OFFSET 56ull 168 #define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off)) 169 #define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET) 170 #define LRH_SC_MASK 128ull 171 #define LRH_SC_VALUE 0ull 172 173 /* SC[n..0] QW 0, OFFSET 60 - for select */ 174 #define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull)) 175 176 /* QPN[m+n:1] QW 1, OFFSET 1 */ 177 #define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull)) 178 179 /* RSM fields for Vnic */ 180 /* L2_TYPE: QW 0, OFFSET 61 - for match */ 181 #define L2_TYPE_QW 0ull 182 #define L2_TYPE_BIT_OFFSET 61ull 183 #define L2_TYPE_OFFSET(off) ((L2_TYPE_QW << QW_SHIFT) | (off)) 184 #define L2_TYPE_MATCH_OFFSET L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET) 185 #define L2_TYPE_MASK 3ull 186 #define L2_16B_VALUE 2ull 187 188 /* L4_TYPE QW 1, OFFSET 0 - for match */ 189 #define L4_TYPE_QW 1ull 190 #define L4_TYPE_BIT_OFFSET 0ull 191 #define L4_TYPE_OFFSET(off) ((L4_TYPE_QW << QW_SHIFT) | (off)) 192 #define L4_TYPE_MATCH_OFFSET L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET) 193 #define L4_16B_TYPE_MASK 0xFFull 194 #define L4_16B_ETH_VALUE 0x78ull 195 196 /* 16B VESWID - for select */ 197 #define L4_16B_HDR_VESWID_OFFSET ((2 << QW_SHIFT) | (16ull)) 198 /* 16B ENTROPY - for select */ 199 #define L2_16B_ENTROPY_OFFSET ((1 << QW_SHIFT) | (32ull)) 200 201 /* defines to build power on SC2VL table */ 202 #define SC2VL_VAL( \ 203 num, \ 204 sc0, sc0val, \ 205 sc1, sc1val, \ 206 sc2, sc2val, \ 207 sc3, sc3val, \ 208 sc4, sc4val, \ 209 sc5, sc5val, \ 210 sc6, sc6val, \ 211 sc7, sc7val) \ 212 ( \ 213 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \ 214 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \ 215 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \ 216 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \ 217 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \ 218 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \ 219 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \ 220 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \ 221 ) 222 223 #define DC_SC_VL_VAL( \ 224 range, \ 225 e0, e0val, \ 226 e1, e1val, \ 227 e2, e2val, \ 228 e3, e3val, \ 229 e4, e4val, \ 230 e5, e5val, \ 231 e6, e6val, \ 232 e7, e7val, \ 233 e8, e8val, \ 234 e9, e9val, \ 235 e10, e10val, \ 236 e11, e11val, \ 237 e12, e12val, \ 238 e13, e13val, \ 239 e14, e14val, \ 240 e15, e15val) \ 241 ( \ 242 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \ 243 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \ 244 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \ 245 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \ 246 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \ 247 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \ 248 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \ 249 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \ 250 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \ 251 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \ 252 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \ 253 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \ 254 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \ 255 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \ 256 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \ 257 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \ 258 ) 259 260 /* all CceStatus sub-block freeze bits */ 261 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \ 262 | CCE_STATUS_RXE_FROZE_SMASK \ 263 | CCE_STATUS_TXE_FROZE_SMASK \ 264 | CCE_STATUS_TXE_PIO_FROZE_SMASK) 265 /* all CceStatus sub-block TXE pause bits */ 266 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \ 267 | CCE_STATUS_TXE_PAUSED_SMASK \ 268 | CCE_STATUS_SDMA_PAUSED_SMASK) 269 /* all CceStatus sub-block RXE pause bits */ 270 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK 271 272 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL 273 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF 274 275 /* 276 * CCE Error flags. 277 */ 278 static struct flag_table cce_err_status_flags[] = { 279 /* 0*/ FLAG_ENTRY0("CceCsrParityErr", 280 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK), 281 /* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr", 282 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK), 283 /* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr", 284 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK), 285 /* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr", 286 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK), 287 /* 4*/ FLAG_ENTRY0("CceTrgtAccessErr", 288 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK), 289 /* 5*/ FLAG_ENTRY0("CceRspdDataParityErr", 290 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK), 291 /* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr", 292 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK), 293 /* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr", 294 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK), 295 /* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr", 296 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK), 297 /* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 298 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK), 299 /*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 300 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK), 301 /*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError", 302 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK), 303 /*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError", 304 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK), 305 /*13*/ FLAG_ENTRY0("PcicRetryMemCorErr", 306 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK), 307 /*14*/ FLAG_ENTRY0("PcicRetryMemCorErr", 308 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK), 309 /*15*/ FLAG_ENTRY0("PcicPostHdQCorErr", 310 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK), 311 /*16*/ FLAG_ENTRY0("PcicPostHdQCorErr", 312 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK), 313 /*17*/ FLAG_ENTRY0("PcicPostHdQCorErr", 314 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK), 315 /*18*/ FLAG_ENTRY0("PcicCplDatQCorErr", 316 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK), 317 /*19*/ FLAG_ENTRY0("PcicNPostHQParityErr", 318 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK), 319 /*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr", 320 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK), 321 /*21*/ FLAG_ENTRY0("PcicRetryMemUncErr", 322 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK), 323 /*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr", 324 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK), 325 /*23*/ FLAG_ENTRY0("PcicPostHdQUncErr", 326 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK), 327 /*24*/ FLAG_ENTRY0("PcicPostDatQUncErr", 328 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK), 329 /*25*/ FLAG_ENTRY0("PcicCplHdQUncErr", 330 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK), 331 /*26*/ FLAG_ENTRY0("PcicCplDatQUncErr", 332 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK), 333 /*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr", 334 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK), 335 /*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr", 336 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK), 337 /*29*/ FLAG_ENTRY0("PcicReceiveParityErr", 338 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK), 339 /*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr", 340 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK), 341 /*31*/ FLAG_ENTRY0("LATriggered", 342 CCE_ERR_STATUS_LA_TRIGGERED_SMASK), 343 /*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr", 344 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK), 345 /*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr", 346 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK), 347 /*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr", 348 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK), 349 /*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr", 350 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK), 351 /*36*/ FLAG_ENTRY0("CceMsixTableCorErr", 352 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK), 353 /*37*/ FLAG_ENTRY0("CceMsixTableUncErr", 354 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK), 355 /*38*/ FLAG_ENTRY0("CceIntMapCorErr", 356 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK), 357 /*39*/ FLAG_ENTRY0("CceIntMapUncErr", 358 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK), 359 /*40*/ FLAG_ENTRY0("CceMsixCsrParityErr", 360 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK), 361 /*41-63 reserved*/ 362 }; 363 364 /* 365 * Misc Error flags 366 */ 367 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK 368 static struct flag_table misc_err_status_flags[] = { 369 /* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)), 370 /* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)), 371 /* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)), 372 /* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)), 373 /* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)), 374 /* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)), 375 /* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)), 376 /* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)), 377 /* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)), 378 /* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)), 379 /*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)), 380 /*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)), 381 /*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL)) 382 }; 383 384 /* 385 * TXE PIO Error flags and consequences 386 */ 387 static struct flag_table pio_err_status_flags[] = { 388 /* 0*/ FLAG_ENTRY("PioWriteBadCtxt", 389 SEC_WRITE_DROPPED, 390 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK), 391 /* 1*/ FLAG_ENTRY("PioWriteAddrParity", 392 SEC_SPC_FREEZE, 393 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK), 394 /* 2*/ FLAG_ENTRY("PioCsrParity", 395 SEC_SPC_FREEZE, 396 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK), 397 /* 3*/ FLAG_ENTRY("PioSbMemFifo0", 398 SEC_SPC_FREEZE, 399 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK), 400 /* 4*/ FLAG_ENTRY("PioSbMemFifo1", 401 SEC_SPC_FREEZE, 402 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK), 403 /* 5*/ FLAG_ENTRY("PioPccFifoParity", 404 SEC_SPC_FREEZE, 405 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK), 406 /* 6*/ FLAG_ENTRY("PioPecFifoParity", 407 SEC_SPC_FREEZE, 408 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK), 409 /* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity", 410 SEC_SPC_FREEZE, 411 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK), 412 /* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity", 413 SEC_SPC_FREEZE, 414 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK), 415 /* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr", 416 SEC_SPC_FREEZE, 417 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK), 418 /*10*/ FLAG_ENTRY("PioSmPktResetParity", 419 SEC_SPC_FREEZE, 420 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK), 421 /*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc", 422 SEC_SPC_FREEZE, 423 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK), 424 /*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc", 425 SEC_SPC_FREEZE, 426 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK), 427 /*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor", 428 0, 429 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK), 430 /*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor", 431 0, 432 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK), 433 /*15*/ FLAG_ENTRY("PioCreditRetFifoParity", 434 SEC_SPC_FREEZE, 435 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK), 436 /*16*/ FLAG_ENTRY("PioPpmcPblFifo", 437 SEC_SPC_FREEZE, 438 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK), 439 /*17*/ FLAG_ENTRY("PioInitSmIn", 440 0, 441 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK), 442 /*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm", 443 SEC_SPC_FREEZE, 444 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK), 445 /*19*/ FLAG_ENTRY("PioHostAddrMemUnc", 446 SEC_SPC_FREEZE, 447 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK), 448 /*20*/ FLAG_ENTRY("PioHostAddrMemCor", 449 0, 450 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK), 451 /*21*/ FLAG_ENTRY("PioWriteDataParity", 452 SEC_SPC_FREEZE, 453 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK), 454 /*22*/ FLAG_ENTRY("PioStateMachine", 455 SEC_SPC_FREEZE, 456 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK), 457 /*23*/ FLAG_ENTRY("PioWriteQwValidParity", 458 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 459 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK), 460 /*24*/ FLAG_ENTRY("PioBlockQwCountParity", 461 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 462 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK), 463 /*25*/ FLAG_ENTRY("PioVlfVlLenParity", 464 SEC_SPC_FREEZE, 465 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK), 466 /*26*/ FLAG_ENTRY("PioVlfSopParity", 467 SEC_SPC_FREEZE, 468 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK), 469 /*27*/ FLAG_ENTRY("PioVlFifoParity", 470 SEC_SPC_FREEZE, 471 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK), 472 /*28*/ FLAG_ENTRY("PioPpmcBqcMemParity", 473 SEC_SPC_FREEZE, 474 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK), 475 /*29*/ FLAG_ENTRY("PioPpmcSopLen", 476 SEC_SPC_FREEZE, 477 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK), 478 /*30-31 reserved*/ 479 /*32*/ FLAG_ENTRY("PioCurrentFreeCntParity", 480 SEC_SPC_FREEZE, 481 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK), 482 /*33*/ FLAG_ENTRY("PioLastReturnedCntParity", 483 SEC_SPC_FREEZE, 484 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK), 485 /*34*/ FLAG_ENTRY("PioPccSopHeadParity", 486 SEC_SPC_FREEZE, 487 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK), 488 /*35*/ FLAG_ENTRY("PioPecSopHeadParityErr", 489 SEC_SPC_FREEZE, 490 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK), 491 /*36-63 reserved*/ 492 }; 493 494 /* TXE PIO errors that cause an SPC freeze */ 495 #define ALL_PIO_FREEZE_ERR \ 496 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \ 497 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \ 498 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \ 499 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \ 500 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \ 501 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \ 502 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \ 503 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \ 504 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \ 505 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \ 506 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \ 507 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \ 508 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \ 509 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \ 510 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \ 511 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \ 512 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \ 513 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \ 514 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \ 515 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \ 516 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \ 517 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \ 518 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \ 519 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \ 520 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \ 521 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \ 522 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \ 523 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \ 524 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK) 525 526 /* 527 * TXE SDMA Error flags 528 */ 529 static struct flag_table sdma_err_status_flags[] = { 530 /* 0*/ FLAG_ENTRY0("SDmaRpyTagErr", 531 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK), 532 /* 1*/ FLAG_ENTRY0("SDmaCsrParityErr", 533 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK), 534 /* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr", 535 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK), 536 /* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr", 537 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK), 538 /*04-63 reserved*/ 539 }; 540 541 /* TXE SDMA errors that cause an SPC freeze */ 542 #define ALL_SDMA_FREEZE_ERR \ 543 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \ 544 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \ 545 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK) 546 547 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */ 548 #define PORT_DISCARD_EGRESS_ERRS \ 549 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \ 550 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \ 551 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK) 552 553 /* 554 * TXE Egress Error flags 555 */ 556 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK 557 static struct flag_table egress_err_status_flags[] = { 558 /* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)), 559 /* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)), 560 /* 2 reserved */ 561 /* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr", 562 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)), 563 /* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)), 564 /* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)), 565 /* 6 reserved */ 566 /* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr", 567 SEES(TX_PIO_LAUNCH_INTF_PARITY)), 568 /* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr", 569 SEES(TX_SDMA_LAUNCH_INTF_PARITY)), 570 /* 9-10 reserved */ 571 /*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr", 572 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)), 573 /*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)), 574 /*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)), 575 /*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)), 576 /*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)), 577 /*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr", 578 SEES(TX_SDMA0_DISALLOWED_PACKET)), 579 /*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr", 580 SEES(TX_SDMA1_DISALLOWED_PACKET)), 581 /*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr", 582 SEES(TX_SDMA2_DISALLOWED_PACKET)), 583 /*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr", 584 SEES(TX_SDMA3_DISALLOWED_PACKET)), 585 /*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr", 586 SEES(TX_SDMA4_DISALLOWED_PACKET)), 587 /*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr", 588 SEES(TX_SDMA5_DISALLOWED_PACKET)), 589 /*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr", 590 SEES(TX_SDMA6_DISALLOWED_PACKET)), 591 /*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr", 592 SEES(TX_SDMA7_DISALLOWED_PACKET)), 593 /*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr", 594 SEES(TX_SDMA8_DISALLOWED_PACKET)), 595 /*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr", 596 SEES(TX_SDMA9_DISALLOWED_PACKET)), 597 /*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr", 598 SEES(TX_SDMA10_DISALLOWED_PACKET)), 599 /*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr", 600 SEES(TX_SDMA11_DISALLOWED_PACKET)), 601 /*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr", 602 SEES(TX_SDMA12_DISALLOWED_PACKET)), 603 /*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr", 604 SEES(TX_SDMA13_DISALLOWED_PACKET)), 605 /*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr", 606 SEES(TX_SDMA14_DISALLOWED_PACKET)), 607 /*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr", 608 SEES(TX_SDMA15_DISALLOWED_PACKET)), 609 /*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr", 610 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)), 611 /*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr", 612 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)), 613 /*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr", 614 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)), 615 /*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr", 616 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)), 617 /*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr", 618 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)), 619 /*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr", 620 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)), 621 /*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr", 622 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)), 623 /*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr", 624 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)), 625 /*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr", 626 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)), 627 /*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)), 628 /*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)), 629 /*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)), 630 /*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)), 631 /*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)), 632 /*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)), 633 /*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)), 634 /*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)), 635 /*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)), 636 /*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)), 637 /*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)), 638 /*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)), 639 /*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)), 640 /*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)), 641 /*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)), 642 /*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)), 643 /*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)), 644 /*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)), 645 /*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)), 646 /*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)), 647 /*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)), 648 /*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr", 649 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)), 650 /*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr", 651 SEES(TX_READ_PIO_MEMORY_CSR_UNC)), 652 }; 653 654 /* 655 * TXE Egress Error Info flags 656 */ 657 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK 658 static struct flag_table egress_err_info_flags[] = { 659 /* 0*/ FLAG_ENTRY0("Reserved", 0ull), 660 /* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)), 661 /* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 662 /* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 663 /* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)), 664 /* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)), 665 /* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)), 666 /* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)), 667 /* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)), 668 /* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)), 669 /*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)), 670 /*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)), 671 /*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)), 672 /*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)), 673 /*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)), 674 /*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)), 675 /*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)), 676 /*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)), 677 /*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)), 678 /*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)), 679 /*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)), 680 /*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)), 681 }; 682 683 /* TXE Egress errors that cause an SPC freeze */ 684 #define ALL_TXE_EGRESS_FREEZE_ERR \ 685 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \ 686 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \ 687 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \ 688 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \ 689 | SEES(TX_LAUNCH_CSR_PARITY) \ 690 | SEES(TX_SBRD_CTL_CSR_PARITY) \ 691 | SEES(TX_CONFIG_PARITY) \ 692 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \ 693 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \ 694 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \ 695 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \ 696 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \ 697 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \ 698 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \ 699 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \ 700 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \ 701 | SEES(TX_CREDIT_RETURN_PARITY)) 702 703 /* 704 * TXE Send error flags 705 */ 706 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK 707 static struct flag_table send_err_status_flags[] = { 708 /* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)), 709 /* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)), 710 /* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR)) 711 }; 712 713 /* 714 * TXE Send Context Error flags and consequences 715 */ 716 static struct flag_table sc_err_status_flags[] = { 717 /* 0*/ FLAG_ENTRY("InconsistentSop", 718 SEC_PACKET_DROPPED | SEC_SC_HALTED, 719 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK), 720 /* 1*/ FLAG_ENTRY("DisallowedPacket", 721 SEC_PACKET_DROPPED | SEC_SC_HALTED, 722 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK), 723 /* 2*/ FLAG_ENTRY("WriteCrossesBoundary", 724 SEC_WRITE_DROPPED | SEC_SC_HALTED, 725 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK), 726 /* 3*/ FLAG_ENTRY("WriteOverflow", 727 SEC_WRITE_DROPPED | SEC_SC_HALTED, 728 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK), 729 /* 4*/ FLAG_ENTRY("WriteOutOfBounds", 730 SEC_WRITE_DROPPED | SEC_SC_HALTED, 731 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK), 732 /* 5-63 reserved*/ 733 }; 734 735 /* 736 * RXE Receive Error flags 737 */ 738 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK 739 static struct flag_table rxe_err_status_flags[] = { 740 /* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)), 741 /* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)), 742 /* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)), 743 /* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)), 744 /* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)), 745 /* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)), 746 /* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)), 747 /* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)), 748 /* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)), 749 /* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)), 750 /*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)), 751 /*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)), 752 /*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)), 753 /*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)), 754 /*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)), 755 /*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)), 756 /*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr", 757 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)), 758 /*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)), 759 /*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)), 760 /*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr", 761 RXES(RBUF_BLOCK_LIST_READ_UNC)), 762 /*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr", 763 RXES(RBUF_BLOCK_LIST_READ_COR)), 764 /*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr", 765 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)), 766 /*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr", 767 RXES(RBUF_CSR_QENT_CNT_PARITY)), 768 /*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr", 769 RXES(RBUF_CSR_QNEXT_BUF_PARITY)), 770 /*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr", 771 RXES(RBUF_CSR_QVLD_BIT_PARITY)), 772 /*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)), 773 /*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)), 774 /*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr", 775 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)), 776 /*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)), 777 /*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)), 778 /*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)), 779 /*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)), 780 /*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)), 781 /*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)), 782 /*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)), 783 /*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr", 784 RXES(RBUF_FL_INITDONE_PARITY)), 785 /*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr", 786 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)), 787 /*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)), 788 /*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)), 789 /*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)), 790 /*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr", 791 RXES(LOOKUP_DES_PART1_UNC_COR)), 792 /*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr", 793 RXES(LOOKUP_DES_PART2_PARITY)), 794 /*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)), 795 /*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)), 796 /*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)), 797 /*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)), 798 /*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)), 799 /*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)), 800 /*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)), 801 /*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)), 802 /*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)), 803 /*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)), 804 /*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)), 805 /*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)), 806 /*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)), 807 /*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)), 808 /*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)), 809 /*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)), 810 /*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)), 811 /*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)), 812 /*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)), 813 /*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)), 814 /*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)), 815 /*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY)) 816 }; 817 818 /* RXE errors that will trigger an SPC freeze */ 819 #define ALL_RXE_FREEZE_ERR \ 820 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \ 821 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \ 822 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \ 823 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \ 824 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \ 825 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \ 826 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \ 827 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \ 828 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \ 829 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \ 830 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \ 831 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \ 832 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \ 833 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \ 834 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \ 835 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \ 836 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \ 837 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \ 838 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \ 839 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \ 840 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \ 841 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \ 842 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \ 843 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \ 844 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \ 845 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \ 846 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \ 847 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \ 848 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \ 849 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \ 850 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \ 851 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \ 852 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \ 853 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \ 854 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \ 855 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \ 856 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \ 857 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \ 858 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \ 859 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \ 860 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \ 861 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \ 862 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \ 863 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK) 864 865 #define RXE_FREEZE_ABORT_MASK \ 866 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \ 867 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \ 868 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK) 869 870 /* 871 * DCC Error Flags 872 */ 873 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK 874 static struct flag_table dcc_err_flags[] = { 875 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)), 876 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)), 877 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)), 878 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)), 879 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)), 880 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)), 881 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)), 882 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)), 883 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)), 884 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)), 885 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)), 886 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)), 887 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)), 888 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)), 889 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)), 890 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)), 891 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)), 892 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)), 893 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)), 894 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)), 895 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)), 896 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)), 897 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)), 898 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)), 899 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)), 900 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)), 901 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)), 902 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)), 903 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)), 904 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)), 905 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)), 906 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)), 907 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)), 908 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)), 909 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)), 910 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)), 911 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)), 912 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)), 913 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)), 914 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)), 915 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)), 916 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)), 917 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)), 918 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)), 919 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)), 920 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)), 921 }; 922 923 /* 924 * LCB error flags 925 */ 926 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK 927 static struct flag_table lcb_err_flags[] = { 928 /* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)), 929 /* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)), 930 /* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)), 931 /* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST", 932 LCBE(ALL_LNS_FAILED_REINIT_TEST)), 933 /* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)), 934 /* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)), 935 /* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)), 936 /* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)), 937 /* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)), 938 /* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)), 939 /*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)), 940 /*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)), 941 /*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)), 942 /*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER", 943 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)), 944 /*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)), 945 /*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)), 946 /*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)), 947 /*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)), 948 /*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)), 949 /*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE", 950 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)), 951 /*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)), 952 /*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)), 953 /*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)), 954 /*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)), 955 /*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)), 956 /*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)), 957 /*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP", 958 LCBE(RST_FOR_INCOMPLT_RND_TRIP)), 959 /*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)), 960 /*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE", 961 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)), 962 /*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR", 963 LCBE(REDUNDANT_FLIT_PARITY_ERR)) 964 }; 965 966 /* 967 * DC8051 Error Flags 968 */ 969 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK 970 static struct flag_table dc8051_err_flags[] = { 971 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)), 972 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)), 973 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)), 974 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)), 975 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)), 976 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)), 977 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)), 978 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)), 979 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES", 980 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)), 981 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)), 982 }; 983 984 /* 985 * DC8051 Information Error flags 986 * 987 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field. 988 */ 989 static struct flag_table dc8051_info_err_flags[] = { 990 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED), 991 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME), 992 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET), 993 FLAG_ENTRY0("Serdes internal loopback failure", 994 FAILED_SERDES_INTERNAL_LOOPBACK), 995 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT), 996 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING), 997 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE), 998 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM), 999 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ), 1000 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1), 1001 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2), 1002 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT), 1003 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT), 1004 FLAG_ENTRY0("External Device Request Timeout", 1005 EXTERNAL_DEVICE_REQ_TIMEOUT), 1006 }; 1007 1008 /* 1009 * DC8051 Information Host Information flags 1010 * 1011 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field. 1012 */ 1013 static struct flag_table dc8051_info_host_msg_flags[] = { 1014 FLAG_ENTRY0("Host request done", 0x0001), 1015 FLAG_ENTRY0("BC PWR_MGM message", 0x0002), 1016 FLAG_ENTRY0("BC SMA message", 0x0004), 1017 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008), 1018 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010), 1019 FLAG_ENTRY0("External device config request", 0x0020), 1020 FLAG_ENTRY0("VerifyCap all frames received", 0x0040), 1021 FLAG_ENTRY0("LinkUp achieved", 0x0080), 1022 FLAG_ENTRY0("Link going down", 0x0100), 1023 FLAG_ENTRY0("Link width downgraded", 0x0200), 1024 }; 1025 1026 static u32 encoded_size(u32 size); 1027 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate); 1028 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state); 1029 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 1030 u8 *continuous); 1031 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 1032 u8 *vcu, u16 *vl15buf, u8 *crc_sizes); 1033 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 1034 u8 *remote_tx_rate, u16 *link_widths); 1035 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits, 1036 u8 *flag_bits, u16 *link_widths); 1037 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 1038 u8 *device_rev); 1039 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx); 1040 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx, 1041 u8 *tx_polarity_inversion, 1042 u8 *rx_polarity_inversion, u8 *max_rate); 1043 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 1044 unsigned int context, u64 err_status); 1045 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg); 1046 static void handle_dcc_err(struct hfi1_devdata *dd, 1047 unsigned int context, u64 err_status); 1048 static void handle_lcb_err(struct hfi1_devdata *dd, 1049 unsigned int context, u64 err_status); 1050 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg); 1051 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1052 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1053 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1054 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1055 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1056 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1057 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1058 static void set_partition_keys(struct hfi1_pportdata *ppd); 1059 static const char *link_state_name(u32 state); 1060 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, 1061 u32 state); 1062 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 1063 u64 *out_data); 1064 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data); 1065 static int thermal_init(struct hfi1_devdata *dd); 1066 1067 static void update_statusp(struct hfi1_pportdata *ppd, u32 state); 1068 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, 1069 int msecs); 1070 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1071 int msecs); 1072 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state); 1073 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state); 1074 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1075 int msecs); 1076 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc); 1077 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr); 1078 static void handle_temp_err(struct hfi1_devdata *dd); 1079 static void dc_shutdown(struct hfi1_devdata *dd); 1080 static void dc_start(struct hfi1_devdata *dd); 1081 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 1082 unsigned int *np); 1083 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); 1084 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); 1085 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index); 1086 1087 /* 1088 * Error interrupt table entry. This is used as input to the interrupt 1089 * "clear down" routine used for all second tier error interrupt register. 1090 * Second tier interrupt registers have a single bit representing them 1091 * in the top-level CceIntStatus. 1092 */ 1093 struct err_reg_info { 1094 u32 status; /* status CSR offset */ 1095 u32 clear; /* clear CSR offset */ 1096 u32 mask; /* mask CSR offset */ 1097 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg); 1098 const char *desc; 1099 }; 1100 1101 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END - IS_GENERAL_ERR_START) 1102 #define NUM_DC_ERRS (IS_DC_END - IS_DC_START) 1103 #define NUM_VARIOUS (IS_VARIOUS_END - IS_VARIOUS_START) 1104 1105 /* 1106 * Helpers for building HFI and DC error interrupt table entries. Different 1107 * helpers are needed because of inconsistent register names. 1108 */ 1109 #define EE(reg, handler, desc) \ 1110 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \ 1111 handler, desc } 1112 #define DC_EE1(reg, handler, desc) \ 1113 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc } 1114 #define DC_EE2(reg, handler, desc) \ 1115 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc } 1116 1117 /* 1118 * Table of the "misc" grouping of error interrupts. Each entry refers to 1119 * another register containing more information. 1120 */ 1121 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = { 1122 /* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"), 1123 /* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"), 1124 /* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"), 1125 /* 3*/ { 0, 0, 0, NULL }, /* reserved */ 1126 /* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"), 1127 /* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"), 1128 /* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"), 1129 /* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr") 1130 /* the rest are reserved */ 1131 }; 1132 1133 /* 1134 * Index into the Various section of the interrupt sources 1135 * corresponding to the Critical Temperature interrupt. 1136 */ 1137 #define TCRIT_INT_SOURCE 4 1138 1139 /* 1140 * SDMA error interrupt entry - refers to another register containing more 1141 * information. 1142 */ 1143 static const struct err_reg_info sdma_eng_err = 1144 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr"); 1145 1146 static const struct err_reg_info various_err[NUM_VARIOUS] = { 1147 /* 0*/ { 0, 0, 0, NULL }, /* PbcInt */ 1148 /* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */ 1149 /* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"), 1150 /* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"), 1151 /* 4*/ { 0, 0, 0, NULL }, /* TCritInt */ 1152 /* rest are reserved */ 1153 }; 1154 1155 /* 1156 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG 1157 * register can not be derived from the MTU value because 10K is not 1158 * a power of 2. Therefore, we need a constant. Everything else can 1159 * be calculated. 1160 */ 1161 #define DCC_CFG_PORT_MTU_CAP_10240 7 1162 1163 /* 1164 * Table of the DC grouping of error interrupts. Each entry refers to 1165 * another register containing more information. 1166 */ 1167 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = { 1168 /* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"), 1169 /* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"), 1170 /* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"), 1171 /* 3*/ /* dc_lbm_int - special, see is_dc_int() */ 1172 /* the rest are reserved */ 1173 }; 1174 1175 struct cntr_entry { 1176 /* 1177 * counter name 1178 */ 1179 char *name; 1180 1181 /* 1182 * csr to read for name (if applicable) 1183 */ 1184 u64 csr; 1185 1186 /* 1187 * offset into dd or ppd to store the counter's value 1188 */ 1189 int offset; 1190 1191 /* 1192 * flags 1193 */ 1194 u8 flags; 1195 1196 /* 1197 * accessor for stat element, context either dd or ppd 1198 */ 1199 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl, 1200 int mode, u64 data); 1201 }; 1202 1203 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0 1204 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159 1205 1206 #define CNTR_ELEM(name, csr, offset, flags, accessor) \ 1207 { \ 1208 name, \ 1209 csr, \ 1210 offset, \ 1211 flags, \ 1212 accessor \ 1213 } 1214 1215 /* 32bit RXE */ 1216 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1217 CNTR_ELEM(#name, \ 1218 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1219 0, flags | CNTR_32BIT, \ 1220 port_access_u32_csr) 1221 1222 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \ 1223 CNTR_ELEM(#name, \ 1224 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1225 0, flags | CNTR_32BIT, \ 1226 dev_access_u32_csr) 1227 1228 /* 64bit RXE */ 1229 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1230 CNTR_ELEM(#name, \ 1231 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1232 0, flags, \ 1233 port_access_u64_csr) 1234 1235 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \ 1236 CNTR_ELEM(#name, \ 1237 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1238 0, flags, \ 1239 dev_access_u64_csr) 1240 1241 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx 1242 #define OVR_ELM(ctx) \ 1243 CNTR_ELEM("RcvHdrOvr" #ctx, \ 1244 (RCV_HDR_OVFL_CNT + ctx * 0x100), \ 1245 0, CNTR_NORMAL, port_access_u64_csr) 1246 1247 /* 32bit TXE */ 1248 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1249 CNTR_ELEM(#name, \ 1250 (counter * 8 + SEND_COUNTER_ARRAY32), \ 1251 0, flags | CNTR_32BIT, \ 1252 port_access_u32_csr) 1253 1254 /* 64bit TXE */ 1255 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1256 CNTR_ELEM(#name, \ 1257 (counter * 8 + SEND_COUNTER_ARRAY64), \ 1258 0, flags, \ 1259 port_access_u64_csr) 1260 1261 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \ 1262 CNTR_ELEM(#name,\ 1263 counter * 8 + SEND_COUNTER_ARRAY64, \ 1264 0, \ 1265 flags, \ 1266 dev_access_u64_csr) 1267 1268 /* CCE */ 1269 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \ 1270 CNTR_ELEM(#name, \ 1271 (counter * 8 + CCE_COUNTER_ARRAY32), \ 1272 0, flags | CNTR_32BIT, \ 1273 dev_access_u32_csr) 1274 1275 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \ 1276 CNTR_ELEM(#name, \ 1277 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \ 1278 0, flags | CNTR_32BIT, \ 1279 dev_access_u32_csr) 1280 1281 /* DC */ 1282 #define DC_PERF_CNTR(name, counter, flags) \ 1283 CNTR_ELEM(#name, \ 1284 counter, \ 1285 0, \ 1286 flags, \ 1287 dev_access_u64_csr) 1288 1289 #define DC_PERF_CNTR_LCB(name, counter, flags) \ 1290 CNTR_ELEM(#name, \ 1291 counter, \ 1292 0, \ 1293 flags, \ 1294 dc_access_lcb_cntr) 1295 1296 /* ibp counters */ 1297 #define SW_IBP_CNTR(name, cntr) \ 1298 CNTR_ELEM(#name, \ 1299 0, \ 1300 0, \ 1301 CNTR_SYNTH, \ 1302 access_ibp_##cntr) 1303 1304 /** 1305 * hfi_addr_from_offset - return addr for readq/writeq 1306 * @dd - the dd device 1307 * @offset - the offset of the CSR within bar0 1308 * 1309 * This routine selects the appropriate base address 1310 * based on the indicated offset. 1311 */ 1312 static inline void __iomem *hfi1_addr_from_offset( 1313 const struct hfi1_devdata *dd, 1314 u32 offset) 1315 { 1316 if (offset >= dd->base2_start) 1317 return dd->kregbase2 + (offset - dd->base2_start); 1318 return dd->kregbase1 + offset; 1319 } 1320 1321 /** 1322 * read_csr - read CSR at the indicated offset 1323 * @dd - the dd device 1324 * @offset - the offset of the CSR within bar0 1325 * 1326 * Return: the value read or all FF's if there 1327 * is no mapping 1328 */ 1329 u64 read_csr(const struct hfi1_devdata *dd, u32 offset) 1330 { 1331 if (dd->flags & HFI1_PRESENT) 1332 return readq(hfi1_addr_from_offset(dd, offset)); 1333 return -1; 1334 } 1335 1336 /** 1337 * write_csr - write CSR at the indicated offset 1338 * @dd - the dd device 1339 * @offset - the offset of the CSR within bar0 1340 * @value - value to write 1341 */ 1342 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value) 1343 { 1344 if (dd->flags & HFI1_PRESENT) { 1345 void __iomem *base = hfi1_addr_from_offset(dd, offset); 1346 1347 /* avoid write to RcvArray */ 1348 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start)) 1349 return; 1350 writeq(value, base); 1351 } 1352 } 1353 1354 /** 1355 * get_csr_addr - return te iomem address for offset 1356 * @dd - the dd device 1357 * @offset - the offset of the CSR within bar0 1358 * 1359 * Return: The iomem address to use in subsequent 1360 * writeq/readq operations. 1361 */ 1362 void __iomem *get_csr_addr( 1363 const struct hfi1_devdata *dd, 1364 u32 offset) 1365 { 1366 if (dd->flags & HFI1_PRESENT) 1367 return hfi1_addr_from_offset(dd, offset); 1368 return NULL; 1369 } 1370 1371 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr, 1372 int mode, u64 value) 1373 { 1374 u64 ret; 1375 1376 if (mode == CNTR_MODE_R) { 1377 ret = read_csr(dd, csr); 1378 } else if (mode == CNTR_MODE_W) { 1379 write_csr(dd, csr, value); 1380 ret = value; 1381 } else { 1382 dd_dev_err(dd, "Invalid cntr register access mode"); 1383 return 0; 1384 } 1385 1386 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode); 1387 return ret; 1388 } 1389 1390 /* Dev Access */ 1391 static u64 dev_access_u32_csr(const struct cntr_entry *entry, 1392 void *context, int vl, int mode, u64 data) 1393 { 1394 struct hfi1_devdata *dd = context; 1395 u64 csr = entry->csr; 1396 1397 if (entry->flags & CNTR_SDMA) { 1398 if (vl == CNTR_INVALID_VL) 1399 return 0; 1400 csr += 0x100 * vl; 1401 } else { 1402 if (vl != CNTR_INVALID_VL) 1403 return 0; 1404 } 1405 return read_write_csr(dd, csr, mode, data); 1406 } 1407 1408 static u64 access_sde_err_cnt(const struct cntr_entry *entry, 1409 void *context, int idx, int mode, u64 data) 1410 { 1411 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1412 1413 if (dd->per_sdma && idx < dd->num_sdma) 1414 return dd->per_sdma[idx].err_cnt; 1415 return 0; 1416 } 1417 1418 static u64 access_sde_int_cnt(const struct cntr_entry *entry, 1419 void *context, int idx, int mode, u64 data) 1420 { 1421 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1422 1423 if (dd->per_sdma && idx < dd->num_sdma) 1424 return dd->per_sdma[idx].sdma_int_cnt; 1425 return 0; 1426 } 1427 1428 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry, 1429 void *context, int idx, int mode, u64 data) 1430 { 1431 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1432 1433 if (dd->per_sdma && idx < dd->num_sdma) 1434 return dd->per_sdma[idx].idle_int_cnt; 1435 return 0; 1436 } 1437 1438 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry, 1439 void *context, int idx, int mode, 1440 u64 data) 1441 { 1442 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1443 1444 if (dd->per_sdma && idx < dd->num_sdma) 1445 return dd->per_sdma[idx].progress_int_cnt; 1446 return 0; 1447 } 1448 1449 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context, 1450 int vl, int mode, u64 data) 1451 { 1452 struct hfi1_devdata *dd = context; 1453 1454 u64 val = 0; 1455 u64 csr = entry->csr; 1456 1457 if (entry->flags & CNTR_VL) { 1458 if (vl == CNTR_INVALID_VL) 1459 return 0; 1460 csr += 8 * vl; 1461 } else { 1462 if (vl != CNTR_INVALID_VL) 1463 return 0; 1464 } 1465 1466 val = read_write_csr(dd, csr, mode, data); 1467 return val; 1468 } 1469 1470 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context, 1471 int vl, int mode, u64 data) 1472 { 1473 struct hfi1_devdata *dd = context; 1474 u32 csr = entry->csr; 1475 int ret = 0; 1476 1477 if (vl != CNTR_INVALID_VL) 1478 return 0; 1479 if (mode == CNTR_MODE_R) 1480 ret = read_lcb_csr(dd, csr, &data); 1481 else if (mode == CNTR_MODE_W) 1482 ret = write_lcb_csr(dd, csr, data); 1483 1484 if (ret) { 1485 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr); 1486 return 0; 1487 } 1488 1489 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode); 1490 return data; 1491 } 1492 1493 /* Port Access */ 1494 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context, 1495 int vl, int mode, u64 data) 1496 { 1497 struct hfi1_pportdata *ppd = context; 1498 1499 if (vl != CNTR_INVALID_VL) 1500 return 0; 1501 return read_write_csr(ppd->dd, entry->csr, mode, data); 1502 } 1503 1504 static u64 port_access_u64_csr(const struct cntr_entry *entry, 1505 void *context, int vl, int mode, u64 data) 1506 { 1507 struct hfi1_pportdata *ppd = context; 1508 u64 val; 1509 u64 csr = entry->csr; 1510 1511 if (entry->flags & CNTR_VL) { 1512 if (vl == CNTR_INVALID_VL) 1513 return 0; 1514 csr += 8 * vl; 1515 } else { 1516 if (vl != CNTR_INVALID_VL) 1517 return 0; 1518 } 1519 val = read_write_csr(ppd->dd, csr, mode, data); 1520 return val; 1521 } 1522 1523 /* Software defined */ 1524 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode, 1525 u64 data) 1526 { 1527 u64 ret; 1528 1529 if (mode == CNTR_MODE_R) { 1530 ret = *cntr; 1531 } else if (mode == CNTR_MODE_W) { 1532 *cntr = data; 1533 ret = data; 1534 } else { 1535 dd_dev_err(dd, "Invalid cntr sw access mode"); 1536 return 0; 1537 } 1538 1539 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode); 1540 1541 return ret; 1542 } 1543 1544 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context, 1545 int vl, int mode, u64 data) 1546 { 1547 struct hfi1_pportdata *ppd = context; 1548 1549 if (vl != CNTR_INVALID_VL) 1550 return 0; 1551 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data); 1552 } 1553 1554 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context, 1555 int vl, int mode, u64 data) 1556 { 1557 struct hfi1_pportdata *ppd = context; 1558 1559 if (vl != CNTR_INVALID_VL) 1560 return 0; 1561 return read_write_sw(ppd->dd, &ppd->link_up, mode, data); 1562 } 1563 1564 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry, 1565 void *context, int vl, int mode, 1566 u64 data) 1567 { 1568 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1569 1570 if (vl != CNTR_INVALID_VL) 1571 return 0; 1572 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data); 1573 } 1574 1575 static u64 access_sw_xmit_discards(const struct cntr_entry *entry, 1576 void *context, int vl, int mode, u64 data) 1577 { 1578 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1579 u64 zero = 0; 1580 u64 *counter; 1581 1582 if (vl == CNTR_INVALID_VL) 1583 counter = &ppd->port_xmit_discards; 1584 else if (vl >= 0 && vl < C_VL_COUNT) 1585 counter = &ppd->port_xmit_discards_vl[vl]; 1586 else 1587 counter = &zero; 1588 1589 return read_write_sw(ppd->dd, counter, mode, data); 1590 } 1591 1592 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry, 1593 void *context, int vl, int mode, 1594 u64 data) 1595 { 1596 struct hfi1_pportdata *ppd = context; 1597 1598 if (vl != CNTR_INVALID_VL) 1599 return 0; 1600 1601 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors, 1602 mode, data); 1603 } 1604 1605 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry, 1606 void *context, int vl, int mode, u64 data) 1607 { 1608 struct hfi1_pportdata *ppd = context; 1609 1610 if (vl != CNTR_INVALID_VL) 1611 return 0; 1612 1613 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors, 1614 mode, data); 1615 } 1616 1617 u64 get_all_cpu_total(u64 __percpu *cntr) 1618 { 1619 int cpu; 1620 u64 counter = 0; 1621 1622 for_each_possible_cpu(cpu) 1623 counter += *per_cpu_ptr(cntr, cpu); 1624 return counter; 1625 } 1626 1627 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val, 1628 u64 __percpu *cntr, 1629 int vl, int mode, u64 data) 1630 { 1631 u64 ret = 0; 1632 1633 if (vl != CNTR_INVALID_VL) 1634 return 0; 1635 1636 if (mode == CNTR_MODE_R) { 1637 ret = get_all_cpu_total(cntr) - *z_val; 1638 } else if (mode == CNTR_MODE_W) { 1639 /* A write can only zero the counter */ 1640 if (data == 0) 1641 *z_val = get_all_cpu_total(cntr); 1642 else 1643 dd_dev_err(dd, "Per CPU cntrs can only be zeroed"); 1644 } else { 1645 dd_dev_err(dd, "Invalid cntr sw cpu access mode"); 1646 return 0; 1647 } 1648 1649 return ret; 1650 } 1651 1652 static u64 access_sw_cpu_intr(const struct cntr_entry *entry, 1653 void *context, int vl, int mode, u64 data) 1654 { 1655 struct hfi1_devdata *dd = context; 1656 1657 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl, 1658 mode, data); 1659 } 1660 1661 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry, 1662 void *context, int vl, int mode, u64 data) 1663 { 1664 struct hfi1_devdata *dd = context; 1665 1666 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl, 1667 mode, data); 1668 } 1669 1670 static u64 access_sw_pio_wait(const struct cntr_entry *entry, 1671 void *context, int vl, int mode, u64 data) 1672 { 1673 struct hfi1_devdata *dd = context; 1674 1675 return dd->verbs_dev.n_piowait; 1676 } 1677 1678 static u64 access_sw_pio_drain(const struct cntr_entry *entry, 1679 void *context, int vl, int mode, u64 data) 1680 { 1681 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1682 1683 return dd->verbs_dev.n_piodrain; 1684 } 1685 1686 static u64 access_sw_vtx_wait(const struct cntr_entry *entry, 1687 void *context, int vl, int mode, u64 data) 1688 { 1689 struct hfi1_devdata *dd = context; 1690 1691 return dd->verbs_dev.n_txwait; 1692 } 1693 1694 static u64 access_sw_kmem_wait(const struct cntr_entry *entry, 1695 void *context, int vl, int mode, u64 data) 1696 { 1697 struct hfi1_devdata *dd = context; 1698 1699 return dd->verbs_dev.n_kmem_wait; 1700 } 1701 1702 static u64 access_sw_send_schedule(const struct cntr_entry *entry, 1703 void *context, int vl, int mode, u64 data) 1704 { 1705 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1706 1707 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl, 1708 mode, data); 1709 } 1710 1711 /* Software counters for the error status bits within MISC_ERR_STATUS */ 1712 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry, 1713 void *context, int vl, int mode, 1714 u64 data) 1715 { 1716 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1717 1718 return dd->misc_err_status_cnt[12]; 1719 } 1720 1721 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry, 1722 void *context, int vl, int mode, 1723 u64 data) 1724 { 1725 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1726 1727 return dd->misc_err_status_cnt[11]; 1728 } 1729 1730 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry, 1731 void *context, int vl, int mode, 1732 u64 data) 1733 { 1734 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1735 1736 return dd->misc_err_status_cnt[10]; 1737 } 1738 1739 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry, 1740 void *context, int vl, 1741 int mode, u64 data) 1742 { 1743 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1744 1745 return dd->misc_err_status_cnt[9]; 1746 } 1747 1748 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry, 1749 void *context, int vl, int mode, 1750 u64 data) 1751 { 1752 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1753 1754 return dd->misc_err_status_cnt[8]; 1755 } 1756 1757 static u64 access_misc_efuse_read_bad_addr_err_cnt( 1758 const struct cntr_entry *entry, 1759 void *context, int vl, int mode, u64 data) 1760 { 1761 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1762 1763 return dd->misc_err_status_cnt[7]; 1764 } 1765 1766 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry, 1767 void *context, int vl, 1768 int mode, u64 data) 1769 { 1770 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1771 1772 return dd->misc_err_status_cnt[6]; 1773 } 1774 1775 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry, 1776 void *context, int vl, int mode, 1777 u64 data) 1778 { 1779 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1780 1781 return dd->misc_err_status_cnt[5]; 1782 } 1783 1784 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry, 1785 void *context, int vl, int mode, 1786 u64 data) 1787 { 1788 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1789 1790 return dd->misc_err_status_cnt[4]; 1791 } 1792 1793 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry, 1794 void *context, int vl, 1795 int mode, u64 data) 1796 { 1797 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1798 1799 return dd->misc_err_status_cnt[3]; 1800 } 1801 1802 static u64 access_misc_csr_write_bad_addr_err_cnt( 1803 const struct cntr_entry *entry, 1804 void *context, int vl, int mode, u64 data) 1805 { 1806 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1807 1808 return dd->misc_err_status_cnt[2]; 1809 } 1810 1811 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1812 void *context, int vl, 1813 int mode, u64 data) 1814 { 1815 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1816 1817 return dd->misc_err_status_cnt[1]; 1818 } 1819 1820 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry, 1821 void *context, int vl, int mode, 1822 u64 data) 1823 { 1824 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1825 1826 return dd->misc_err_status_cnt[0]; 1827 } 1828 1829 /* 1830 * Software counter for the aggregate of 1831 * individual CceErrStatus counters 1832 */ 1833 static u64 access_sw_cce_err_status_aggregated_cnt( 1834 const struct cntr_entry *entry, 1835 void *context, int vl, int mode, u64 data) 1836 { 1837 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1838 1839 return dd->sw_cce_err_status_aggregate; 1840 } 1841 1842 /* 1843 * Software counters corresponding to each of the 1844 * error status bits within CceErrStatus 1845 */ 1846 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry, 1847 void *context, int vl, int mode, 1848 u64 data) 1849 { 1850 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1851 1852 return dd->cce_err_status_cnt[40]; 1853 } 1854 1855 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry, 1856 void *context, int vl, int mode, 1857 u64 data) 1858 { 1859 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1860 1861 return dd->cce_err_status_cnt[39]; 1862 } 1863 1864 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry, 1865 void *context, int vl, int mode, 1866 u64 data) 1867 { 1868 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1869 1870 return dd->cce_err_status_cnt[38]; 1871 } 1872 1873 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry, 1874 void *context, int vl, int mode, 1875 u64 data) 1876 { 1877 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1878 1879 return dd->cce_err_status_cnt[37]; 1880 } 1881 1882 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry, 1883 void *context, int vl, int mode, 1884 u64 data) 1885 { 1886 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1887 1888 return dd->cce_err_status_cnt[36]; 1889 } 1890 1891 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt( 1892 const struct cntr_entry *entry, 1893 void *context, int vl, int mode, u64 data) 1894 { 1895 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1896 1897 return dd->cce_err_status_cnt[35]; 1898 } 1899 1900 static u64 access_cce_rcpl_async_fifo_parity_err_cnt( 1901 const struct cntr_entry *entry, 1902 void *context, int vl, int mode, u64 data) 1903 { 1904 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1905 1906 return dd->cce_err_status_cnt[34]; 1907 } 1908 1909 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry, 1910 void *context, int vl, 1911 int mode, u64 data) 1912 { 1913 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1914 1915 return dd->cce_err_status_cnt[33]; 1916 } 1917 1918 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1919 void *context, int vl, int mode, 1920 u64 data) 1921 { 1922 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1923 1924 return dd->cce_err_status_cnt[32]; 1925 } 1926 1927 static u64 access_la_triggered_cnt(const struct cntr_entry *entry, 1928 void *context, int vl, int mode, u64 data) 1929 { 1930 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1931 1932 return dd->cce_err_status_cnt[31]; 1933 } 1934 1935 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry, 1936 void *context, int vl, int mode, 1937 u64 data) 1938 { 1939 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1940 1941 return dd->cce_err_status_cnt[30]; 1942 } 1943 1944 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry, 1945 void *context, int vl, int mode, 1946 u64 data) 1947 { 1948 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1949 1950 return dd->cce_err_status_cnt[29]; 1951 } 1952 1953 static u64 access_pcic_transmit_back_parity_err_cnt( 1954 const struct cntr_entry *entry, 1955 void *context, int vl, int mode, u64 data) 1956 { 1957 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1958 1959 return dd->cce_err_status_cnt[28]; 1960 } 1961 1962 static u64 access_pcic_transmit_front_parity_err_cnt( 1963 const struct cntr_entry *entry, 1964 void *context, int vl, int mode, u64 data) 1965 { 1966 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1967 1968 return dd->cce_err_status_cnt[27]; 1969 } 1970 1971 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1972 void *context, int vl, int mode, 1973 u64 data) 1974 { 1975 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1976 1977 return dd->cce_err_status_cnt[26]; 1978 } 1979 1980 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1981 void *context, int vl, int mode, 1982 u64 data) 1983 { 1984 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1985 1986 return dd->cce_err_status_cnt[25]; 1987 } 1988 1989 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1990 void *context, int vl, int mode, 1991 u64 data) 1992 { 1993 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1994 1995 return dd->cce_err_status_cnt[24]; 1996 } 1997 1998 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1999 void *context, int vl, int mode, 2000 u64 data) 2001 { 2002 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2003 2004 return dd->cce_err_status_cnt[23]; 2005 } 2006 2007 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry, 2008 void *context, int vl, 2009 int mode, u64 data) 2010 { 2011 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2012 2013 return dd->cce_err_status_cnt[22]; 2014 } 2015 2016 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry, 2017 void *context, int vl, int mode, 2018 u64 data) 2019 { 2020 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2021 2022 return dd->cce_err_status_cnt[21]; 2023 } 2024 2025 static u64 access_pcic_n_post_dat_q_parity_err_cnt( 2026 const struct cntr_entry *entry, 2027 void *context, int vl, int mode, u64 data) 2028 { 2029 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2030 2031 return dd->cce_err_status_cnt[20]; 2032 } 2033 2034 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry, 2035 void *context, int vl, 2036 int mode, u64 data) 2037 { 2038 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2039 2040 return dd->cce_err_status_cnt[19]; 2041 } 2042 2043 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry, 2044 void *context, int vl, int mode, 2045 u64 data) 2046 { 2047 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2048 2049 return dd->cce_err_status_cnt[18]; 2050 } 2051 2052 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry, 2053 void *context, int vl, int mode, 2054 u64 data) 2055 { 2056 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2057 2058 return dd->cce_err_status_cnt[17]; 2059 } 2060 2061 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry, 2062 void *context, int vl, int mode, 2063 u64 data) 2064 { 2065 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2066 2067 return dd->cce_err_status_cnt[16]; 2068 } 2069 2070 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry, 2071 void *context, int vl, int mode, 2072 u64 data) 2073 { 2074 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2075 2076 return dd->cce_err_status_cnt[15]; 2077 } 2078 2079 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry, 2080 void *context, int vl, 2081 int mode, u64 data) 2082 { 2083 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2084 2085 return dd->cce_err_status_cnt[14]; 2086 } 2087 2088 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry, 2089 void *context, int vl, int mode, 2090 u64 data) 2091 { 2092 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2093 2094 return dd->cce_err_status_cnt[13]; 2095 } 2096 2097 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt( 2098 const struct cntr_entry *entry, 2099 void *context, int vl, int mode, u64 data) 2100 { 2101 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2102 2103 return dd->cce_err_status_cnt[12]; 2104 } 2105 2106 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt( 2107 const struct cntr_entry *entry, 2108 void *context, int vl, int mode, u64 data) 2109 { 2110 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2111 2112 return dd->cce_err_status_cnt[11]; 2113 } 2114 2115 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt( 2116 const struct cntr_entry *entry, 2117 void *context, int vl, int mode, u64 data) 2118 { 2119 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2120 2121 return dd->cce_err_status_cnt[10]; 2122 } 2123 2124 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt( 2125 const struct cntr_entry *entry, 2126 void *context, int vl, int mode, u64 data) 2127 { 2128 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2129 2130 return dd->cce_err_status_cnt[9]; 2131 } 2132 2133 static u64 access_cce_cli2_async_fifo_parity_err_cnt( 2134 const struct cntr_entry *entry, 2135 void *context, int vl, int mode, u64 data) 2136 { 2137 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2138 2139 return dd->cce_err_status_cnt[8]; 2140 } 2141 2142 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry, 2143 void *context, int vl, 2144 int mode, u64 data) 2145 { 2146 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2147 2148 return dd->cce_err_status_cnt[7]; 2149 } 2150 2151 static u64 access_cce_cli0_async_fifo_parity_err_cnt( 2152 const struct cntr_entry *entry, 2153 void *context, int vl, int mode, u64 data) 2154 { 2155 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2156 2157 return dd->cce_err_status_cnt[6]; 2158 } 2159 2160 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry, 2161 void *context, int vl, int mode, 2162 u64 data) 2163 { 2164 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2165 2166 return dd->cce_err_status_cnt[5]; 2167 } 2168 2169 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry, 2170 void *context, int vl, int mode, 2171 u64 data) 2172 { 2173 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2174 2175 return dd->cce_err_status_cnt[4]; 2176 } 2177 2178 static u64 access_cce_trgt_async_fifo_parity_err_cnt( 2179 const struct cntr_entry *entry, 2180 void *context, int vl, int mode, u64 data) 2181 { 2182 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2183 2184 return dd->cce_err_status_cnt[3]; 2185 } 2186 2187 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2188 void *context, int vl, 2189 int mode, u64 data) 2190 { 2191 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2192 2193 return dd->cce_err_status_cnt[2]; 2194 } 2195 2196 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2197 void *context, int vl, 2198 int mode, u64 data) 2199 { 2200 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2201 2202 return dd->cce_err_status_cnt[1]; 2203 } 2204 2205 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry, 2206 void *context, int vl, int mode, 2207 u64 data) 2208 { 2209 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2210 2211 return dd->cce_err_status_cnt[0]; 2212 } 2213 2214 /* 2215 * Software counters corresponding to each of the 2216 * error status bits within RcvErrStatus 2217 */ 2218 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry, 2219 void *context, int vl, int mode, 2220 u64 data) 2221 { 2222 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2223 2224 return dd->rcv_err_status_cnt[63]; 2225 } 2226 2227 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2228 void *context, int vl, 2229 int mode, u64 data) 2230 { 2231 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2232 2233 return dd->rcv_err_status_cnt[62]; 2234 } 2235 2236 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2237 void *context, int vl, int mode, 2238 u64 data) 2239 { 2240 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2241 2242 return dd->rcv_err_status_cnt[61]; 2243 } 2244 2245 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry, 2246 void *context, int vl, int mode, 2247 u64 data) 2248 { 2249 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2250 2251 return dd->rcv_err_status_cnt[60]; 2252 } 2253 2254 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2255 void *context, int vl, 2256 int mode, u64 data) 2257 { 2258 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2259 2260 return dd->rcv_err_status_cnt[59]; 2261 } 2262 2263 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2264 void *context, int vl, 2265 int mode, u64 data) 2266 { 2267 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2268 2269 return dd->rcv_err_status_cnt[58]; 2270 } 2271 2272 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry, 2273 void *context, int vl, int mode, 2274 u64 data) 2275 { 2276 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2277 2278 return dd->rcv_err_status_cnt[57]; 2279 } 2280 2281 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry, 2282 void *context, int vl, int mode, 2283 u64 data) 2284 { 2285 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2286 2287 return dd->rcv_err_status_cnt[56]; 2288 } 2289 2290 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry, 2291 void *context, int vl, int mode, 2292 u64 data) 2293 { 2294 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2295 2296 return dd->rcv_err_status_cnt[55]; 2297 } 2298 2299 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt( 2300 const struct cntr_entry *entry, 2301 void *context, int vl, int mode, u64 data) 2302 { 2303 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2304 2305 return dd->rcv_err_status_cnt[54]; 2306 } 2307 2308 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt( 2309 const struct cntr_entry *entry, 2310 void *context, int vl, int mode, u64 data) 2311 { 2312 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2313 2314 return dd->rcv_err_status_cnt[53]; 2315 } 2316 2317 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry, 2318 void *context, int vl, 2319 int mode, u64 data) 2320 { 2321 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2322 2323 return dd->rcv_err_status_cnt[52]; 2324 } 2325 2326 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry, 2327 void *context, int vl, 2328 int mode, u64 data) 2329 { 2330 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2331 2332 return dd->rcv_err_status_cnt[51]; 2333 } 2334 2335 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry, 2336 void *context, int vl, 2337 int mode, u64 data) 2338 { 2339 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2340 2341 return dd->rcv_err_status_cnt[50]; 2342 } 2343 2344 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry, 2345 void *context, int vl, 2346 int mode, u64 data) 2347 { 2348 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2349 2350 return dd->rcv_err_status_cnt[49]; 2351 } 2352 2353 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry, 2354 void *context, int vl, 2355 int mode, u64 data) 2356 { 2357 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2358 2359 return dd->rcv_err_status_cnt[48]; 2360 } 2361 2362 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry, 2363 void *context, int vl, 2364 int mode, u64 data) 2365 { 2366 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2367 2368 return dd->rcv_err_status_cnt[47]; 2369 } 2370 2371 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry, 2372 void *context, int vl, int mode, 2373 u64 data) 2374 { 2375 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2376 2377 return dd->rcv_err_status_cnt[46]; 2378 } 2379 2380 static u64 access_rx_hq_intr_csr_parity_err_cnt( 2381 const struct cntr_entry *entry, 2382 void *context, int vl, int mode, u64 data) 2383 { 2384 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2385 2386 return dd->rcv_err_status_cnt[45]; 2387 } 2388 2389 static u64 access_rx_lookup_csr_parity_err_cnt( 2390 const struct cntr_entry *entry, 2391 void *context, int vl, int mode, u64 data) 2392 { 2393 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2394 2395 return dd->rcv_err_status_cnt[44]; 2396 } 2397 2398 static u64 access_rx_lookup_rcv_array_cor_err_cnt( 2399 const struct cntr_entry *entry, 2400 void *context, int vl, int mode, u64 data) 2401 { 2402 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2403 2404 return dd->rcv_err_status_cnt[43]; 2405 } 2406 2407 static u64 access_rx_lookup_rcv_array_unc_err_cnt( 2408 const struct cntr_entry *entry, 2409 void *context, int vl, int mode, u64 data) 2410 { 2411 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2412 2413 return dd->rcv_err_status_cnt[42]; 2414 } 2415 2416 static u64 access_rx_lookup_des_part2_parity_err_cnt( 2417 const struct cntr_entry *entry, 2418 void *context, int vl, int mode, u64 data) 2419 { 2420 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2421 2422 return dd->rcv_err_status_cnt[41]; 2423 } 2424 2425 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt( 2426 const struct cntr_entry *entry, 2427 void *context, int vl, int mode, u64 data) 2428 { 2429 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2430 2431 return dd->rcv_err_status_cnt[40]; 2432 } 2433 2434 static u64 access_rx_lookup_des_part1_unc_err_cnt( 2435 const struct cntr_entry *entry, 2436 void *context, int vl, int mode, u64 data) 2437 { 2438 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2439 2440 return dd->rcv_err_status_cnt[39]; 2441 } 2442 2443 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt( 2444 const struct cntr_entry *entry, 2445 void *context, int vl, int mode, u64 data) 2446 { 2447 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2448 2449 return dd->rcv_err_status_cnt[38]; 2450 } 2451 2452 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt( 2453 const struct cntr_entry *entry, 2454 void *context, int vl, int mode, u64 data) 2455 { 2456 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2457 2458 return dd->rcv_err_status_cnt[37]; 2459 } 2460 2461 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt( 2462 const struct cntr_entry *entry, 2463 void *context, int vl, int mode, u64 data) 2464 { 2465 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2466 2467 return dd->rcv_err_status_cnt[36]; 2468 } 2469 2470 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt( 2471 const struct cntr_entry *entry, 2472 void *context, int vl, int mode, u64 data) 2473 { 2474 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2475 2476 return dd->rcv_err_status_cnt[35]; 2477 } 2478 2479 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt( 2480 const struct cntr_entry *entry, 2481 void *context, int vl, int mode, u64 data) 2482 { 2483 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2484 2485 return dd->rcv_err_status_cnt[34]; 2486 } 2487 2488 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt( 2489 const struct cntr_entry *entry, 2490 void *context, int vl, int mode, u64 data) 2491 { 2492 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2493 2494 return dd->rcv_err_status_cnt[33]; 2495 } 2496 2497 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry, 2498 void *context, int vl, int mode, 2499 u64 data) 2500 { 2501 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2502 2503 return dd->rcv_err_status_cnt[32]; 2504 } 2505 2506 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry, 2507 void *context, int vl, int mode, 2508 u64 data) 2509 { 2510 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2511 2512 return dd->rcv_err_status_cnt[31]; 2513 } 2514 2515 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry, 2516 void *context, int vl, int mode, 2517 u64 data) 2518 { 2519 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2520 2521 return dd->rcv_err_status_cnt[30]; 2522 } 2523 2524 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry, 2525 void *context, int vl, int mode, 2526 u64 data) 2527 { 2528 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2529 2530 return dd->rcv_err_status_cnt[29]; 2531 } 2532 2533 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry, 2534 void *context, int vl, 2535 int mode, u64 data) 2536 { 2537 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2538 2539 return dd->rcv_err_status_cnt[28]; 2540 } 2541 2542 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt( 2543 const struct cntr_entry *entry, 2544 void *context, int vl, int mode, u64 data) 2545 { 2546 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2547 2548 return dd->rcv_err_status_cnt[27]; 2549 } 2550 2551 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt( 2552 const struct cntr_entry *entry, 2553 void *context, int vl, int mode, u64 data) 2554 { 2555 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2556 2557 return dd->rcv_err_status_cnt[26]; 2558 } 2559 2560 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt( 2561 const struct cntr_entry *entry, 2562 void *context, int vl, int mode, u64 data) 2563 { 2564 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2565 2566 return dd->rcv_err_status_cnt[25]; 2567 } 2568 2569 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt( 2570 const struct cntr_entry *entry, 2571 void *context, int vl, int mode, u64 data) 2572 { 2573 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2574 2575 return dd->rcv_err_status_cnt[24]; 2576 } 2577 2578 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt( 2579 const struct cntr_entry *entry, 2580 void *context, int vl, int mode, u64 data) 2581 { 2582 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2583 2584 return dd->rcv_err_status_cnt[23]; 2585 } 2586 2587 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt( 2588 const struct cntr_entry *entry, 2589 void *context, int vl, int mode, u64 data) 2590 { 2591 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2592 2593 return dd->rcv_err_status_cnt[22]; 2594 } 2595 2596 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt( 2597 const struct cntr_entry *entry, 2598 void *context, int vl, int mode, u64 data) 2599 { 2600 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2601 2602 return dd->rcv_err_status_cnt[21]; 2603 } 2604 2605 static u64 access_rx_rbuf_block_list_read_cor_err_cnt( 2606 const struct cntr_entry *entry, 2607 void *context, int vl, int mode, u64 data) 2608 { 2609 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2610 2611 return dd->rcv_err_status_cnt[20]; 2612 } 2613 2614 static u64 access_rx_rbuf_block_list_read_unc_err_cnt( 2615 const struct cntr_entry *entry, 2616 void *context, int vl, int mode, u64 data) 2617 { 2618 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2619 2620 return dd->rcv_err_status_cnt[19]; 2621 } 2622 2623 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry, 2624 void *context, int vl, 2625 int mode, u64 data) 2626 { 2627 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2628 2629 return dd->rcv_err_status_cnt[18]; 2630 } 2631 2632 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry, 2633 void *context, int vl, 2634 int mode, u64 data) 2635 { 2636 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2637 2638 return dd->rcv_err_status_cnt[17]; 2639 } 2640 2641 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt( 2642 const struct cntr_entry *entry, 2643 void *context, int vl, int mode, u64 data) 2644 { 2645 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2646 2647 return dd->rcv_err_status_cnt[16]; 2648 } 2649 2650 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt( 2651 const struct cntr_entry *entry, 2652 void *context, int vl, int mode, u64 data) 2653 { 2654 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2655 2656 return dd->rcv_err_status_cnt[15]; 2657 } 2658 2659 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry, 2660 void *context, int vl, 2661 int mode, u64 data) 2662 { 2663 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2664 2665 return dd->rcv_err_status_cnt[14]; 2666 } 2667 2668 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry, 2669 void *context, int vl, 2670 int mode, u64 data) 2671 { 2672 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2673 2674 return dd->rcv_err_status_cnt[13]; 2675 } 2676 2677 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2678 void *context, int vl, int mode, 2679 u64 data) 2680 { 2681 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2682 2683 return dd->rcv_err_status_cnt[12]; 2684 } 2685 2686 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry, 2687 void *context, int vl, int mode, 2688 u64 data) 2689 { 2690 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2691 2692 return dd->rcv_err_status_cnt[11]; 2693 } 2694 2695 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry, 2696 void *context, int vl, int mode, 2697 u64 data) 2698 { 2699 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2700 2701 return dd->rcv_err_status_cnt[10]; 2702 } 2703 2704 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry, 2705 void *context, int vl, int mode, 2706 u64 data) 2707 { 2708 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2709 2710 return dd->rcv_err_status_cnt[9]; 2711 } 2712 2713 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry, 2714 void *context, int vl, int mode, 2715 u64 data) 2716 { 2717 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2718 2719 return dd->rcv_err_status_cnt[8]; 2720 } 2721 2722 static u64 access_rx_rcv_qp_map_table_cor_err_cnt( 2723 const struct cntr_entry *entry, 2724 void *context, int vl, int mode, u64 data) 2725 { 2726 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2727 2728 return dd->rcv_err_status_cnt[7]; 2729 } 2730 2731 static u64 access_rx_rcv_qp_map_table_unc_err_cnt( 2732 const struct cntr_entry *entry, 2733 void *context, int vl, int mode, u64 data) 2734 { 2735 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2736 2737 return dd->rcv_err_status_cnt[6]; 2738 } 2739 2740 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry, 2741 void *context, int vl, int mode, 2742 u64 data) 2743 { 2744 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2745 2746 return dd->rcv_err_status_cnt[5]; 2747 } 2748 2749 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry, 2750 void *context, int vl, int mode, 2751 u64 data) 2752 { 2753 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2754 2755 return dd->rcv_err_status_cnt[4]; 2756 } 2757 2758 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry, 2759 void *context, int vl, int mode, 2760 u64 data) 2761 { 2762 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2763 2764 return dd->rcv_err_status_cnt[3]; 2765 } 2766 2767 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry, 2768 void *context, int vl, int mode, 2769 u64 data) 2770 { 2771 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2772 2773 return dd->rcv_err_status_cnt[2]; 2774 } 2775 2776 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry, 2777 void *context, int vl, int mode, 2778 u64 data) 2779 { 2780 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2781 2782 return dd->rcv_err_status_cnt[1]; 2783 } 2784 2785 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry, 2786 void *context, int vl, int mode, 2787 u64 data) 2788 { 2789 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2790 2791 return dd->rcv_err_status_cnt[0]; 2792 } 2793 2794 /* 2795 * Software counters corresponding to each of the 2796 * error status bits within SendPioErrStatus 2797 */ 2798 static u64 access_pio_pec_sop_head_parity_err_cnt( 2799 const struct cntr_entry *entry, 2800 void *context, int vl, int mode, u64 data) 2801 { 2802 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2803 2804 return dd->send_pio_err_status_cnt[35]; 2805 } 2806 2807 static u64 access_pio_pcc_sop_head_parity_err_cnt( 2808 const struct cntr_entry *entry, 2809 void *context, int vl, int mode, u64 data) 2810 { 2811 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2812 2813 return dd->send_pio_err_status_cnt[34]; 2814 } 2815 2816 static u64 access_pio_last_returned_cnt_parity_err_cnt( 2817 const struct cntr_entry *entry, 2818 void *context, int vl, int mode, u64 data) 2819 { 2820 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2821 2822 return dd->send_pio_err_status_cnt[33]; 2823 } 2824 2825 static u64 access_pio_current_free_cnt_parity_err_cnt( 2826 const struct cntr_entry *entry, 2827 void *context, int vl, int mode, u64 data) 2828 { 2829 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2830 2831 return dd->send_pio_err_status_cnt[32]; 2832 } 2833 2834 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry, 2835 void *context, int vl, int mode, 2836 u64 data) 2837 { 2838 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2839 2840 return dd->send_pio_err_status_cnt[31]; 2841 } 2842 2843 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry, 2844 void *context, int vl, int mode, 2845 u64 data) 2846 { 2847 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2848 2849 return dd->send_pio_err_status_cnt[30]; 2850 } 2851 2852 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry, 2853 void *context, int vl, int mode, 2854 u64 data) 2855 { 2856 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2857 2858 return dd->send_pio_err_status_cnt[29]; 2859 } 2860 2861 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt( 2862 const struct cntr_entry *entry, 2863 void *context, int vl, int mode, u64 data) 2864 { 2865 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2866 2867 return dd->send_pio_err_status_cnt[28]; 2868 } 2869 2870 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry, 2871 void *context, int vl, int mode, 2872 u64 data) 2873 { 2874 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2875 2876 return dd->send_pio_err_status_cnt[27]; 2877 } 2878 2879 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry, 2880 void *context, int vl, int mode, 2881 u64 data) 2882 { 2883 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2884 2885 return dd->send_pio_err_status_cnt[26]; 2886 } 2887 2888 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry, 2889 void *context, int vl, 2890 int mode, u64 data) 2891 { 2892 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2893 2894 return dd->send_pio_err_status_cnt[25]; 2895 } 2896 2897 static u64 access_pio_block_qw_count_parity_err_cnt( 2898 const struct cntr_entry *entry, 2899 void *context, int vl, int mode, u64 data) 2900 { 2901 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2902 2903 return dd->send_pio_err_status_cnt[24]; 2904 } 2905 2906 static u64 access_pio_write_qw_valid_parity_err_cnt( 2907 const struct cntr_entry *entry, 2908 void *context, int vl, int mode, u64 data) 2909 { 2910 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2911 2912 return dd->send_pio_err_status_cnt[23]; 2913 } 2914 2915 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry, 2916 void *context, int vl, int mode, 2917 u64 data) 2918 { 2919 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2920 2921 return dd->send_pio_err_status_cnt[22]; 2922 } 2923 2924 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry, 2925 void *context, int vl, 2926 int mode, u64 data) 2927 { 2928 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2929 2930 return dd->send_pio_err_status_cnt[21]; 2931 } 2932 2933 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry, 2934 void *context, int vl, 2935 int mode, u64 data) 2936 { 2937 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2938 2939 return dd->send_pio_err_status_cnt[20]; 2940 } 2941 2942 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry, 2943 void *context, int vl, 2944 int mode, u64 data) 2945 { 2946 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2947 2948 return dd->send_pio_err_status_cnt[19]; 2949 } 2950 2951 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt( 2952 const struct cntr_entry *entry, 2953 void *context, int vl, int mode, u64 data) 2954 { 2955 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2956 2957 return dd->send_pio_err_status_cnt[18]; 2958 } 2959 2960 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry, 2961 void *context, int vl, int mode, 2962 u64 data) 2963 { 2964 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2965 2966 return dd->send_pio_err_status_cnt[17]; 2967 } 2968 2969 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry, 2970 void *context, int vl, int mode, 2971 u64 data) 2972 { 2973 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2974 2975 return dd->send_pio_err_status_cnt[16]; 2976 } 2977 2978 static u64 access_pio_credit_ret_fifo_parity_err_cnt( 2979 const struct cntr_entry *entry, 2980 void *context, int vl, int mode, u64 data) 2981 { 2982 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2983 2984 return dd->send_pio_err_status_cnt[15]; 2985 } 2986 2987 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt( 2988 const struct cntr_entry *entry, 2989 void *context, int vl, int mode, u64 data) 2990 { 2991 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2992 2993 return dd->send_pio_err_status_cnt[14]; 2994 } 2995 2996 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt( 2997 const struct cntr_entry *entry, 2998 void *context, int vl, int mode, u64 data) 2999 { 3000 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3001 3002 return dd->send_pio_err_status_cnt[13]; 3003 } 3004 3005 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt( 3006 const struct cntr_entry *entry, 3007 void *context, int vl, int mode, u64 data) 3008 { 3009 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3010 3011 return dd->send_pio_err_status_cnt[12]; 3012 } 3013 3014 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt( 3015 const struct cntr_entry *entry, 3016 void *context, int vl, int mode, u64 data) 3017 { 3018 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3019 3020 return dd->send_pio_err_status_cnt[11]; 3021 } 3022 3023 static u64 access_pio_sm_pkt_reset_parity_err_cnt( 3024 const struct cntr_entry *entry, 3025 void *context, int vl, int mode, u64 data) 3026 { 3027 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3028 3029 return dd->send_pio_err_status_cnt[10]; 3030 } 3031 3032 static u64 access_pio_pkt_evict_fifo_parity_err_cnt( 3033 const struct cntr_entry *entry, 3034 void *context, int vl, int mode, u64 data) 3035 { 3036 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3037 3038 return dd->send_pio_err_status_cnt[9]; 3039 } 3040 3041 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt( 3042 const struct cntr_entry *entry, 3043 void *context, int vl, int mode, u64 data) 3044 { 3045 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3046 3047 return dd->send_pio_err_status_cnt[8]; 3048 } 3049 3050 static u64 access_pio_sbrdctl_crrel_parity_err_cnt( 3051 const struct cntr_entry *entry, 3052 void *context, int vl, int mode, u64 data) 3053 { 3054 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3055 3056 return dd->send_pio_err_status_cnt[7]; 3057 } 3058 3059 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry, 3060 void *context, int vl, int mode, 3061 u64 data) 3062 { 3063 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3064 3065 return dd->send_pio_err_status_cnt[6]; 3066 } 3067 3068 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry, 3069 void *context, int vl, int mode, 3070 u64 data) 3071 { 3072 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3073 3074 return dd->send_pio_err_status_cnt[5]; 3075 } 3076 3077 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry, 3078 void *context, int vl, int mode, 3079 u64 data) 3080 { 3081 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3082 3083 return dd->send_pio_err_status_cnt[4]; 3084 } 3085 3086 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry, 3087 void *context, int vl, int mode, 3088 u64 data) 3089 { 3090 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3091 3092 return dd->send_pio_err_status_cnt[3]; 3093 } 3094 3095 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry, 3096 void *context, int vl, int mode, 3097 u64 data) 3098 { 3099 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3100 3101 return dd->send_pio_err_status_cnt[2]; 3102 } 3103 3104 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry, 3105 void *context, int vl, 3106 int mode, u64 data) 3107 { 3108 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3109 3110 return dd->send_pio_err_status_cnt[1]; 3111 } 3112 3113 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry, 3114 void *context, int vl, int mode, 3115 u64 data) 3116 { 3117 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3118 3119 return dd->send_pio_err_status_cnt[0]; 3120 } 3121 3122 /* 3123 * Software counters corresponding to each of the 3124 * error status bits within SendDmaErrStatus 3125 */ 3126 static u64 access_sdma_pcie_req_tracking_cor_err_cnt( 3127 const struct cntr_entry *entry, 3128 void *context, int vl, int mode, u64 data) 3129 { 3130 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3131 3132 return dd->send_dma_err_status_cnt[3]; 3133 } 3134 3135 static u64 access_sdma_pcie_req_tracking_unc_err_cnt( 3136 const struct cntr_entry *entry, 3137 void *context, int vl, int mode, u64 data) 3138 { 3139 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3140 3141 return dd->send_dma_err_status_cnt[2]; 3142 } 3143 3144 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry, 3145 void *context, int vl, int mode, 3146 u64 data) 3147 { 3148 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3149 3150 return dd->send_dma_err_status_cnt[1]; 3151 } 3152 3153 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry, 3154 void *context, int vl, int mode, 3155 u64 data) 3156 { 3157 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3158 3159 return dd->send_dma_err_status_cnt[0]; 3160 } 3161 3162 /* 3163 * Software counters corresponding to each of the 3164 * error status bits within SendEgressErrStatus 3165 */ 3166 static u64 access_tx_read_pio_memory_csr_unc_err_cnt( 3167 const struct cntr_entry *entry, 3168 void *context, int vl, int mode, u64 data) 3169 { 3170 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3171 3172 return dd->send_egress_err_status_cnt[63]; 3173 } 3174 3175 static u64 access_tx_read_sdma_memory_csr_err_cnt( 3176 const struct cntr_entry *entry, 3177 void *context, int vl, int mode, u64 data) 3178 { 3179 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3180 3181 return dd->send_egress_err_status_cnt[62]; 3182 } 3183 3184 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry, 3185 void *context, int vl, int mode, 3186 u64 data) 3187 { 3188 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3189 3190 return dd->send_egress_err_status_cnt[61]; 3191 } 3192 3193 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry, 3194 void *context, int vl, 3195 int mode, u64 data) 3196 { 3197 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3198 3199 return dd->send_egress_err_status_cnt[60]; 3200 } 3201 3202 static u64 access_tx_read_sdma_memory_cor_err_cnt( 3203 const struct cntr_entry *entry, 3204 void *context, int vl, int mode, u64 data) 3205 { 3206 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3207 3208 return dd->send_egress_err_status_cnt[59]; 3209 } 3210 3211 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry, 3212 void *context, int vl, int mode, 3213 u64 data) 3214 { 3215 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3216 3217 return dd->send_egress_err_status_cnt[58]; 3218 } 3219 3220 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry, 3221 void *context, int vl, int mode, 3222 u64 data) 3223 { 3224 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3225 3226 return dd->send_egress_err_status_cnt[57]; 3227 } 3228 3229 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry, 3230 void *context, int vl, int mode, 3231 u64 data) 3232 { 3233 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3234 3235 return dd->send_egress_err_status_cnt[56]; 3236 } 3237 3238 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry, 3239 void *context, int vl, int mode, 3240 u64 data) 3241 { 3242 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3243 3244 return dd->send_egress_err_status_cnt[55]; 3245 } 3246 3247 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry, 3248 void *context, int vl, int mode, 3249 u64 data) 3250 { 3251 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3252 3253 return dd->send_egress_err_status_cnt[54]; 3254 } 3255 3256 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry, 3257 void *context, int vl, int mode, 3258 u64 data) 3259 { 3260 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3261 3262 return dd->send_egress_err_status_cnt[53]; 3263 } 3264 3265 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry, 3266 void *context, int vl, int mode, 3267 u64 data) 3268 { 3269 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3270 3271 return dd->send_egress_err_status_cnt[52]; 3272 } 3273 3274 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry, 3275 void *context, int vl, int mode, 3276 u64 data) 3277 { 3278 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3279 3280 return dd->send_egress_err_status_cnt[51]; 3281 } 3282 3283 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry, 3284 void *context, int vl, int mode, 3285 u64 data) 3286 { 3287 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3288 3289 return dd->send_egress_err_status_cnt[50]; 3290 } 3291 3292 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry, 3293 void *context, int vl, int mode, 3294 u64 data) 3295 { 3296 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3297 3298 return dd->send_egress_err_status_cnt[49]; 3299 } 3300 3301 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry, 3302 void *context, int vl, int mode, 3303 u64 data) 3304 { 3305 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3306 3307 return dd->send_egress_err_status_cnt[48]; 3308 } 3309 3310 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry, 3311 void *context, int vl, int mode, 3312 u64 data) 3313 { 3314 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3315 3316 return dd->send_egress_err_status_cnt[47]; 3317 } 3318 3319 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry, 3320 void *context, int vl, int mode, 3321 u64 data) 3322 { 3323 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3324 3325 return dd->send_egress_err_status_cnt[46]; 3326 } 3327 3328 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry, 3329 void *context, int vl, int mode, 3330 u64 data) 3331 { 3332 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3333 3334 return dd->send_egress_err_status_cnt[45]; 3335 } 3336 3337 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry, 3338 void *context, int vl, 3339 int mode, u64 data) 3340 { 3341 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3342 3343 return dd->send_egress_err_status_cnt[44]; 3344 } 3345 3346 static u64 access_tx_read_sdma_memory_unc_err_cnt( 3347 const struct cntr_entry *entry, 3348 void *context, int vl, int mode, u64 data) 3349 { 3350 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3351 3352 return dd->send_egress_err_status_cnt[43]; 3353 } 3354 3355 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry, 3356 void *context, int vl, int mode, 3357 u64 data) 3358 { 3359 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3360 3361 return dd->send_egress_err_status_cnt[42]; 3362 } 3363 3364 static u64 access_tx_credit_return_partiy_err_cnt( 3365 const struct cntr_entry *entry, 3366 void *context, int vl, int mode, u64 data) 3367 { 3368 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3369 3370 return dd->send_egress_err_status_cnt[41]; 3371 } 3372 3373 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt( 3374 const struct cntr_entry *entry, 3375 void *context, int vl, int mode, u64 data) 3376 { 3377 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3378 3379 return dd->send_egress_err_status_cnt[40]; 3380 } 3381 3382 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt( 3383 const struct cntr_entry *entry, 3384 void *context, int vl, int mode, u64 data) 3385 { 3386 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3387 3388 return dd->send_egress_err_status_cnt[39]; 3389 } 3390 3391 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt( 3392 const struct cntr_entry *entry, 3393 void *context, int vl, int mode, u64 data) 3394 { 3395 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3396 3397 return dd->send_egress_err_status_cnt[38]; 3398 } 3399 3400 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt( 3401 const struct cntr_entry *entry, 3402 void *context, int vl, int mode, u64 data) 3403 { 3404 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3405 3406 return dd->send_egress_err_status_cnt[37]; 3407 } 3408 3409 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt( 3410 const struct cntr_entry *entry, 3411 void *context, int vl, int mode, u64 data) 3412 { 3413 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3414 3415 return dd->send_egress_err_status_cnt[36]; 3416 } 3417 3418 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt( 3419 const struct cntr_entry *entry, 3420 void *context, int vl, int mode, u64 data) 3421 { 3422 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3423 3424 return dd->send_egress_err_status_cnt[35]; 3425 } 3426 3427 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt( 3428 const struct cntr_entry *entry, 3429 void *context, int vl, int mode, u64 data) 3430 { 3431 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3432 3433 return dd->send_egress_err_status_cnt[34]; 3434 } 3435 3436 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt( 3437 const struct cntr_entry *entry, 3438 void *context, int vl, int mode, u64 data) 3439 { 3440 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3441 3442 return dd->send_egress_err_status_cnt[33]; 3443 } 3444 3445 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt( 3446 const struct cntr_entry *entry, 3447 void *context, int vl, int mode, u64 data) 3448 { 3449 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3450 3451 return dd->send_egress_err_status_cnt[32]; 3452 } 3453 3454 static u64 access_tx_sdma15_disallowed_packet_err_cnt( 3455 const struct cntr_entry *entry, 3456 void *context, int vl, int mode, u64 data) 3457 { 3458 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3459 3460 return dd->send_egress_err_status_cnt[31]; 3461 } 3462 3463 static u64 access_tx_sdma14_disallowed_packet_err_cnt( 3464 const struct cntr_entry *entry, 3465 void *context, int vl, int mode, u64 data) 3466 { 3467 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3468 3469 return dd->send_egress_err_status_cnt[30]; 3470 } 3471 3472 static u64 access_tx_sdma13_disallowed_packet_err_cnt( 3473 const struct cntr_entry *entry, 3474 void *context, int vl, int mode, u64 data) 3475 { 3476 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3477 3478 return dd->send_egress_err_status_cnt[29]; 3479 } 3480 3481 static u64 access_tx_sdma12_disallowed_packet_err_cnt( 3482 const struct cntr_entry *entry, 3483 void *context, int vl, int mode, u64 data) 3484 { 3485 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3486 3487 return dd->send_egress_err_status_cnt[28]; 3488 } 3489 3490 static u64 access_tx_sdma11_disallowed_packet_err_cnt( 3491 const struct cntr_entry *entry, 3492 void *context, int vl, int mode, u64 data) 3493 { 3494 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3495 3496 return dd->send_egress_err_status_cnt[27]; 3497 } 3498 3499 static u64 access_tx_sdma10_disallowed_packet_err_cnt( 3500 const struct cntr_entry *entry, 3501 void *context, int vl, int mode, u64 data) 3502 { 3503 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3504 3505 return dd->send_egress_err_status_cnt[26]; 3506 } 3507 3508 static u64 access_tx_sdma9_disallowed_packet_err_cnt( 3509 const struct cntr_entry *entry, 3510 void *context, int vl, int mode, u64 data) 3511 { 3512 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3513 3514 return dd->send_egress_err_status_cnt[25]; 3515 } 3516 3517 static u64 access_tx_sdma8_disallowed_packet_err_cnt( 3518 const struct cntr_entry *entry, 3519 void *context, int vl, int mode, u64 data) 3520 { 3521 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3522 3523 return dd->send_egress_err_status_cnt[24]; 3524 } 3525 3526 static u64 access_tx_sdma7_disallowed_packet_err_cnt( 3527 const struct cntr_entry *entry, 3528 void *context, int vl, int mode, u64 data) 3529 { 3530 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3531 3532 return dd->send_egress_err_status_cnt[23]; 3533 } 3534 3535 static u64 access_tx_sdma6_disallowed_packet_err_cnt( 3536 const struct cntr_entry *entry, 3537 void *context, int vl, int mode, u64 data) 3538 { 3539 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3540 3541 return dd->send_egress_err_status_cnt[22]; 3542 } 3543 3544 static u64 access_tx_sdma5_disallowed_packet_err_cnt( 3545 const struct cntr_entry *entry, 3546 void *context, int vl, int mode, u64 data) 3547 { 3548 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3549 3550 return dd->send_egress_err_status_cnt[21]; 3551 } 3552 3553 static u64 access_tx_sdma4_disallowed_packet_err_cnt( 3554 const struct cntr_entry *entry, 3555 void *context, int vl, int mode, u64 data) 3556 { 3557 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3558 3559 return dd->send_egress_err_status_cnt[20]; 3560 } 3561 3562 static u64 access_tx_sdma3_disallowed_packet_err_cnt( 3563 const struct cntr_entry *entry, 3564 void *context, int vl, int mode, u64 data) 3565 { 3566 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3567 3568 return dd->send_egress_err_status_cnt[19]; 3569 } 3570 3571 static u64 access_tx_sdma2_disallowed_packet_err_cnt( 3572 const struct cntr_entry *entry, 3573 void *context, int vl, int mode, u64 data) 3574 { 3575 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3576 3577 return dd->send_egress_err_status_cnt[18]; 3578 } 3579 3580 static u64 access_tx_sdma1_disallowed_packet_err_cnt( 3581 const struct cntr_entry *entry, 3582 void *context, int vl, int mode, u64 data) 3583 { 3584 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3585 3586 return dd->send_egress_err_status_cnt[17]; 3587 } 3588 3589 static u64 access_tx_sdma0_disallowed_packet_err_cnt( 3590 const struct cntr_entry *entry, 3591 void *context, int vl, int mode, u64 data) 3592 { 3593 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3594 3595 return dd->send_egress_err_status_cnt[16]; 3596 } 3597 3598 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry, 3599 void *context, int vl, int mode, 3600 u64 data) 3601 { 3602 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3603 3604 return dd->send_egress_err_status_cnt[15]; 3605 } 3606 3607 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry, 3608 void *context, int vl, 3609 int mode, u64 data) 3610 { 3611 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3612 3613 return dd->send_egress_err_status_cnt[14]; 3614 } 3615 3616 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry, 3617 void *context, int vl, int mode, 3618 u64 data) 3619 { 3620 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3621 3622 return dd->send_egress_err_status_cnt[13]; 3623 } 3624 3625 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry, 3626 void *context, int vl, int mode, 3627 u64 data) 3628 { 3629 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3630 3631 return dd->send_egress_err_status_cnt[12]; 3632 } 3633 3634 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt( 3635 const struct cntr_entry *entry, 3636 void *context, int vl, int mode, u64 data) 3637 { 3638 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3639 3640 return dd->send_egress_err_status_cnt[11]; 3641 } 3642 3643 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry, 3644 void *context, int vl, int mode, 3645 u64 data) 3646 { 3647 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3648 3649 return dd->send_egress_err_status_cnt[10]; 3650 } 3651 3652 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry, 3653 void *context, int vl, int mode, 3654 u64 data) 3655 { 3656 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3657 3658 return dd->send_egress_err_status_cnt[9]; 3659 } 3660 3661 static u64 access_tx_sdma_launch_intf_parity_err_cnt( 3662 const struct cntr_entry *entry, 3663 void *context, int vl, int mode, u64 data) 3664 { 3665 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3666 3667 return dd->send_egress_err_status_cnt[8]; 3668 } 3669 3670 static u64 access_tx_pio_launch_intf_parity_err_cnt( 3671 const struct cntr_entry *entry, 3672 void *context, int vl, int mode, u64 data) 3673 { 3674 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3675 3676 return dd->send_egress_err_status_cnt[7]; 3677 } 3678 3679 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry, 3680 void *context, int vl, int mode, 3681 u64 data) 3682 { 3683 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3684 3685 return dd->send_egress_err_status_cnt[6]; 3686 } 3687 3688 static u64 access_tx_incorrect_link_state_err_cnt( 3689 const struct cntr_entry *entry, 3690 void *context, int vl, int mode, u64 data) 3691 { 3692 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3693 3694 return dd->send_egress_err_status_cnt[5]; 3695 } 3696 3697 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry, 3698 void *context, int vl, int mode, 3699 u64 data) 3700 { 3701 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3702 3703 return dd->send_egress_err_status_cnt[4]; 3704 } 3705 3706 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt( 3707 const struct cntr_entry *entry, 3708 void *context, int vl, int mode, u64 data) 3709 { 3710 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3711 3712 return dd->send_egress_err_status_cnt[3]; 3713 } 3714 3715 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry, 3716 void *context, int vl, int mode, 3717 u64 data) 3718 { 3719 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3720 3721 return dd->send_egress_err_status_cnt[2]; 3722 } 3723 3724 static u64 access_tx_pkt_integrity_mem_unc_err_cnt( 3725 const struct cntr_entry *entry, 3726 void *context, int vl, int mode, u64 data) 3727 { 3728 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3729 3730 return dd->send_egress_err_status_cnt[1]; 3731 } 3732 3733 static u64 access_tx_pkt_integrity_mem_cor_err_cnt( 3734 const struct cntr_entry *entry, 3735 void *context, int vl, int mode, u64 data) 3736 { 3737 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3738 3739 return dd->send_egress_err_status_cnt[0]; 3740 } 3741 3742 /* 3743 * Software counters corresponding to each of the 3744 * error status bits within SendErrStatus 3745 */ 3746 static u64 access_send_csr_write_bad_addr_err_cnt( 3747 const struct cntr_entry *entry, 3748 void *context, int vl, int mode, u64 data) 3749 { 3750 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3751 3752 return dd->send_err_status_cnt[2]; 3753 } 3754 3755 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 3756 void *context, int vl, 3757 int mode, u64 data) 3758 { 3759 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3760 3761 return dd->send_err_status_cnt[1]; 3762 } 3763 3764 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry, 3765 void *context, int vl, int mode, 3766 u64 data) 3767 { 3768 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3769 3770 return dd->send_err_status_cnt[0]; 3771 } 3772 3773 /* 3774 * Software counters corresponding to each of the 3775 * error status bits within SendCtxtErrStatus 3776 */ 3777 static u64 access_pio_write_out_of_bounds_err_cnt( 3778 const struct cntr_entry *entry, 3779 void *context, int vl, int mode, u64 data) 3780 { 3781 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3782 3783 return dd->sw_ctxt_err_status_cnt[4]; 3784 } 3785 3786 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry, 3787 void *context, int vl, int mode, 3788 u64 data) 3789 { 3790 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3791 3792 return dd->sw_ctxt_err_status_cnt[3]; 3793 } 3794 3795 static u64 access_pio_write_crosses_boundary_err_cnt( 3796 const struct cntr_entry *entry, 3797 void *context, int vl, int mode, u64 data) 3798 { 3799 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3800 3801 return dd->sw_ctxt_err_status_cnt[2]; 3802 } 3803 3804 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry, 3805 void *context, int vl, 3806 int mode, u64 data) 3807 { 3808 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3809 3810 return dd->sw_ctxt_err_status_cnt[1]; 3811 } 3812 3813 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry, 3814 void *context, int vl, int mode, 3815 u64 data) 3816 { 3817 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3818 3819 return dd->sw_ctxt_err_status_cnt[0]; 3820 } 3821 3822 /* 3823 * Software counters corresponding to each of the 3824 * error status bits within SendDmaEngErrStatus 3825 */ 3826 static u64 access_sdma_header_request_fifo_cor_err_cnt( 3827 const struct cntr_entry *entry, 3828 void *context, int vl, int mode, u64 data) 3829 { 3830 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3831 3832 return dd->sw_send_dma_eng_err_status_cnt[23]; 3833 } 3834 3835 static u64 access_sdma_header_storage_cor_err_cnt( 3836 const struct cntr_entry *entry, 3837 void *context, int vl, int mode, u64 data) 3838 { 3839 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3840 3841 return dd->sw_send_dma_eng_err_status_cnt[22]; 3842 } 3843 3844 static u64 access_sdma_packet_tracking_cor_err_cnt( 3845 const struct cntr_entry *entry, 3846 void *context, int vl, int mode, u64 data) 3847 { 3848 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3849 3850 return dd->sw_send_dma_eng_err_status_cnt[21]; 3851 } 3852 3853 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry, 3854 void *context, int vl, int mode, 3855 u64 data) 3856 { 3857 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3858 3859 return dd->sw_send_dma_eng_err_status_cnt[20]; 3860 } 3861 3862 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry, 3863 void *context, int vl, int mode, 3864 u64 data) 3865 { 3866 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3867 3868 return dd->sw_send_dma_eng_err_status_cnt[19]; 3869 } 3870 3871 static u64 access_sdma_header_request_fifo_unc_err_cnt( 3872 const struct cntr_entry *entry, 3873 void *context, int vl, int mode, u64 data) 3874 { 3875 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3876 3877 return dd->sw_send_dma_eng_err_status_cnt[18]; 3878 } 3879 3880 static u64 access_sdma_header_storage_unc_err_cnt( 3881 const struct cntr_entry *entry, 3882 void *context, int vl, int mode, u64 data) 3883 { 3884 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3885 3886 return dd->sw_send_dma_eng_err_status_cnt[17]; 3887 } 3888 3889 static u64 access_sdma_packet_tracking_unc_err_cnt( 3890 const struct cntr_entry *entry, 3891 void *context, int vl, int mode, u64 data) 3892 { 3893 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3894 3895 return dd->sw_send_dma_eng_err_status_cnt[16]; 3896 } 3897 3898 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry, 3899 void *context, int vl, int mode, 3900 u64 data) 3901 { 3902 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3903 3904 return dd->sw_send_dma_eng_err_status_cnt[15]; 3905 } 3906 3907 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry, 3908 void *context, int vl, int mode, 3909 u64 data) 3910 { 3911 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3912 3913 return dd->sw_send_dma_eng_err_status_cnt[14]; 3914 } 3915 3916 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry, 3917 void *context, int vl, int mode, 3918 u64 data) 3919 { 3920 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3921 3922 return dd->sw_send_dma_eng_err_status_cnt[13]; 3923 } 3924 3925 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry, 3926 void *context, int vl, int mode, 3927 u64 data) 3928 { 3929 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3930 3931 return dd->sw_send_dma_eng_err_status_cnt[12]; 3932 } 3933 3934 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry, 3935 void *context, int vl, int mode, 3936 u64 data) 3937 { 3938 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3939 3940 return dd->sw_send_dma_eng_err_status_cnt[11]; 3941 } 3942 3943 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry, 3944 void *context, int vl, int mode, 3945 u64 data) 3946 { 3947 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3948 3949 return dd->sw_send_dma_eng_err_status_cnt[10]; 3950 } 3951 3952 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry, 3953 void *context, int vl, int mode, 3954 u64 data) 3955 { 3956 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3957 3958 return dd->sw_send_dma_eng_err_status_cnt[9]; 3959 } 3960 3961 static u64 access_sdma_packet_desc_overflow_err_cnt( 3962 const struct cntr_entry *entry, 3963 void *context, int vl, int mode, u64 data) 3964 { 3965 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3966 3967 return dd->sw_send_dma_eng_err_status_cnt[8]; 3968 } 3969 3970 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry, 3971 void *context, int vl, 3972 int mode, u64 data) 3973 { 3974 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3975 3976 return dd->sw_send_dma_eng_err_status_cnt[7]; 3977 } 3978 3979 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry, 3980 void *context, int vl, int mode, u64 data) 3981 { 3982 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3983 3984 return dd->sw_send_dma_eng_err_status_cnt[6]; 3985 } 3986 3987 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry, 3988 void *context, int vl, int mode, 3989 u64 data) 3990 { 3991 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3992 3993 return dd->sw_send_dma_eng_err_status_cnt[5]; 3994 } 3995 3996 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry, 3997 void *context, int vl, int mode, 3998 u64 data) 3999 { 4000 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4001 4002 return dd->sw_send_dma_eng_err_status_cnt[4]; 4003 } 4004 4005 static u64 access_sdma_tail_out_of_bounds_err_cnt( 4006 const struct cntr_entry *entry, 4007 void *context, int vl, int mode, u64 data) 4008 { 4009 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4010 4011 return dd->sw_send_dma_eng_err_status_cnt[3]; 4012 } 4013 4014 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry, 4015 void *context, int vl, int mode, 4016 u64 data) 4017 { 4018 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4019 4020 return dd->sw_send_dma_eng_err_status_cnt[2]; 4021 } 4022 4023 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry, 4024 void *context, int vl, int mode, 4025 u64 data) 4026 { 4027 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4028 4029 return dd->sw_send_dma_eng_err_status_cnt[1]; 4030 } 4031 4032 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry, 4033 void *context, int vl, int mode, 4034 u64 data) 4035 { 4036 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4037 4038 return dd->sw_send_dma_eng_err_status_cnt[0]; 4039 } 4040 4041 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry, 4042 void *context, int vl, int mode, 4043 u64 data) 4044 { 4045 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4046 4047 u64 val = 0; 4048 u64 csr = entry->csr; 4049 4050 val = read_write_csr(dd, csr, mode, data); 4051 if (mode == CNTR_MODE_R) { 4052 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ? 4053 CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors; 4054 } else if (mode == CNTR_MODE_W) { 4055 dd->sw_rcv_bypass_packet_errors = 0; 4056 } else { 4057 dd_dev_err(dd, "Invalid cntr register access mode"); 4058 return 0; 4059 } 4060 return val; 4061 } 4062 4063 #define def_access_sw_cpu(cntr) \ 4064 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \ 4065 void *context, int vl, int mode, u64 data) \ 4066 { \ 4067 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 4068 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \ 4069 ppd->ibport_data.rvp.cntr, vl, \ 4070 mode, data); \ 4071 } 4072 4073 def_access_sw_cpu(rc_acks); 4074 def_access_sw_cpu(rc_qacks); 4075 def_access_sw_cpu(rc_delayed_comp); 4076 4077 #define def_access_ibp_counter(cntr) \ 4078 static u64 access_ibp_##cntr(const struct cntr_entry *entry, \ 4079 void *context, int vl, int mode, u64 data) \ 4080 { \ 4081 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 4082 \ 4083 if (vl != CNTR_INVALID_VL) \ 4084 return 0; \ 4085 \ 4086 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \ 4087 mode, data); \ 4088 } 4089 4090 def_access_ibp_counter(loop_pkts); 4091 def_access_ibp_counter(rc_resends); 4092 def_access_ibp_counter(rnr_naks); 4093 def_access_ibp_counter(other_naks); 4094 def_access_ibp_counter(rc_timeouts); 4095 def_access_ibp_counter(pkt_drops); 4096 def_access_ibp_counter(dmawait); 4097 def_access_ibp_counter(rc_seqnak); 4098 def_access_ibp_counter(rc_dupreq); 4099 def_access_ibp_counter(rdma_seq); 4100 def_access_ibp_counter(unaligned); 4101 def_access_ibp_counter(seq_naks); 4102 4103 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { 4104 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH), 4105 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT, 4106 CNTR_NORMAL), 4107 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT, 4108 CNTR_NORMAL), 4109 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs, 4110 RCV_TID_FLOW_GEN_MISMATCH_CNT, 4111 CNTR_NORMAL), 4112 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL, 4113 CNTR_NORMAL), 4114 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs, 4115 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL), 4116 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt, 4117 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL), 4118 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT, 4119 CNTR_NORMAL), 4120 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT, 4121 CNTR_NORMAL), 4122 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT, 4123 CNTR_NORMAL), 4124 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT, 4125 CNTR_NORMAL), 4126 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT, 4127 CNTR_NORMAL), 4128 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT, 4129 CNTR_NORMAL), 4130 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt, 4131 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL), 4132 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt, 4133 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL), 4134 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT, 4135 CNTR_SYNTH), 4136 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH, 4137 access_dc_rcv_err_cnt), 4138 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT, 4139 CNTR_SYNTH), 4140 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT, 4141 CNTR_SYNTH), 4142 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT, 4143 CNTR_SYNTH), 4144 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts, 4145 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH), 4146 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts, 4147 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT, 4148 CNTR_SYNTH), 4149 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr, 4150 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH), 4151 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT, 4152 CNTR_SYNTH), 4153 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT, 4154 CNTR_SYNTH), 4155 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT, 4156 CNTR_SYNTH), 4157 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT, 4158 CNTR_SYNTH), 4159 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT, 4160 CNTR_SYNTH), 4161 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT, 4162 CNTR_SYNTH), 4163 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT, 4164 CNTR_SYNTH), 4165 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT, 4166 CNTR_SYNTH | CNTR_VL), 4167 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT, 4168 CNTR_SYNTH | CNTR_VL), 4169 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH), 4170 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT, 4171 CNTR_SYNTH | CNTR_VL), 4172 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH), 4173 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT, 4174 CNTR_SYNTH | CNTR_VL), 4175 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT, 4176 CNTR_SYNTH), 4177 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT, 4178 CNTR_SYNTH | CNTR_VL), 4179 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT, 4180 CNTR_SYNTH), 4181 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT, 4182 CNTR_SYNTH | CNTR_VL), 4183 [C_DC_TOTAL_CRC] = 4184 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR, 4185 CNTR_SYNTH), 4186 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0, 4187 CNTR_SYNTH), 4188 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1, 4189 CNTR_SYNTH), 4190 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2, 4191 CNTR_SYNTH), 4192 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3, 4193 CNTR_SYNTH), 4194 [C_DC_CRC_MULT_LN] = 4195 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN, 4196 CNTR_SYNTH), 4197 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT, 4198 CNTR_SYNTH), 4199 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT, 4200 CNTR_SYNTH), 4201 [C_DC_SEQ_CRC_CNT] = 4202 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT, 4203 CNTR_SYNTH), 4204 [C_DC_ESC0_ONLY_CNT] = 4205 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT, 4206 CNTR_SYNTH), 4207 [C_DC_ESC0_PLUS1_CNT] = 4208 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT, 4209 CNTR_SYNTH), 4210 [C_DC_ESC0_PLUS2_CNT] = 4211 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT, 4212 CNTR_SYNTH), 4213 [C_DC_REINIT_FROM_PEER_CNT] = 4214 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 4215 CNTR_SYNTH), 4216 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT, 4217 CNTR_SYNTH), 4218 [C_DC_MISC_FLG_CNT] = 4219 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT, 4220 CNTR_SYNTH), 4221 [C_DC_PRF_GOOD_LTP_CNT] = 4222 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH), 4223 [C_DC_PRF_ACCEPTED_LTP_CNT] = 4224 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT, 4225 CNTR_SYNTH), 4226 [C_DC_PRF_RX_FLIT_CNT] = 4227 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH), 4228 [C_DC_PRF_TX_FLIT_CNT] = 4229 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH), 4230 [C_DC_PRF_CLK_CNTR] = 4231 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH), 4232 [C_DC_PG_DBG_FLIT_CRDTS_CNT] = 4233 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH), 4234 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] = 4235 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT, 4236 CNTR_SYNTH), 4237 [C_DC_PG_STS_TX_SBE_CNT] = 4238 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH), 4239 [C_DC_PG_STS_TX_MBE_CNT] = 4240 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT, 4241 CNTR_SYNTH), 4242 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL, 4243 access_sw_cpu_intr), 4244 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL, 4245 access_sw_cpu_rcv_limit), 4246 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL, 4247 access_sw_vtx_wait), 4248 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL, 4249 access_sw_pio_wait), 4250 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL, 4251 access_sw_pio_drain), 4252 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL, 4253 access_sw_kmem_wait), 4254 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL, 4255 access_sw_send_schedule), 4256 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn", 4257 SEND_DMA_DESC_FETCHED_CNT, 0, 4258 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4259 dev_access_u32_csr), 4260 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0, 4261 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4262 access_sde_int_cnt), 4263 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0, 4264 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4265 access_sde_err_cnt), 4266 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0, 4267 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4268 access_sde_idle_int_cnt), 4269 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0, 4270 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4271 access_sde_progress_int_cnt), 4272 /* MISC_ERR_STATUS */ 4273 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0, 4274 CNTR_NORMAL, 4275 access_misc_pll_lock_fail_err_cnt), 4276 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0, 4277 CNTR_NORMAL, 4278 access_misc_mbist_fail_err_cnt), 4279 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0, 4280 CNTR_NORMAL, 4281 access_misc_invalid_eep_cmd_err_cnt), 4282 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0, 4283 CNTR_NORMAL, 4284 access_misc_efuse_done_parity_err_cnt), 4285 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0, 4286 CNTR_NORMAL, 4287 access_misc_efuse_write_err_cnt), 4288 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0, 4289 0, CNTR_NORMAL, 4290 access_misc_efuse_read_bad_addr_err_cnt), 4291 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0, 4292 CNTR_NORMAL, 4293 access_misc_efuse_csr_parity_err_cnt), 4294 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0, 4295 CNTR_NORMAL, 4296 access_misc_fw_auth_failed_err_cnt), 4297 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0, 4298 CNTR_NORMAL, 4299 access_misc_key_mismatch_err_cnt), 4300 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0, 4301 CNTR_NORMAL, 4302 access_misc_sbus_write_failed_err_cnt), 4303 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0, 4304 CNTR_NORMAL, 4305 access_misc_csr_write_bad_addr_err_cnt), 4306 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0, 4307 CNTR_NORMAL, 4308 access_misc_csr_read_bad_addr_err_cnt), 4309 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0, 4310 CNTR_NORMAL, 4311 access_misc_csr_parity_err_cnt), 4312 /* CceErrStatus */ 4313 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0, 4314 CNTR_NORMAL, 4315 access_sw_cce_err_status_aggregated_cnt), 4316 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0, 4317 CNTR_NORMAL, 4318 access_cce_msix_csr_parity_err_cnt), 4319 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0, 4320 CNTR_NORMAL, 4321 access_cce_int_map_unc_err_cnt), 4322 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0, 4323 CNTR_NORMAL, 4324 access_cce_int_map_cor_err_cnt), 4325 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0, 4326 CNTR_NORMAL, 4327 access_cce_msix_table_unc_err_cnt), 4328 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0, 4329 CNTR_NORMAL, 4330 access_cce_msix_table_cor_err_cnt), 4331 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0, 4332 0, CNTR_NORMAL, 4333 access_cce_rxdma_conv_fifo_parity_err_cnt), 4334 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0, 4335 0, CNTR_NORMAL, 4336 access_cce_rcpl_async_fifo_parity_err_cnt), 4337 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0, 4338 CNTR_NORMAL, 4339 access_cce_seg_write_bad_addr_err_cnt), 4340 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0, 4341 CNTR_NORMAL, 4342 access_cce_seg_read_bad_addr_err_cnt), 4343 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0, 4344 CNTR_NORMAL, 4345 access_la_triggered_cnt), 4346 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0, 4347 CNTR_NORMAL, 4348 access_cce_trgt_cpl_timeout_err_cnt), 4349 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0, 4350 CNTR_NORMAL, 4351 access_pcic_receive_parity_err_cnt), 4352 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0, 4353 CNTR_NORMAL, 4354 access_pcic_transmit_back_parity_err_cnt), 4355 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0, 4356 0, CNTR_NORMAL, 4357 access_pcic_transmit_front_parity_err_cnt), 4358 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0, 4359 CNTR_NORMAL, 4360 access_pcic_cpl_dat_q_unc_err_cnt), 4361 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0, 4362 CNTR_NORMAL, 4363 access_pcic_cpl_hd_q_unc_err_cnt), 4364 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0, 4365 CNTR_NORMAL, 4366 access_pcic_post_dat_q_unc_err_cnt), 4367 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0, 4368 CNTR_NORMAL, 4369 access_pcic_post_hd_q_unc_err_cnt), 4370 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0, 4371 CNTR_NORMAL, 4372 access_pcic_retry_sot_mem_unc_err_cnt), 4373 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0, 4374 CNTR_NORMAL, 4375 access_pcic_retry_mem_unc_err), 4376 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0, 4377 CNTR_NORMAL, 4378 access_pcic_n_post_dat_q_parity_err_cnt), 4379 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0, 4380 CNTR_NORMAL, 4381 access_pcic_n_post_h_q_parity_err_cnt), 4382 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0, 4383 CNTR_NORMAL, 4384 access_pcic_cpl_dat_q_cor_err_cnt), 4385 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0, 4386 CNTR_NORMAL, 4387 access_pcic_cpl_hd_q_cor_err_cnt), 4388 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0, 4389 CNTR_NORMAL, 4390 access_pcic_post_dat_q_cor_err_cnt), 4391 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0, 4392 CNTR_NORMAL, 4393 access_pcic_post_hd_q_cor_err_cnt), 4394 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0, 4395 CNTR_NORMAL, 4396 access_pcic_retry_sot_mem_cor_err_cnt), 4397 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0, 4398 CNTR_NORMAL, 4399 access_pcic_retry_mem_cor_err_cnt), 4400 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM( 4401 "CceCli1AsyncFifoDbgParityError", 0, 0, 4402 CNTR_NORMAL, 4403 access_cce_cli1_async_fifo_dbg_parity_err_cnt), 4404 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM( 4405 "CceCli1AsyncFifoRxdmaParityError", 0, 0, 4406 CNTR_NORMAL, 4407 access_cce_cli1_async_fifo_rxdma_parity_err_cnt 4408 ), 4409 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM( 4410 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0, 4411 CNTR_NORMAL, 4412 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt), 4413 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM( 4414 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0, 4415 CNTR_NORMAL, 4416 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt), 4417 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0, 4418 0, CNTR_NORMAL, 4419 access_cce_cli2_async_fifo_parity_err_cnt), 4420 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0, 4421 CNTR_NORMAL, 4422 access_cce_csr_cfg_bus_parity_err_cnt), 4423 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0, 4424 0, CNTR_NORMAL, 4425 access_cce_cli0_async_fifo_parity_err_cnt), 4426 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0, 4427 CNTR_NORMAL, 4428 access_cce_rspd_data_parity_err_cnt), 4429 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0, 4430 CNTR_NORMAL, 4431 access_cce_trgt_access_err_cnt), 4432 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0, 4433 0, CNTR_NORMAL, 4434 access_cce_trgt_async_fifo_parity_err_cnt), 4435 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0, 4436 CNTR_NORMAL, 4437 access_cce_csr_write_bad_addr_err_cnt), 4438 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0, 4439 CNTR_NORMAL, 4440 access_cce_csr_read_bad_addr_err_cnt), 4441 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0, 4442 CNTR_NORMAL, 4443 access_ccs_csr_parity_err_cnt), 4444 4445 /* RcvErrStatus */ 4446 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0, 4447 CNTR_NORMAL, 4448 access_rx_csr_parity_err_cnt), 4449 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0, 4450 CNTR_NORMAL, 4451 access_rx_csr_write_bad_addr_err_cnt), 4452 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0, 4453 CNTR_NORMAL, 4454 access_rx_csr_read_bad_addr_err_cnt), 4455 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0, 4456 CNTR_NORMAL, 4457 access_rx_dma_csr_unc_err_cnt), 4458 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0, 4459 CNTR_NORMAL, 4460 access_rx_dma_dq_fsm_encoding_err_cnt), 4461 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0, 4462 CNTR_NORMAL, 4463 access_rx_dma_eq_fsm_encoding_err_cnt), 4464 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0, 4465 CNTR_NORMAL, 4466 access_rx_dma_csr_parity_err_cnt), 4467 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0, 4468 CNTR_NORMAL, 4469 access_rx_rbuf_data_cor_err_cnt), 4470 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0, 4471 CNTR_NORMAL, 4472 access_rx_rbuf_data_unc_err_cnt), 4473 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0, 4474 CNTR_NORMAL, 4475 access_rx_dma_data_fifo_rd_cor_err_cnt), 4476 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0, 4477 CNTR_NORMAL, 4478 access_rx_dma_data_fifo_rd_unc_err_cnt), 4479 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0, 4480 CNTR_NORMAL, 4481 access_rx_dma_hdr_fifo_rd_cor_err_cnt), 4482 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0, 4483 CNTR_NORMAL, 4484 access_rx_dma_hdr_fifo_rd_unc_err_cnt), 4485 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0, 4486 CNTR_NORMAL, 4487 access_rx_rbuf_desc_part2_cor_err_cnt), 4488 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0, 4489 CNTR_NORMAL, 4490 access_rx_rbuf_desc_part2_unc_err_cnt), 4491 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0, 4492 CNTR_NORMAL, 4493 access_rx_rbuf_desc_part1_cor_err_cnt), 4494 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0, 4495 CNTR_NORMAL, 4496 access_rx_rbuf_desc_part1_unc_err_cnt), 4497 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0, 4498 CNTR_NORMAL, 4499 access_rx_hq_intr_fsm_err_cnt), 4500 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0, 4501 CNTR_NORMAL, 4502 access_rx_hq_intr_csr_parity_err_cnt), 4503 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0, 4504 CNTR_NORMAL, 4505 access_rx_lookup_csr_parity_err_cnt), 4506 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0, 4507 CNTR_NORMAL, 4508 access_rx_lookup_rcv_array_cor_err_cnt), 4509 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0, 4510 CNTR_NORMAL, 4511 access_rx_lookup_rcv_array_unc_err_cnt), 4512 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0, 4513 0, CNTR_NORMAL, 4514 access_rx_lookup_des_part2_parity_err_cnt), 4515 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0, 4516 0, CNTR_NORMAL, 4517 access_rx_lookup_des_part1_unc_cor_err_cnt), 4518 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0, 4519 CNTR_NORMAL, 4520 access_rx_lookup_des_part1_unc_err_cnt), 4521 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0, 4522 CNTR_NORMAL, 4523 access_rx_rbuf_next_free_buf_cor_err_cnt), 4524 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0, 4525 CNTR_NORMAL, 4526 access_rx_rbuf_next_free_buf_unc_err_cnt), 4527 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM( 4528 "RxRbufFlInitWrAddrParityErr", 0, 0, 4529 CNTR_NORMAL, 4530 access_rbuf_fl_init_wr_addr_parity_err_cnt), 4531 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0, 4532 0, CNTR_NORMAL, 4533 access_rx_rbuf_fl_initdone_parity_err_cnt), 4534 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0, 4535 0, CNTR_NORMAL, 4536 access_rx_rbuf_fl_write_addr_parity_err_cnt), 4537 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0, 4538 CNTR_NORMAL, 4539 access_rx_rbuf_fl_rd_addr_parity_err_cnt), 4540 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0, 4541 CNTR_NORMAL, 4542 access_rx_rbuf_empty_err_cnt), 4543 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0, 4544 CNTR_NORMAL, 4545 access_rx_rbuf_full_err_cnt), 4546 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0, 4547 CNTR_NORMAL, 4548 access_rbuf_bad_lookup_err_cnt), 4549 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0, 4550 CNTR_NORMAL, 4551 access_rbuf_ctx_id_parity_err_cnt), 4552 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0, 4553 CNTR_NORMAL, 4554 access_rbuf_csr_qeopdw_parity_err_cnt), 4555 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM( 4556 "RxRbufCsrQNumOfPktParityErr", 0, 0, 4557 CNTR_NORMAL, 4558 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt), 4559 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM( 4560 "RxRbufCsrQTlPtrParityErr", 0, 0, 4561 CNTR_NORMAL, 4562 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt), 4563 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0, 4564 0, CNTR_NORMAL, 4565 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt), 4566 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0, 4567 0, CNTR_NORMAL, 4568 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt), 4569 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr", 4570 0, 0, CNTR_NORMAL, 4571 access_rx_rbuf_csr_q_next_buf_parity_err_cnt), 4572 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0, 4573 0, CNTR_NORMAL, 4574 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt), 4575 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM( 4576 "RxRbufCsrQHeadBufNumParityErr", 0, 0, 4577 CNTR_NORMAL, 4578 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt), 4579 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0, 4580 0, CNTR_NORMAL, 4581 access_rx_rbuf_block_list_read_cor_err_cnt), 4582 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0, 4583 0, CNTR_NORMAL, 4584 access_rx_rbuf_block_list_read_unc_err_cnt), 4585 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0, 4586 CNTR_NORMAL, 4587 access_rx_rbuf_lookup_des_cor_err_cnt), 4588 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0, 4589 CNTR_NORMAL, 4590 access_rx_rbuf_lookup_des_unc_err_cnt), 4591 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM( 4592 "RxRbufLookupDesRegUncCorErr", 0, 0, 4593 CNTR_NORMAL, 4594 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt), 4595 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0, 4596 CNTR_NORMAL, 4597 access_rx_rbuf_lookup_des_reg_unc_err_cnt), 4598 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0, 4599 CNTR_NORMAL, 4600 access_rx_rbuf_free_list_cor_err_cnt), 4601 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0, 4602 CNTR_NORMAL, 4603 access_rx_rbuf_free_list_unc_err_cnt), 4604 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0, 4605 CNTR_NORMAL, 4606 access_rx_rcv_fsm_encoding_err_cnt), 4607 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0, 4608 CNTR_NORMAL, 4609 access_rx_dma_flag_cor_err_cnt), 4610 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0, 4611 CNTR_NORMAL, 4612 access_rx_dma_flag_unc_err_cnt), 4613 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0, 4614 CNTR_NORMAL, 4615 access_rx_dc_sop_eop_parity_err_cnt), 4616 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0, 4617 CNTR_NORMAL, 4618 access_rx_rcv_csr_parity_err_cnt), 4619 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0, 4620 CNTR_NORMAL, 4621 access_rx_rcv_qp_map_table_cor_err_cnt), 4622 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0, 4623 CNTR_NORMAL, 4624 access_rx_rcv_qp_map_table_unc_err_cnt), 4625 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0, 4626 CNTR_NORMAL, 4627 access_rx_rcv_data_cor_err_cnt), 4628 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0, 4629 CNTR_NORMAL, 4630 access_rx_rcv_data_unc_err_cnt), 4631 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0, 4632 CNTR_NORMAL, 4633 access_rx_rcv_hdr_cor_err_cnt), 4634 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0, 4635 CNTR_NORMAL, 4636 access_rx_rcv_hdr_unc_err_cnt), 4637 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0, 4638 CNTR_NORMAL, 4639 access_rx_dc_intf_parity_err_cnt), 4640 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0, 4641 CNTR_NORMAL, 4642 access_rx_dma_csr_cor_err_cnt), 4643 /* SendPioErrStatus */ 4644 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0, 4645 CNTR_NORMAL, 4646 access_pio_pec_sop_head_parity_err_cnt), 4647 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0, 4648 CNTR_NORMAL, 4649 access_pio_pcc_sop_head_parity_err_cnt), 4650 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr", 4651 0, 0, CNTR_NORMAL, 4652 access_pio_last_returned_cnt_parity_err_cnt), 4653 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0, 4654 0, CNTR_NORMAL, 4655 access_pio_current_free_cnt_parity_err_cnt), 4656 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0, 4657 CNTR_NORMAL, 4658 access_pio_reserved_31_err_cnt), 4659 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0, 4660 CNTR_NORMAL, 4661 access_pio_reserved_30_err_cnt), 4662 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0, 4663 CNTR_NORMAL, 4664 access_pio_ppmc_sop_len_err_cnt), 4665 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0, 4666 CNTR_NORMAL, 4667 access_pio_ppmc_bqc_mem_parity_err_cnt), 4668 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0, 4669 CNTR_NORMAL, 4670 access_pio_vl_fifo_parity_err_cnt), 4671 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0, 4672 CNTR_NORMAL, 4673 access_pio_vlf_sop_parity_err_cnt), 4674 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0, 4675 CNTR_NORMAL, 4676 access_pio_vlf_v1_len_parity_err_cnt), 4677 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0, 4678 CNTR_NORMAL, 4679 access_pio_block_qw_count_parity_err_cnt), 4680 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0, 4681 CNTR_NORMAL, 4682 access_pio_write_qw_valid_parity_err_cnt), 4683 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0, 4684 CNTR_NORMAL, 4685 access_pio_state_machine_err_cnt), 4686 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0, 4687 CNTR_NORMAL, 4688 access_pio_write_data_parity_err_cnt), 4689 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0, 4690 CNTR_NORMAL, 4691 access_pio_host_addr_mem_cor_err_cnt), 4692 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0, 4693 CNTR_NORMAL, 4694 access_pio_host_addr_mem_unc_err_cnt), 4695 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0, 4696 CNTR_NORMAL, 4697 access_pio_pkt_evict_sm_or_arb_sm_err_cnt), 4698 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0, 4699 CNTR_NORMAL, 4700 access_pio_init_sm_in_err_cnt), 4701 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0, 4702 CNTR_NORMAL, 4703 access_pio_ppmc_pbl_fifo_err_cnt), 4704 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0, 4705 0, CNTR_NORMAL, 4706 access_pio_credit_ret_fifo_parity_err_cnt), 4707 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0, 4708 CNTR_NORMAL, 4709 access_pio_v1_len_mem_bank1_cor_err_cnt), 4710 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0, 4711 CNTR_NORMAL, 4712 access_pio_v1_len_mem_bank0_cor_err_cnt), 4713 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0, 4714 CNTR_NORMAL, 4715 access_pio_v1_len_mem_bank1_unc_err_cnt), 4716 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0, 4717 CNTR_NORMAL, 4718 access_pio_v1_len_mem_bank0_unc_err_cnt), 4719 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0, 4720 CNTR_NORMAL, 4721 access_pio_sm_pkt_reset_parity_err_cnt), 4722 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0, 4723 CNTR_NORMAL, 4724 access_pio_pkt_evict_fifo_parity_err_cnt), 4725 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM( 4726 "PioSbrdctrlCrrelFifoParityErr", 0, 0, 4727 CNTR_NORMAL, 4728 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt), 4729 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0, 4730 CNTR_NORMAL, 4731 access_pio_sbrdctl_crrel_parity_err_cnt), 4732 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0, 4733 CNTR_NORMAL, 4734 access_pio_pec_fifo_parity_err_cnt), 4735 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0, 4736 CNTR_NORMAL, 4737 access_pio_pcc_fifo_parity_err_cnt), 4738 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0, 4739 CNTR_NORMAL, 4740 access_pio_sb_mem_fifo1_err_cnt), 4741 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0, 4742 CNTR_NORMAL, 4743 access_pio_sb_mem_fifo0_err_cnt), 4744 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0, 4745 CNTR_NORMAL, 4746 access_pio_csr_parity_err_cnt), 4747 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0, 4748 CNTR_NORMAL, 4749 access_pio_write_addr_parity_err_cnt), 4750 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0, 4751 CNTR_NORMAL, 4752 access_pio_write_bad_ctxt_err_cnt), 4753 /* SendDmaErrStatus */ 4754 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0, 4755 0, CNTR_NORMAL, 4756 access_sdma_pcie_req_tracking_cor_err_cnt), 4757 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0, 4758 0, CNTR_NORMAL, 4759 access_sdma_pcie_req_tracking_unc_err_cnt), 4760 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0, 4761 CNTR_NORMAL, 4762 access_sdma_csr_parity_err_cnt), 4763 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0, 4764 CNTR_NORMAL, 4765 access_sdma_rpy_tag_err_cnt), 4766 /* SendEgressErrStatus */ 4767 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0, 4768 CNTR_NORMAL, 4769 access_tx_read_pio_memory_csr_unc_err_cnt), 4770 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0, 4771 0, CNTR_NORMAL, 4772 access_tx_read_sdma_memory_csr_err_cnt), 4773 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0, 4774 CNTR_NORMAL, 4775 access_tx_egress_fifo_cor_err_cnt), 4776 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0, 4777 CNTR_NORMAL, 4778 access_tx_read_pio_memory_cor_err_cnt), 4779 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0, 4780 CNTR_NORMAL, 4781 access_tx_read_sdma_memory_cor_err_cnt), 4782 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0, 4783 CNTR_NORMAL, 4784 access_tx_sb_hdr_cor_err_cnt), 4785 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0, 4786 CNTR_NORMAL, 4787 access_tx_credit_overrun_err_cnt), 4788 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0, 4789 CNTR_NORMAL, 4790 access_tx_launch_fifo8_cor_err_cnt), 4791 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0, 4792 CNTR_NORMAL, 4793 access_tx_launch_fifo7_cor_err_cnt), 4794 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0, 4795 CNTR_NORMAL, 4796 access_tx_launch_fifo6_cor_err_cnt), 4797 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0, 4798 CNTR_NORMAL, 4799 access_tx_launch_fifo5_cor_err_cnt), 4800 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0, 4801 CNTR_NORMAL, 4802 access_tx_launch_fifo4_cor_err_cnt), 4803 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0, 4804 CNTR_NORMAL, 4805 access_tx_launch_fifo3_cor_err_cnt), 4806 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0, 4807 CNTR_NORMAL, 4808 access_tx_launch_fifo2_cor_err_cnt), 4809 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0, 4810 CNTR_NORMAL, 4811 access_tx_launch_fifo1_cor_err_cnt), 4812 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0, 4813 CNTR_NORMAL, 4814 access_tx_launch_fifo0_cor_err_cnt), 4815 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0, 4816 CNTR_NORMAL, 4817 access_tx_credit_return_vl_err_cnt), 4818 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0, 4819 CNTR_NORMAL, 4820 access_tx_hcrc_insertion_err_cnt), 4821 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0, 4822 CNTR_NORMAL, 4823 access_tx_egress_fifo_unc_err_cnt), 4824 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0, 4825 CNTR_NORMAL, 4826 access_tx_read_pio_memory_unc_err_cnt), 4827 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0, 4828 CNTR_NORMAL, 4829 access_tx_read_sdma_memory_unc_err_cnt), 4830 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0, 4831 CNTR_NORMAL, 4832 access_tx_sb_hdr_unc_err_cnt), 4833 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0, 4834 CNTR_NORMAL, 4835 access_tx_credit_return_partiy_err_cnt), 4836 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr", 4837 0, 0, CNTR_NORMAL, 4838 access_tx_launch_fifo8_unc_or_parity_err_cnt), 4839 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr", 4840 0, 0, CNTR_NORMAL, 4841 access_tx_launch_fifo7_unc_or_parity_err_cnt), 4842 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr", 4843 0, 0, CNTR_NORMAL, 4844 access_tx_launch_fifo6_unc_or_parity_err_cnt), 4845 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr", 4846 0, 0, CNTR_NORMAL, 4847 access_tx_launch_fifo5_unc_or_parity_err_cnt), 4848 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr", 4849 0, 0, CNTR_NORMAL, 4850 access_tx_launch_fifo4_unc_or_parity_err_cnt), 4851 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr", 4852 0, 0, CNTR_NORMAL, 4853 access_tx_launch_fifo3_unc_or_parity_err_cnt), 4854 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr", 4855 0, 0, CNTR_NORMAL, 4856 access_tx_launch_fifo2_unc_or_parity_err_cnt), 4857 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr", 4858 0, 0, CNTR_NORMAL, 4859 access_tx_launch_fifo1_unc_or_parity_err_cnt), 4860 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr", 4861 0, 0, CNTR_NORMAL, 4862 access_tx_launch_fifo0_unc_or_parity_err_cnt), 4863 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr", 4864 0, 0, CNTR_NORMAL, 4865 access_tx_sdma15_disallowed_packet_err_cnt), 4866 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr", 4867 0, 0, CNTR_NORMAL, 4868 access_tx_sdma14_disallowed_packet_err_cnt), 4869 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr", 4870 0, 0, CNTR_NORMAL, 4871 access_tx_sdma13_disallowed_packet_err_cnt), 4872 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr", 4873 0, 0, CNTR_NORMAL, 4874 access_tx_sdma12_disallowed_packet_err_cnt), 4875 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr", 4876 0, 0, CNTR_NORMAL, 4877 access_tx_sdma11_disallowed_packet_err_cnt), 4878 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr", 4879 0, 0, CNTR_NORMAL, 4880 access_tx_sdma10_disallowed_packet_err_cnt), 4881 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr", 4882 0, 0, CNTR_NORMAL, 4883 access_tx_sdma9_disallowed_packet_err_cnt), 4884 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr", 4885 0, 0, CNTR_NORMAL, 4886 access_tx_sdma8_disallowed_packet_err_cnt), 4887 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr", 4888 0, 0, CNTR_NORMAL, 4889 access_tx_sdma7_disallowed_packet_err_cnt), 4890 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr", 4891 0, 0, CNTR_NORMAL, 4892 access_tx_sdma6_disallowed_packet_err_cnt), 4893 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr", 4894 0, 0, CNTR_NORMAL, 4895 access_tx_sdma5_disallowed_packet_err_cnt), 4896 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr", 4897 0, 0, CNTR_NORMAL, 4898 access_tx_sdma4_disallowed_packet_err_cnt), 4899 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr", 4900 0, 0, CNTR_NORMAL, 4901 access_tx_sdma3_disallowed_packet_err_cnt), 4902 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr", 4903 0, 0, CNTR_NORMAL, 4904 access_tx_sdma2_disallowed_packet_err_cnt), 4905 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr", 4906 0, 0, CNTR_NORMAL, 4907 access_tx_sdma1_disallowed_packet_err_cnt), 4908 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr", 4909 0, 0, CNTR_NORMAL, 4910 access_tx_sdma0_disallowed_packet_err_cnt), 4911 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0, 4912 CNTR_NORMAL, 4913 access_tx_config_parity_err_cnt), 4914 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0, 4915 CNTR_NORMAL, 4916 access_tx_sbrd_ctl_csr_parity_err_cnt), 4917 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0, 4918 CNTR_NORMAL, 4919 access_tx_launch_csr_parity_err_cnt), 4920 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0, 4921 CNTR_NORMAL, 4922 access_tx_illegal_vl_err_cnt), 4923 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM( 4924 "TxSbrdCtlStateMachineParityErr", 0, 0, 4925 CNTR_NORMAL, 4926 access_tx_sbrd_ctl_state_machine_parity_err_cnt), 4927 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0, 4928 CNTR_NORMAL, 4929 access_egress_reserved_10_err_cnt), 4930 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0, 4931 CNTR_NORMAL, 4932 access_egress_reserved_9_err_cnt), 4933 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr", 4934 0, 0, CNTR_NORMAL, 4935 access_tx_sdma_launch_intf_parity_err_cnt), 4936 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0, 4937 CNTR_NORMAL, 4938 access_tx_pio_launch_intf_parity_err_cnt), 4939 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0, 4940 CNTR_NORMAL, 4941 access_egress_reserved_6_err_cnt), 4942 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0, 4943 CNTR_NORMAL, 4944 access_tx_incorrect_link_state_err_cnt), 4945 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0, 4946 CNTR_NORMAL, 4947 access_tx_linkdown_err_cnt), 4948 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM( 4949 "EgressFifoUnderrunOrParityErr", 0, 0, 4950 CNTR_NORMAL, 4951 access_tx_egress_fifi_underrun_or_parity_err_cnt), 4952 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0, 4953 CNTR_NORMAL, 4954 access_egress_reserved_2_err_cnt), 4955 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0, 4956 CNTR_NORMAL, 4957 access_tx_pkt_integrity_mem_unc_err_cnt), 4958 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0, 4959 CNTR_NORMAL, 4960 access_tx_pkt_integrity_mem_cor_err_cnt), 4961 /* SendErrStatus */ 4962 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0, 4963 CNTR_NORMAL, 4964 access_send_csr_write_bad_addr_err_cnt), 4965 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0, 4966 CNTR_NORMAL, 4967 access_send_csr_read_bad_addr_err_cnt), 4968 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0, 4969 CNTR_NORMAL, 4970 access_send_csr_parity_cnt), 4971 /* SendCtxtErrStatus */ 4972 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0, 4973 CNTR_NORMAL, 4974 access_pio_write_out_of_bounds_err_cnt), 4975 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0, 4976 CNTR_NORMAL, 4977 access_pio_write_overflow_err_cnt), 4978 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr", 4979 0, 0, CNTR_NORMAL, 4980 access_pio_write_crosses_boundary_err_cnt), 4981 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0, 4982 CNTR_NORMAL, 4983 access_pio_disallowed_packet_err_cnt), 4984 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0, 4985 CNTR_NORMAL, 4986 access_pio_inconsistent_sop_err_cnt), 4987 /* SendDmaEngErrStatus */ 4988 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr", 4989 0, 0, CNTR_NORMAL, 4990 access_sdma_header_request_fifo_cor_err_cnt), 4991 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0, 4992 CNTR_NORMAL, 4993 access_sdma_header_storage_cor_err_cnt), 4994 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0, 4995 CNTR_NORMAL, 4996 access_sdma_packet_tracking_cor_err_cnt), 4997 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0, 4998 CNTR_NORMAL, 4999 access_sdma_assembly_cor_err_cnt), 5000 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0, 5001 CNTR_NORMAL, 5002 access_sdma_desc_table_cor_err_cnt), 5003 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr", 5004 0, 0, CNTR_NORMAL, 5005 access_sdma_header_request_fifo_unc_err_cnt), 5006 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0, 5007 CNTR_NORMAL, 5008 access_sdma_header_storage_unc_err_cnt), 5009 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0, 5010 CNTR_NORMAL, 5011 access_sdma_packet_tracking_unc_err_cnt), 5012 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0, 5013 CNTR_NORMAL, 5014 access_sdma_assembly_unc_err_cnt), 5015 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0, 5016 CNTR_NORMAL, 5017 access_sdma_desc_table_unc_err_cnt), 5018 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0, 5019 CNTR_NORMAL, 5020 access_sdma_timeout_err_cnt), 5021 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0, 5022 CNTR_NORMAL, 5023 access_sdma_header_length_err_cnt), 5024 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0, 5025 CNTR_NORMAL, 5026 access_sdma_header_address_err_cnt), 5027 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0, 5028 CNTR_NORMAL, 5029 access_sdma_header_select_err_cnt), 5030 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0, 5031 CNTR_NORMAL, 5032 access_sdma_reserved_9_err_cnt), 5033 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0, 5034 CNTR_NORMAL, 5035 access_sdma_packet_desc_overflow_err_cnt), 5036 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0, 5037 CNTR_NORMAL, 5038 access_sdma_length_mismatch_err_cnt), 5039 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0, 5040 CNTR_NORMAL, 5041 access_sdma_halt_err_cnt), 5042 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0, 5043 CNTR_NORMAL, 5044 access_sdma_mem_read_err_cnt), 5045 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0, 5046 CNTR_NORMAL, 5047 access_sdma_first_desc_err_cnt), 5048 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0, 5049 CNTR_NORMAL, 5050 access_sdma_tail_out_of_bounds_err_cnt), 5051 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0, 5052 CNTR_NORMAL, 5053 access_sdma_too_long_err_cnt), 5054 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0, 5055 CNTR_NORMAL, 5056 access_sdma_gen_mismatch_err_cnt), 5057 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0, 5058 CNTR_NORMAL, 5059 access_sdma_wrong_dw_err_cnt), 5060 }; 5061 5062 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = { 5063 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT, 5064 CNTR_NORMAL), 5065 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT, 5066 CNTR_NORMAL), 5067 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT, 5068 CNTR_NORMAL), 5069 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT, 5070 CNTR_NORMAL), 5071 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT, 5072 CNTR_NORMAL), 5073 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT, 5074 CNTR_NORMAL), 5075 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT, 5076 CNTR_NORMAL), 5077 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL), 5078 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL), 5079 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH), 5080 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT, 5081 CNTR_SYNTH | CNTR_VL), 5082 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT, 5083 CNTR_SYNTH | CNTR_VL), 5084 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT, 5085 CNTR_SYNTH | CNTR_VL), 5086 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL), 5087 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL), 5088 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5089 access_sw_link_dn_cnt), 5090 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5091 access_sw_link_up_cnt), 5092 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL, 5093 access_sw_unknown_frame_cnt), 5094 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5095 access_sw_xmit_discards), 5096 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0, 5097 CNTR_SYNTH | CNTR_32BIT | CNTR_VL, 5098 access_sw_xmit_discards), 5099 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH, 5100 access_xmit_constraint_errs), 5101 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH, 5102 access_rcv_constraint_errs), 5103 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts), 5104 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends), 5105 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks), 5106 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks), 5107 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts), 5108 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops), 5109 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait), 5110 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak), 5111 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq), 5112 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq), 5113 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned), 5114 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks), 5115 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL, 5116 access_sw_cpu_rc_acks), 5117 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL, 5118 access_sw_cpu_rc_qacks), 5119 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL, 5120 access_sw_cpu_rc_delayed_comp), 5121 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1), 5122 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3), 5123 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5), 5124 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7), 5125 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9), 5126 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11), 5127 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13), 5128 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15), 5129 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17), 5130 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19), 5131 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21), 5132 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23), 5133 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25), 5134 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27), 5135 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29), 5136 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31), 5137 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33), 5138 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35), 5139 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37), 5140 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39), 5141 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41), 5142 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43), 5143 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45), 5144 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47), 5145 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49), 5146 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51), 5147 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53), 5148 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55), 5149 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57), 5150 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59), 5151 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61), 5152 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63), 5153 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65), 5154 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67), 5155 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69), 5156 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71), 5157 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73), 5158 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75), 5159 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77), 5160 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79), 5161 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81), 5162 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83), 5163 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85), 5164 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87), 5165 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89), 5166 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91), 5167 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93), 5168 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95), 5169 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97), 5170 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99), 5171 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101), 5172 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103), 5173 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105), 5174 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107), 5175 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109), 5176 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111), 5177 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113), 5178 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115), 5179 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117), 5180 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119), 5181 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121), 5182 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123), 5183 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125), 5184 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127), 5185 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129), 5186 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131), 5187 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133), 5188 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135), 5189 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137), 5190 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139), 5191 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141), 5192 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143), 5193 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145), 5194 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147), 5195 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149), 5196 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151), 5197 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153), 5198 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155), 5199 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157), 5200 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159), 5201 }; 5202 5203 /* ======================================================================== */ 5204 5205 /* return true if this is chip revision revision a */ 5206 int is_ax(struct hfi1_devdata *dd) 5207 { 5208 u8 chip_rev_minor = 5209 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5210 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5211 return (chip_rev_minor & 0xf0) == 0; 5212 } 5213 5214 /* return true if this is chip revision revision b */ 5215 int is_bx(struct hfi1_devdata *dd) 5216 { 5217 u8 chip_rev_minor = 5218 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5219 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5220 return (chip_rev_minor & 0xF0) == 0x10; 5221 } 5222 5223 /* 5224 * Append string s to buffer buf. Arguments curp and len are the current 5225 * position and remaining length, respectively. 5226 * 5227 * return 0 on success, 1 on out of room 5228 */ 5229 static int append_str(char *buf, char **curp, int *lenp, const char *s) 5230 { 5231 char *p = *curp; 5232 int len = *lenp; 5233 int result = 0; /* success */ 5234 char c; 5235 5236 /* add a comma, if first in the buffer */ 5237 if (p != buf) { 5238 if (len == 0) { 5239 result = 1; /* out of room */ 5240 goto done; 5241 } 5242 *p++ = ','; 5243 len--; 5244 } 5245 5246 /* copy the string */ 5247 while ((c = *s++) != 0) { 5248 if (len == 0) { 5249 result = 1; /* out of room */ 5250 goto done; 5251 } 5252 *p++ = c; 5253 len--; 5254 } 5255 5256 done: 5257 /* write return values */ 5258 *curp = p; 5259 *lenp = len; 5260 5261 return result; 5262 } 5263 5264 /* 5265 * Using the given flag table, print a comma separated string into 5266 * the buffer. End in '*' if the buffer is too short. 5267 */ 5268 static char *flag_string(char *buf, int buf_len, u64 flags, 5269 struct flag_table *table, int table_size) 5270 { 5271 char extra[32]; 5272 char *p = buf; 5273 int len = buf_len; 5274 int no_room = 0; 5275 int i; 5276 5277 /* make sure there is at least 2 so we can form "*" */ 5278 if (len < 2) 5279 return ""; 5280 5281 len--; /* leave room for a nul */ 5282 for (i = 0; i < table_size; i++) { 5283 if (flags & table[i].flag) { 5284 no_room = append_str(buf, &p, &len, table[i].str); 5285 if (no_room) 5286 break; 5287 flags &= ~table[i].flag; 5288 } 5289 } 5290 5291 /* any undocumented bits left? */ 5292 if (!no_room && flags) { 5293 snprintf(extra, sizeof(extra), "bits 0x%llx", flags); 5294 no_room = append_str(buf, &p, &len, extra); 5295 } 5296 5297 /* add * if ran out of room */ 5298 if (no_room) { 5299 /* may need to back up to add space for a '*' */ 5300 if (len == 0) 5301 --p; 5302 *p++ = '*'; 5303 } 5304 5305 /* add final nul - space already allocated above */ 5306 *p = 0; 5307 return buf; 5308 } 5309 5310 /* first 8 CCE error interrupt source names */ 5311 static const char * const cce_misc_names[] = { 5312 "CceErrInt", /* 0 */ 5313 "RxeErrInt", /* 1 */ 5314 "MiscErrInt", /* 2 */ 5315 "Reserved3", /* 3 */ 5316 "PioErrInt", /* 4 */ 5317 "SDmaErrInt", /* 5 */ 5318 "EgressErrInt", /* 6 */ 5319 "TxeErrInt" /* 7 */ 5320 }; 5321 5322 /* 5323 * Return the miscellaneous error interrupt name. 5324 */ 5325 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source) 5326 { 5327 if (source < ARRAY_SIZE(cce_misc_names)) 5328 strncpy(buf, cce_misc_names[source], bsize); 5329 else 5330 snprintf(buf, bsize, "Reserved%u", 5331 source + IS_GENERAL_ERR_START); 5332 5333 return buf; 5334 } 5335 5336 /* 5337 * Return the SDMA engine error interrupt name. 5338 */ 5339 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source) 5340 { 5341 snprintf(buf, bsize, "SDmaEngErrInt%u", source); 5342 return buf; 5343 } 5344 5345 /* 5346 * Return the send context error interrupt name. 5347 */ 5348 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source) 5349 { 5350 snprintf(buf, bsize, "SendCtxtErrInt%u", source); 5351 return buf; 5352 } 5353 5354 static const char * const various_names[] = { 5355 "PbcInt", 5356 "GpioAssertInt", 5357 "Qsfp1Int", 5358 "Qsfp2Int", 5359 "TCritInt" 5360 }; 5361 5362 /* 5363 * Return the various interrupt name. 5364 */ 5365 static char *is_various_name(char *buf, size_t bsize, unsigned int source) 5366 { 5367 if (source < ARRAY_SIZE(various_names)) 5368 strncpy(buf, various_names[source], bsize); 5369 else 5370 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START); 5371 return buf; 5372 } 5373 5374 /* 5375 * Return the DC interrupt name. 5376 */ 5377 static char *is_dc_name(char *buf, size_t bsize, unsigned int source) 5378 { 5379 static const char * const dc_int_names[] = { 5380 "common", 5381 "lcb", 5382 "8051", 5383 "lbm" /* local block merge */ 5384 }; 5385 5386 if (source < ARRAY_SIZE(dc_int_names)) 5387 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]); 5388 else 5389 snprintf(buf, bsize, "DCInt%u", source); 5390 return buf; 5391 } 5392 5393 static const char * const sdma_int_names[] = { 5394 "SDmaInt", 5395 "SdmaIdleInt", 5396 "SdmaProgressInt", 5397 }; 5398 5399 /* 5400 * Return the SDMA engine interrupt name. 5401 */ 5402 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source) 5403 { 5404 /* what interrupt */ 5405 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 5406 /* which engine */ 5407 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 5408 5409 if (likely(what < 3)) 5410 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which); 5411 else 5412 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source); 5413 return buf; 5414 } 5415 5416 /* 5417 * Return the receive available interrupt name. 5418 */ 5419 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source) 5420 { 5421 snprintf(buf, bsize, "RcvAvailInt%u", source); 5422 return buf; 5423 } 5424 5425 /* 5426 * Return the receive urgent interrupt name. 5427 */ 5428 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source) 5429 { 5430 snprintf(buf, bsize, "RcvUrgentInt%u", source); 5431 return buf; 5432 } 5433 5434 /* 5435 * Return the send credit interrupt name. 5436 */ 5437 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source) 5438 { 5439 snprintf(buf, bsize, "SendCreditInt%u", source); 5440 return buf; 5441 } 5442 5443 /* 5444 * Return the reserved interrupt name. 5445 */ 5446 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source) 5447 { 5448 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START); 5449 return buf; 5450 } 5451 5452 static char *cce_err_status_string(char *buf, int buf_len, u64 flags) 5453 { 5454 return flag_string(buf, buf_len, flags, 5455 cce_err_status_flags, 5456 ARRAY_SIZE(cce_err_status_flags)); 5457 } 5458 5459 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags) 5460 { 5461 return flag_string(buf, buf_len, flags, 5462 rxe_err_status_flags, 5463 ARRAY_SIZE(rxe_err_status_flags)); 5464 } 5465 5466 static char *misc_err_status_string(char *buf, int buf_len, u64 flags) 5467 { 5468 return flag_string(buf, buf_len, flags, misc_err_status_flags, 5469 ARRAY_SIZE(misc_err_status_flags)); 5470 } 5471 5472 static char *pio_err_status_string(char *buf, int buf_len, u64 flags) 5473 { 5474 return flag_string(buf, buf_len, flags, 5475 pio_err_status_flags, 5476 ARRAY_SIZE(pio_err_status_flags)); 5477 } 5478 5479 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags) 5480 { 5481 return flag_string(buf, buf_len, flags, 5482 sdma_err_status_flags, 5483 ARRAY_SIZE(sdma_err_status_flags)); 5484 } 5485 5486 static char *egress_err_status_string(char *buf, int buf_len, u64 flags) 5487 { 5488 return flag_string(buf, buf_len, flags, 5489 egress_err_status_flags, 5490 ARRAY_SIZE(egress_err_status_flags)); 5491 } 5492 5493 static char *egress_err_info_string(char *buf, int buf_len, u64 flags) 5494 { 5495 return flag_string(buf, buf_len, flags, 5496 egress_err_info_flags, 5497 ARRAY_SIZE(egress_err_info_flags)); 5498 } 5499 5500 static char *send_err_status_string(char *buf, int buf_len, u64 flags) 5501 { 5502 return flag_string(buf, buf_len, flags, 5503 send_err_status_flags, 5504 ARRAY_SIZE(send_err_status_flags)); 5505 } 5506 5507 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5508 { 5509 char buf[96]; 5510 int i = 0; 5511 5512 /* 5513 * For most these errors, there is nothing that can be done except 5514 * report or record it. 5515 */ 5516 dd_dev_info(dd, "CCE Error: %s\n", 5517 cce_err_status_string(buf, sizeof(buf), reg)); 5518 5519 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) && 5520 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) { 5521 /* this error requires a manual drop into SPC freeze mode */ 5522 /* then a fix up */ 5523 start_freeze_handling(dd->pport, FREEZE_SELF); 5524 } 5525 5526 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) { 5527 if (reg & (1ull << i)) { 5528 incr_cntr64(&dd->cce_err_status_cnt[i]); 5529 /* maintain a counter over all cce_err_status errors */ 5530 incr_cntr64(&dd->sw_cce_err_status_aggregate); 5531 } 5532 } 5533 } 5534 5535 /* 5536 * Check counters for receive errors that do not have an interrupt 5537 * associated with them. 5538 */ 5539 #define RCVERR_CHECK_TIME 10 5540 static void update_rcverr_timer(struct timer_list *t) 5541 { 5542 struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer); 5543 struct hfi1_pportdata *ppd = dd->pport; 5544 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL); 5545 5546 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt && 5547 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) { 5548 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__); 5549 set_link_down_reason( 5550 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0, 5551 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN); 5552 queue_work(ppd->link_wq, &ppd->link_bounce_work); 5553 } 5554 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt; 5555 5556 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5557 } 5558 5559 static int init_rcverr(struct hfi1_devdata *dd) 5560 { 5561 timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0); 5562 /* Assume the hardware counter has been reset */ 5563 dd->rcv_ovfl_cnt = 0; 5564 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5565 } 5566 5567 static void free_rcverr(struct hfi1_devdata *dd) 5568 { 5569 if (dd->rcverr_timer.function) 5570 del_timer_sync(&dd->rcverr_timer); 5571 } 5572 5573 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5574 { 5575 char buf[96]; 5576 int i = 0; 5577 5578 dd_dev_info(dd, "Receive Error: %s\n", 5579 rxe_err_status_string(buf, sizeof(buf), reg)); 5580 5581 if (reg & ALL_RXE_FREEZE_ERR) { 5582 int flags = 0; 5583 5584 /* 5585 * Freeze mode recovery is disabled for the errors 5586 * in RXE_FREEZE_ABORT_MASK 5587 */ 5588 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK)) 5589 flags = FREEZE_ABORT; 5590 5591 start_freeze_handling(dd->pport, flags); 5592 } 5593 5594 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) { 5595 if (reg & (1ull << i)) 5596 incr_cntr64(&dd->rcv_err_status_cnt[i]); 5597 } 5598 } 5599 5600 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5601 { 5602 char buf[96]; 5603 int i = 0; 5604 5605 dd_dev_info(dd, "Misc Error: %s", 5606 misc_err_status_string(buf, sizeof(buf), reg)); 5607 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) { 5608 if (reg & (1ull << i)) 5609 incr_cntr64(&dd->misc_err_status_cnt[i]); 5610 } 5611 } 5612 5613 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5614 { 5615 char buf[96]; 5616 int i = 0; 5617 5618 dd_dev_info(dd, "PIO Error: %s\n", 5619 pio_err_status_string(buf, sizeof(buf), reg)); 5620 5621 if (reg & ALL_PIO_FREEZE_ERR) 5622 start_freeze_handling(dd->pport, 0); 5623 5624 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) { 5625 if (reg & (1ull << i)) 5626 incr_cntr64(&dd->send_pio_err_status_cnt[i]); 5627 } 5628 } 5629 5630 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5631 { 5632 char buf[96]; 5633 int i = 0; 5634 5635 dd_dev_info(dd, "SDMA Error: %s\n", 5636 sdma_err_status_string(buf, sizeof(buf), reg)); 5637 5638 if (reg & ALL_SDMA_FREEZE_ERR) 5639 start_freeze_handling(dd->pport, 0); 5640 5641 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) { 5642 if (reg & (1ull << i)) 5643 incr_cntr64(&dd->send_dma_err_status_cnt[i]); 5644 } 5645 } 5646 5647 static inline void __count_port_discards(struct hfi1_pportdata *ppd) 5648 { 5649 incr_cntr64(&ppd->port_xmit_discards); 5650 } 5651 5652 static void count_port_inactive(struct hfi1_devdata *dd) 5653 { 5654 __count_port_discards(dd->pport); 5655 } 5656 5657 /* 5658 * We have had a "disallowed packet" error during egress. Determine the 5659 * integrity check which failed, and update relevant error counter, etc. 5660 * 5661 * Note that the SEND_EGRESS_ERR_INFO register has only a single 5662 * bit of state per integrity check, and so we can miss the reason for an 5663 * egress error if more than one packet fails the same integrity check 5664 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO. 5665 */ 5666 static void handle_send_egress_err_info(struct hfi1_devdata *dd, 5667 int vl) 5668 { 5669 struct hfi1_pportdata *ppd = dd->pport; 5670 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */ 5671 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO); 5672 char buf[96]; 5673 5674 /* clear down all observed info as quickly as possible after read */ 5675 write_csr(dd, SEND_EGRESS_ERR_INFO, info); 5676 5677 dd_dev_info(dd, 5678 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n", 5679 info, egress_err_info_string(buf, sizeof(buf), info), src); 5680 5681 /* Eventually add other counters for each bit */ 5682 if (info & PORT_DISCARD_EGRESS_ERRS) { 5683 int weight, i; 5684 5685 /* 5686 * Count all applicable bits as individual errors and 5687 * attribute them to the packet that triggered this handler. 5688 * This may not be completely accurate due to limitations 5689 * on the available hardware error information. There is 5690 * a single information register and any number of error 5691 * packets may have occurred and contributed to it before 5692 * this routine is called. This means that: 5693 * a) If multiple packets with the same error occur before 5694 * this routine is called, earlier packets are missed. 5695 * There is only a single bit for each error type. 5696 * b) Errors may not be attributed to the correct VL. 5697 * The driver is attributing all bits in the info register 5698 * to the packet that triggered this call, but bits 5699 * could be an accumulation of different packets with 5700 * different VLs. 5701 * c) A single error packet may have multiple counts attached 5702 * to it. There is no way for the driver to know if 5703 * multiple bits set in the info register are due to a 5704 * single packet or multiple packets. The driver assumes 5705 * multiple packets. 5706 */ 5707 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS); 5708 for (i = 0; i < weight; i++) { 5709 __count_port_discards(ppd); 5710 if (vl >= 0 && vl < TXE_NUM_DATA_VL) 5711 incr_cntr64(&ppd->port_xmit_discards_vl[vl]); 5712 else if (vl == 15) 5713 incr_cntr64(&ppd->port_xmit_discards_vl 5714 [C_VL_15]); 5715 } 5716 } 5717 } 5718 5719 /* 5720 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5721 * register. Does it represent a 'port inactive' error? 5722 */ 5723 static inline int port_inactive_err(u64 posn) 5724 { 5725 return (posn >= SEES(TX_LINKDOWN) && 5726 posn <= SEES(TX_INCORRECT_LINK_STATE)); 5727 } 5728 5729 /* 5730 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5731 * register. Does it represent a 'disallowed packet' error? 5732 */ 5733 static inline int disallowed_pkt_err(int posn) 5734 { 5735 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) && 5736 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET)); 5737 } 5738 5739 /* 5740 * Input value is a bit position of one of the SDMA engine disallowed 5741 * packet errors. Return which engine. Use of this must be guarded by 5742 * disallowed_pkt_err(). 5743 */ 5744 static inline int disallowed_pkt_engine(int posn) 5745 { 5746 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET); 5747 } 5748 5749 /* 5750 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot 5751 * be done. 5752 */ 5753 static int engine_to_vl(struct hfi1_devdata *dd, int engine) 5754 { 5755 struct sdma_vl_map *m; 5756 int vl; 5757 5758 /* range check */ 5759 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES) 5760 return -1; 5761 5762 rcu_read_lock(); 5763 m = rcu_dereference(dd->sdma_map); 5764 vl = m->engine_to_vl[engine]; 5765 rcu_read_unlock(); 5766 5767 return vl; 5768 } 5769 5770 /* 5771 * Translate the send context (sofware index) into a VL. Return -1 if the 5772 * translation cannot be done. 5773 */ 5774 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index) 5775 { 5776 struct send_context_info *sci; 5777 struct send_context *sc; 5778 int i; 5779 5780 sci = &dd->send_contexts[sw_index]; 5781 5782 /* there is no information for user (PSM) and ack contexts */ 5783 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15)) 5784 return -1; 5785 5786 sc = sci->sc; 5787 if (!sc) 5788 return -1; 5789 if (dd->vld[15].sc == sc) 5790 return 15; 5791 for (i = 0; i < num_vls; i++) 5792 if (dd->vld[i].sc == sc) 5793 return i; 5794 5795 return -1; 5796 } 5797 5798 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5799 { 5800 u64 reg_copy = reg, handled = 0; 5801 char buf[96]; 5802 int i = 0; 5803 5804 if (reg & ALL_TXE_EGRESS_FREEZE_ERR) 5805 start_freeze_handling(dd->pport, 0); 5806 else if (is_ax(dd) && 5807 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) && 5808 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) 5809 start_freeze_handling(dd->pport, 0); 5810 5811 while (reg_copy) { 5812 int posn = fls64(reg_copy); 5813 /* fls64() returns a 1-based offset, we want it zero based */ 5814 int shift = posn - 1; 5815 u64 mask = 1ULL << shift; 5816 5817 if (port_inactive_err(shift)) { 5818 count_port_inactive(dd); 5819 handled |= mask; 5820 } else if (disallowed_pkt_err(shift)) { 5821 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift)); 5822 5823 handle_send_egress_err_info(dd, vl); 5824 handled |= mask; 5825 } 5826 reg_copy &= ~mask; 5827 } 5828 5829 reg &= ~handled; 5830 5831 if (reg) 5832 dd_dev_info(dd, "Egress Error: %s\n", 5833 egress_err_status_string(buf, sizeof(buf), reg)); 5834 5835 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) { 5836 if (reg & (1ull << i)) 5837 incr_cntr64(&dd->send_egress_err_status_cnt[i]); 5838 } 5839 } 5840 5841 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5842 { 5843 char buf[96]; 5844 int i = 0; 5845 5846 dd_dev_info(dd, "Send Error: %s\n", 5847 send_err_status_string(buf, sizeof(buf), reg)); 5848 5849 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) { 5850 if (reg & (1ull << i)) 5851 incr_cntr64(&dd->send_err_status_cnt[i]); 5852 } 5853 } 5854 5855 /* 5856 * The maximum number of times the error clear down will loop before 5857 * blocking a repeating error. This value is arbitrary. 5858 */ 5859 #define MAX_CLEAR_COUNT 20 5860 5861 /* 5862 * Clear and handle an error register. All error interrupts are funneled 5863 * through here to have a central location to correctly handle single- 5864 * or multi-shot errors. 5865 * 5866 * For non per-context registers, call this routine with a context value 5867 * of 0 so the per-context offset is zero. 5868 * 5869 * If the handler loops too many times, assume that something is wrong 5870 * and can't be fixed, so mask the error bits. 5871 */ 5872 static void interrupt_clear_down(struct hfi1_devdata *dd, 5873 u32 context, 5874 const struct err_reg_info *eri) 5875 { 5876 u64 reg; 5877 u32 count; 5878 5879 /* read in a loop until no more errors are seen */ 5880 count = 0; 5881 while (1) { 5882 reg = read_kctxt_csr(dd, context, eri->status); 5883 if (reg == 0) 5884 break; 5885 write_kctxt_csr(dd, context, eri->clear, reg); 5886 if (likely(eri->handler)) 5887 eri->handler(dd, context, reg); 5888 count++; 5889 if (count > MAX_CLEAR_COUNT) { 5890 u64 mask; 5891 5892 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n", 5893 eri->desc, reg); 5894 /* 5895 * Read-modify-write so any other masked bits 5896 * remain masked. 5897 */ 5898 mask = read_kctxt_csr(dd, context, eri->mask); 5899 mask &= ~reg; 5900 write_kctxt_csr(dd, context, eri->mask, mask); 5901 break; 5902 } 5903 } 5904 } 5905 5906 /* 5907 * CCE block "misc" interrupt. Source is < 16. 5908 */ 5909 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source) 5910 { 5911 const struct err_reg_info *eri = &misc_errs[source]; 5912 5913 if (eri->handler) { 5914 interrupt_clear_down(dd, 0, eri); 5915 } else { 5916 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n", 5917 source); 5918 } 5919 } 5920 5921 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags) 5922 { 5923 return flag_string(buf, buf_len, flags, 5924 sc_err_status_flags, 5925 ARRAY_SIZE(sc_err_status_flags)); 5926 } 5927 5928 /* 5929 * Send context error interrupt. Source (hw_context) is < 160. 5930 * 5931 * All send context errors cause the send context to halt. The normal 5932 * clear-down mechanism cannot be used because we cannot clear the 5933 * error bits until several other long-running items are done first. 5934 * This is OK because with the context halted, nothing else is going 5935 * to happen on it anyway. 5936 */ 5937 static void is_sendctxt_err_int(struct hfi1_devdata *dd, 5938 unsigned int hw_context) 5939 { 5940 struct send_context_info *sci; 5941 struct send_context *sc; 5942 char flags[96]; 5943 u64 status; 5944 u32 sw_index; 5945 int i = 0; 5946 5947 sw_index = dd->hw_to_sw[hw_context]; 5948 if (sw_index >= dd->num_send_contexts) { 5949 dd_dev_err(dd, 5950 "out of range sw index %u for send context %u\n", 5951 sw_index, hw_context); 5952 return; 5953 } 5954 sci = &dd->send_contexts[sw_index]; 5955 sc = sci->sc; 5956 if (!sc) { 5957 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__, 5958 sw_index, hw_context); 5959 return; 5960 } 5961 5962 /* tell the software that a halt has begun */ 5963 sc_stop(sc, SCF_HALTED); 5964 5965 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS); 5966 5967 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context, 5968 send_context_err_status_string(flags, sizeof(flags), 5969 status)); 5970 5971 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK) 5972 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index)); 5973 5974 /* 5975 * Automatically restart halted kernel contexts out of interrupt 5976 * context. User contexts must ask the driver to restart the context. 5977 */ 5978 if (sc->type != SC_USER) 5979 queue_work(dd->pport->hfi1_wq, &sc->halt_work); 5980 5981 /* 5982 * Update the counters for the corresponding status bits. 5983 * Note that these particular counters are aggregated over all 5984 * 160 contexts. 5985 */ 5986 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) { 5987 if (status & (1ull << i)) 5988 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]); 5989 } 5990 } 5991 5992 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 5993 unsigned int source, u64 status) 5994 { 5995 struct sdma_engine *sde; 5996 int i = 0; 5997 5998 sde = &dd->per_sdma[source]; 5999 #ifdef CONFIG_SDMA_VERBOSITY 6000 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 6001 slashstrip(__FILE__), __LINE__, __func__); 6002 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n", 6003 sde->this_idx, source, (unsigned long long)status); 6004 #endif 6005 sde->err_cnt++; 6006 sdma_engine_error(sde, status); 6007 6008 /* 6009 * Update the counters for the corresponding status bits. 6010 * Note that these particular counters are aggregated over 6011 * all 16 DMA engines. 6012 */ 6013 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) { 6014 if (status & (1ull << i)) 6015 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]); 6016 } 6017 } 6018 6019 /* 6020 * CCE block SDMA error interrupt. Source is < 16. 6021 */ 6022 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source) 6023 { 6024 #ifdef CONFIG_SDMA_VERBOSITY 6025 struct sdma_engine *sde = &dd->per_sdma[source]; 6026 6027 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 6028 slashstrip(__FILE__), __LINE__, __func__); 6029 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx, 6030 source); 6031 sdma_dumpstate(sde); 6032 #endif 6033 interrupt_clear_down(dd, source, &sdma_eng_err); 6034 } 6035 6036 /* 6037 * CCE block "various" interrupt. Source is < 8. 6038 */ 6039 static void is_various_int(struct hfi1_devdata *dd, unsigned int source) 6040 { 6041 const struct err_reg_info *eri = &various_err[source]; 6042 6043 /* 6044 * TCritInt cannot go through interrupt_clear_down() 6045 * because it is not a second tier interrupt. The handler 6046 * should be called directly. 6047 */ 6048 if (source == TCRIT_INT_SOURCE) 6049 handle_temp_err(dd); 6050 else if (eri->handler) 6051 interrupt_clear_down(dd, 0, eri); 6052 else 6053 dd_dev_info(dd, 6054 "%s: Unimplemented/reserved interrupt %d\n", 6055 __func__, source); 6056 } 6057 6058 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg) 6059 { 6060 /* src_ctx is always zero */ 6061 struct hfi1_pportdata *ppd = dd->pport; 6062 unsigned long flags; 6063 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 6064 6065 if (reg & QSFP_HFI0_MODPRST_N) { 6066 if (!qsfp_mod_present(ppd)) { 6067 dd_dev_info(dd, "%s: QSFP module removed\n", 6068 __func__); 6069 6070 ppd->driver_link_ready = 0; 6071 /* 6072 * Cable removed, reset all our information about the 6073 * cache and cable capabilities 6074 */ 6075 6076 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6077 /* 6078 * We don't set cache_refresh_required here as we expect 6079 * an interrupt when a cable is inserted 6080 */ 6081 ppd->qsfp_info.cache_valid = 0; 6082 ppd->qsfp_info.reset_needed = 0; 6083 ppd->qsfp_info.limiting_active = 0; 6084 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6085 flags); 6086 /* Invert the ModPresent pin now to detect plug-in */ 6087 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6088 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6089 6090 if ((ppd->offline_disabled_reason > 6091 HFI1_ODR_MASK( 6092 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) || 6093 (ppd->offline_disabled_reason == 6094 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))) 6095 ppd->offline_disabled_reason = 6096 HFI1_ODR_MASK( 6097 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED); 6098 6099 if (ppd->host_link_state == HLS_DN_POLL) { 6100 /* 6101 * The link is still in POLL. This means 6102 * that the normal link down processing 6103 * will not happen. We have to do it here 6104 * before turning the DC off. 6105 */ 6106 queue_work(ppd->link_wq, &ppd->link_down_work); 6107 } 6108 } else { 6109 dd_dev_info(dd, "%s: QSFP module inserted\n", 6110 __func__); 6111 6112 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6113 ppd->qsfp_info.cache_valid = 0; 6114 ppd->qsfp_info.cache_refresh_required = 1; 6115 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6116 flags); 6117 6118 /* 6119 * Stop inversion of ModPresent pin to detect 6120 * removal of the cable 6121 */ 6122 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N; 6123 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6124 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6125 6126 ppd->offline_disabled_reason = 6127 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 6128 } 6129 } 6130 6131 if (reg & QSFP_HFI0_INT_N) { 6132 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n", 6133 __func__); 6134 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6135 ppd->qsfp_info.check_interrupt_flags = 1; 6136 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags); 6137 } 6138 6139 /* Schedule the QSFP work only if there is a cable attached. */ 6140 if (qsfp_mod_present(ppd)) 6141 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work); 6142 } 6143 6144 static int request_host_lcb_access(struct hfi1_devdata *dd) 6145 { 6146 int ret; 6147 6148 ret = do_8051_command(dd, HCMD_MISC, 6149 (u64)HCMD_MISC_REQUEST_LCB_ACCESS << 6150 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6151 if (ret != HCMD_SUCCESS) { 6152 dd_dev_err(dd, "%s: command failed with error %d\n", 6153 __func__, ret); 6154 } 6155 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6156 } 6157 6158 static int request_8051_lcb_access(struct hfi1_devdata *dd) 6159 { 6160 int ret; 6161 6162 ret = do_8051_command(dd, HCMD_MISC, 6163 (u64)HCMD_MISC_GRANT_LCB_ACCESS << 6164 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6165 if (ret != HCMD_SUCCESS) { 6166 dd_dev_err(dd, "%s: command failed with error %d\n", 6167 __func__, ret); 6168 } 6169 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6170 } 6171 6172 /* 6173 * Set the LCB selector - allow host access. The DCC selector always 6174 * points to the host. 6175 */ 6176 static inline void set_host_lcb_access(struct hfi1_devdata *dd) 6177 { 6178 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6179 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK | 6180 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK); 6181 } 6182 6183 /* 6184 * Clear the LCB selector - allow 8051 access. The DCC selector always 6185 * points to the host. 6186 */ 6187 static inline void set_8051_lcb_access(struct hfi1_devdata *dd) 6188 { 6189 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6190 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK); 6191 } 6192 6193 /* 6194 * Acquire LCB access from the 8051. If the host already has access, 6195 * just increment a counter. Otherwise, inform the 8051 that the 6196 * host is taking access. 6197 * 6198 * Returns: 6199 * 0 on success 6200 * -EBUSY if the 8051 has control and cannot be disturbed 6201 * -errno if unable to acquire access from the 8051 6202 */ 6203 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6204 { 6205 struct hfi1_pportdata *ppd = dd->pport; 6206 int ret = 0; 6207 6208 /* 6209 * Use the host link state lock so the operation of this routine 6210 * { link state check, selector change, count increment } can occur 6211 * as a unit against a link state change. Otherwise there is a 6212 * race between the state change and the count increment. 6213 */ 6214 if (sleep_ok) { 6215 mutex_lock(&ppd->hls_lock); 6216 } else { 6217 while (!mutex_trylock(&ppd->hls_lock)) 6218 udelay(1); 6219 } 6220 6221 /* this access is valid only when the link is up */ 6222 if (ppd->host_link_state & HLS_DOWN) { 6223 dd_dev_info(dd, "%s: link state %s not up\n", 6224 __func__, link_state_name(ppd->host_link_state)); 6225 ret = -EBUSY; 6226 goto done; 6227 } 6228 6229 if (dd->lcb_access_count == 0) { 6230 ret = request_host_lcb_access(dd); 6231 if (ret) { 6232 dd_dev_err(dd, 6233 "%s: unable to acquire LCB access, err %d\n", 6234 __func__, ret); 6235 goto done; 6236 } 6237 set_host_lcb_access(dd); 6238 } 6239 dd->lcb_access_count++; 6240 done: 6241 mutex_unlock(&ppd->hls_lock); 6242 return ret; 6243 } 6244 6245 /* 6246 * Release LCB access by decrementing the use count. If the count is moving 6247 * from 1 to 0, inform 8051 that it has control back. 6248 * 6249 * Returns: 6250 * 0 on success 6251 * -errno if unable to release access to the 8051 6252 */ 6253 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6254 { 6255 int ret = 0; 6256 6257 /* 6258 * Use the host link state lock because the acquire needed it. 6259 * Here, we only need to keep { selector change, count decrement } 6260 * as a unit. 6261 */ 6262 if (sleep_ok) { 6263 mutex_lock(&dd->pport->hls_lock); 6264 } else { 6265 while (!mutex_trylock(&dd->pport->hls_lock)) 6266 udelay(1); 6267 } 6268 6269 if (dd->lcb_access_count == 0) { 6270 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n", 6271 __func__); 6272 goto done; 6273 } 6274 6275 if (dd->lcb_access_count == 1) { 6276 set_8051_lcb_access(dd); 6277 ret = request_8051_lcb_access(dd); 6278 if (ret) { 6279 dd_dev_err(dd, 6280 "%s: unable to release LCB access, err %d\n", 6281 __func__, ret); 6282 /* restore host access if the grant didn't work */ 6283 set_host_lcb_access(dd); 6284 goto done; 6285 } 6286 } 6287 dd->lcb_access_count--; 6288 done: 6289 mutex_unlock(&dd->pport->hls_lock); 6290 return ret; 6291 } 6292 6293 /* 6294 * Initialize LCB access variables and state. Called during driver load, 6295 * after most of the initialization is finished. 6296 * 6297 * The DC default is LCB access on for the host. The driver defaults to 6298 * leaving access to the 8051. Assign access now - this constrains the call 6299 * to this routine to be after all LCB set-up is done. In particular, after 6300 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts() 6301 */ 6302 static void init_lcb_access(struct hfi1_devdata *dd) 6303 { 6304 dd->lcb_access_count = 0; 6305 } 6306 6307 /* 6308 * Write a response back to a 8051 request. 6309 */ 6310 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data) 6311 { 6312 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 6313 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK | 6314 (u64)return_code << 6315 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT | 6316 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 6317 } 6318 6319 /* 6320 * Handle host requests from the 8051. 6321 */ 6322 static void handle_8051_request(struct hfi1_pportdata *ppd) 6323 { 6324 struct hfi1_devdata *dd = ppd->dd; 6325 u64 reg; 6326 u16 data = 0; 6327 u8 type; 6328 6329 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1); 6330 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0) 6331 return; /* no request */ 6332 6333 /* zero out COMPLETED so the response is seen */ 6334 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0); 6335 6336 /* extract request details */ 6337 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT) 6338 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK; 6339 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT) 6340 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK; 6341 6342 switch (type) { 6343 case HREQ_LOAD_CONFIG: 6344 case HREQ_SAVE_CONFIG: 6345 case HREQ_READ_CONFIG: 6346 case HREQ_SET_TX_EQ_ABS: 6347 case HREQ_SET_TX_EQ_REL: 6348 case HREQ_ENABLE: 6349 dd_dev_info(dd, "8051 request: request 0x%x not supported\n", 6350 type); 6351 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6352 break; 6353 case HREQ_CONFIG_DONE: 6354 hreq_response(dd, HREQ_SUCCESS, 0); 6355 break; 6356 6357 case HREQ_INTERFACE_TEST: 6358 hreq_response(dd, HREQ_SUCCESS, data); 6359 break; 6360 default: 6361 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type); 6362 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6363 break; 6364 } 6365 } 6366 6367 /* 6368 * Set up allocation unit vaulue. 6369 */ 6370 void set_up_vau(struct hfi1_devdata *dd, u8 vau) 6371 { 6372 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 6373 6374 /* do not modify other values in the register */ 6375 reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK; 6376 reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT; 6377 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 6378 } 6379 6380 /* 6381 * Set up initial VL15 credits of the remote. Assumes the rest of 6382 * the CM credit registers are zero from a previous global or credit reset. 6383 * Shared limit for VL15 will always be 0. 6384 */ 6385 void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf) 6386 { 6387 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 6388 6389 /* set initial values for total and shared credit limit */ 6390 reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK | 6391 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK); 6392 6393 /* 6394 * Set total limit to be equal to VL15 credits. 6395 * Leave shared limit at 0. 6396 */ 6397 reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 6398 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 6399 6400 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf 6401 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT); 6402 } 6403 6404 /* 6405 * Zero all credit details from the previous connection and 6406 * reset the CM manager's internal counters. 6407 */ 6408 void reset_link_credits(struct hfi1_devdata *dd) 6409 { 6410 int i; 6411 6412 /* remove all previous VL credit limits */ 6413 for (i = 0; i < TXE_NUM_DATA_VL; i++) 6414 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 6415 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 6416 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0); 6417 /* reset the CM block */ 6418 pio_send_control(dd, PSC_CM_RESET); 6419 /* reset cached value */ 6420 dd->vl15buf_cached = 0; 6421 } 6422 6423 /* convert a vCU to a CU */ 6424 static u32 vcu_to_cu(u8 vcu) 6425 { 6426 return 1 << vcu; 6427 } 6428 6429 /* convert a CU to a vCU */ 6430 static u8 cu_to_vcu(u32 cu) 6431 { 6432 return ilog2(cu); 6433 } 6434 6435 /* convert a vAU to an AU */ 6436 static u32 vau_to_au(u8 vau) 6437 { 6438 return 8 * (1 << vau); 6439 } 6440 6441 static void set_linkup_defaults(struct hfi1_pportdata *ppd) 6442 { 6443 ppd->sm_trap_qp = 0x0; 6444 ppd->sa_qp = 0x1; 6445 } 6446 6447 /* 6448 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset. 6449 */ 6450 static void lcb_shutdown(struct hfi1_devdata *dd, int abort) 6451 { 6452 u64 reg; 6453 6454 /* clear lcb run: LCB_CFG_RUN.EN = 0 */ 6455 write_csr(dd, DC_LCB_CFG_RUN, 0); 6456 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */ 6457 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 6458 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT); 6459 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */ 6460 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN); 6461 reg = read_csr(dd, DCC_CFG_RESET); 6462 write_csr(dd, DCC_CFG_RESET, reg | 6463 (1ull << DCC_CFG_RESET_RESET_LCB_SHIFT) | 6464 (1ull << DCC_CFG_RESET_RESET_RX_FPE_SHIFT)); 6465 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */ 6466 if (!abort) { 6467 udelay(1); /* must hold for the longer of 16cclks or 20ns */ 6468 write_csr(dd, DCC_CFG_RESET, reg); 6469 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6470 } 6471 } 6472 6473 /* 6474 * This routine should be called after the link has been transitioned to 6475 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into 6476 * reset). 6477 * 6478 * The expectation is that the caller of this routine would have taken 6479 * care of properly transitioning the link into the correct state. 6480 * NOTE: the caller needs to acquire the dd->dc8051_lock lock 6481 * before calling this function. 6482 */ 6483 static void _dc_shutdown(struct hfi1_devdata *dd) 6484 { 6485 lockdep_assert_held(&dd->dc8051_lock); 6486 6487 if (dd->dc_shutdown) 6488 return; 6489 6490 dd->dc_shutdown = 1; 6491 /* Shutdown the LCB */ 6492 lcb_shutdown(dd, 1); 6493 /* 6494 * Going to OFFLINE would have causes the 8051 to put the 6495 * SerDes into reset already. Just need to shut down the 8051, 6496 * itself. 6497 */ 6498 write_csr(dd, DC_DC8051_CFG_RST, 0x1); 6499 } 6500 6501 static void dc_shutdown(struct hfi1_devdata *dd) 6502 { 6503 mutex_lock(&dd->dc8051_lock); 6504 _dc_shutdown(dd); 6505 mutex_unlock(&dd->dc8051_lock); 6506 } 6507 6508 /* 6509 * Calling this after the DC has been brought out of reset should not 6510 * do any damage. 6511 * NOTE: the caller needs to acquire the dd->dc8051_lock lock 6512 * before calling this function. 6513 */ 6514 static void _dc_start(struct hfi1_devdata *dd) 6515 { 6516 lockdep_assert_held(&dd->dc8051_lock); 6517 6518 if (!dd->dc_shutdown) 6519 return; 6520 6521 /* 6522 * Take the 8051 out of reset, wait until 8051 is ready, and set host 6523 * version bit. 6524 */ 6525 release_and_wait_ready_8051_firmware(dd); 6526 6527 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */ 6528 write_csr(dd, DCC_CFG_RESET, 0x10); 6529 /* lcb_shutdown() with abort=1 does not restore these */ 6530 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6531 dd->dc_shutdown = 0; 6532 } 6533 6534 static void dc_start(struct hfi1_devdata *dd) 6535 { 6536 mutex_lock(&dd->dc8051_lock); 6537 _dc_start(dd); 6538 mutex_unlock(&dd->dc8051_lock); 6539 } 6540 6541 /* 6542 * These LCB adjustments are for the Aurora SerDes core in the FPGA. 6543 */ 6544 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd) 6545 { 6546 u64 rx_radr, tx_radr; 6547 u32 version; 6548 6549 if (dd->icode != ICODE_FPGA_EMULATION) 6550 return; 6551 6552 /* 6553 * These LCB defaults on emulator _s are good, nothing to do here: 6554 * LCB_CFG_TX_FIFOS_RADR 6555 * LCB_CFG_RX_FIFOS_RADR 6556 * LCB_CFG_LN_DCLK 6557 * LCB_CFG_IGNORE_LOST_RCLK 6558 */ 6559 if (is_emulator_s(dd)) 6560 return; 6561 /* else this is _p */ 6562 6563 version = emulator_rev(dd); 6564 if (!is_ax(dd)) 6565 version = 0x2d; /* all B0 use 0x2d or higher settings */ 6566 6567 if (version <= 0x12) { 6568 /* release 0x12 and below */ 6569 6570 /* 6571 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9 6572 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9 6573 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa 6574 */ 6575 rx_radr = 6576 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6577 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6578 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6579 /* 6580 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default) 6581 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6 6582 */ 6583 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6584 } else if (version <= 0x18) { 6585 /* release 0x13 up to 0x18 */ 6586 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6587 rx_radr = 6588 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6589 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6590 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6591 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6592 } else if (version == 0x19) { 6593 /* release 0x19 */ 6594 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */ 6595 rx_radr = 6596 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6597 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6598 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6599 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6600 } else if (version == 0x1a) { 6601 /* release 0x1a */ 6602 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6603 rx_radr = 6604 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6605 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6606 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6607 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6608 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull); 6609 } else { 6610 /* release 0x1b and higher */ 6611 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */ 6612 rx_radr = 6613 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6614 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6615 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6616 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6617 } 6618 6619 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr); 6620 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */ 6621 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 6622 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 6623 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr); 6624 } 6625 6626 /* 6627 * Handle a SMA idle message 6628 * 6629 * This is a work-queue function outside of the interrupt. 6630 */ 6631 void handle_sma_message(struct work_struct *work) 6632 { 6633 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6634 sma_message_work); 6635 struct hfi1_devdata *dd = ppd->dd; 6636 u64 msg; 6637 int ret; 6638 6639 /* 6640 * msg is bytes 1-4 of the 40-bit idle message - the command code 6641 * is stripped off 6642 */ 6643 ret = read_idle_sma(dd, &msg); 6644 if (ret) 6645 return; 6646 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg); 6647 /* 6648 * React to the SMA message. Byte[1] (0 for us) is the command. 6649 */ 6650 switch (msg & 0xff) { 6651 case SMA_IDLE_ARM: 6652 /* 6653 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6654 * State Transitions 6655 * 6656 * Only expected in INIT or ARMED, discard otherwise. 6657 */ 6658 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED)) 6659 ppd->neighbor_normal = 1; 6660 break; 6661 case SMA_IDLE_ACTIVE: 6662 /* 6663 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6664 * State Transitions 6665 * 6666 * Can activate the node. Discard otherwise. 6667 */ 6668 if (ppd->host_link_state == HLS_UP_ARMED && 6669 ppd->is_active_optimize_enabled) { 6670 ppd->neighbor_normal = 1; 6671 ret = set_link_state(ppd, HLS_UP_ACTIVE); 6672 if (ret) 6673 dd_dev_err( 6674 dd, 6675 "%s: received Active SMA idle message, couldn't set link to Active\n", 6676 __func__); 6677 } 6678 break; 6679 default: 6680 dd_dev_err(dd, 6681 "%s: received unexpected SMA idle message 0x%llx\n", 6682 __func__, msg); 6683 break; 6684 } 6685 } 6686 6687 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear) 6688 { 6689 u64 rcvctrl; 6690 unsigned long flags; 6691 6692 spin_lock_irqsave(&dd->rcvctrl_lock, flags); 6693 rcvctrl = read_csr(dd, RCV_CTRL); 6694 rcvctrl |= add; 6695 rcvctrl &= ~clear; 6696 write_csr(dd, RCV_CTRL, rcvctrl); 6697 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags); 6698 } 6699 6700 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add) 6701 { 6702 adjust_rcvctrl(dd, add, 0); 6703 } 6704 6705 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear) 6706 { 6707 adjust_rcvctrl(dd, 0, clear); 6708 } 6709 6710 /* 6711 * Called from all interrupt handlers to start handling an SPC freeze. 6712 */ 6713 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags) 6714 { 6715 struct hfi1_devdata *dd = ppd->dd; 6716 struct send_context *sc; 6717 int i; 6718 6719 if (flags & FREEZE_SELF) 6720 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6721 6722 /* enter frozen mode */ 6723 dd->flags |= HFI1_FROZEN; 6724 6725 /* notify all SDMA engines that they are going into a freeze */ 6726 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN)); 6727 6728 /* do halt pre-handling on all enabled send contexts */ 6729 for (i = 0; i < dd->num_send_contexts; i++) { 6730 sc = dd->send_contexts[i].sc; 6731 if (sc && (sc->flags & SCF_ENABLED)) 6732 sc_stop(sc, SCF_FROZEN | SCF_HALTED); 6733 } 6734 6735 /* Send context are frozen. Notify user space */ 6736 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT); 6737 6738 if (flags & FREEZE_ABORT) { 6739 dd_dev_err(dd, 6740 "Aborted freeze recovery. Please REBOOT system\n"); 6741 return; 6742 } 6743 /* queue non-interrupt handler */ 6744 queue_work(ppd->hfi1_wq, &ppd->freeze_work); 6745 } 6746 6747 /* 6748 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen, 6749 * depending on the "freeze" parameter. 6750 * 6751 * No need to return an error if it times out, our only option 6752 * is to proceed anyway. 6753 */ 6754 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze) 6755 { 6756 unsigned long timeout; 6757 u64 reg; 6758 6759 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT); 6760 while (1) { 6761 reg = read_csr(dd, CCE_STATUS); 6762 if (freeze) { 6763 /* waiting until all indicators are set */ 6764 if ((reg & ALL_FROZE) == ALL_FROZE) 6765 return; /* all done */ 6766 } else { 6767 /* waiting until all indicators are clear */ 6768 if ((reg & ALL_FROZE) == 0) 6769 return; /* all done */ 6770 } 6771 6772 if (time_after(jiffies, timeout)) { 6773 dd_dev_err(dd, 6774 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing", 6775 freeze ? "" : "un", reg & ALL_FROZE, 6776 freeze ? ALL_FROZE : 0ull); 6777 return; 6778 } 6779 usleep_range(80, 120); 6780 } 6781 } 6782 6783 /* 6784 * Do all freeze handling for the RXE block. 6785 */ 6786 static void rxe_freeze(struct hfi1_devdata *dd) 6787 { 6788 int i; 6789 struct hfi1_ctxtdata *rcd; 6790 6791 /* disable port */ 6792 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6793 6794 /* disable all receive contexts */ 6795 for (i = 0; i < dd->num_rcv_contexts; i++) { 6796 rcd = hfi1_rcd_get_by_index(dd, i); 6797 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd); 6798 hfi1_rcd_put(rcd); 6799 } 6800 } 6801 6802 /* 6803 * Unfreeze handling for the RXE block - kernel contexts only. 6804 * This will also enable the port. User contexts will do unfreeze 6805 * handling on a per-context basis as they call into the driver. 6806 * 6807 */ 6808 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd) 6809 { 6810 u32 rcvmask; 6811 u16 i; 6812 struct hfi1_ctxtdata *rcd; 6813 6814 /* enable all kernel contexts */ 6815 for (i = 0; i < dd->num_rcv_contexts; i++) { 6816 rcd = hfi1_rcd_get_by_index(dd, i); 6817 6818 /* Ensure all non-user contexts(including vnic) are enabled */ 6819 if (!rcd || 6820 (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) { 6821 hfi1_rcd_put(rcd); 6822 continue; 6823 } 6824 rcvmask = HFI1_RCVCTRL_CTXT_ENB; 6825 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */ 6826 rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ? 6827 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS; 6828 hfi1_rcvctrl(dd, rcvmask, rcd); 6829 hfi1_rcd_put(rcd); 6830 } 6831 6832 /* enable port */ 6833 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6834 } 6835 6836 /* 6837 * Non-interrupt SPC freeze handling. 6838 * 6839 * This is a work-queue function outside of the triggering interrupt. 6840 */ 6841 void handle_freeze(struct work_struct *work) 6842 { 6843 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6844 freeze_work); 6845 struct hfi1_devdata *dd = ppd->dd; 6846 6847 /* wait for freeze indicators on all affected blocks */ 6848 wait_for_freeze_status(dd, 1); 6849 6850 /* SPC is now frozen */ 6851 6852 /* do send PIO freeze steps */ 6853 pio_freeze(dd); 6854 6855 /* do send DMA freeze steps */ 6856 sdma_freeze(dd); 6857 6858 /* do send egress freeze steps - nothing to do */ 6859 6860 /* do receive freeze steps */ 6861 rxe_freeze(dd); 6862 6863 /* 6864 * Unfreeze the hardware - clear the freeze, wait for each 6865 * block's frozen bit to clear, then clear the frozen flag. 6866 */ 6867 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6868 wait_for_freeze_status(dd, 0); 6869 6870 if (is_ax(dd)) { 6871 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6872 wait_for_freeze_status(dd, 1); 6873 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6874 wait_for_freeze_status(dd, 0); 6875 } 6876 6877 /* do send PIO unfreeze steps for kernel contexts */ 6878 pio_kernel_unfreeze(dd); 6879 6880 /* do send DMA unfreeze steps */ 6881 sdma_unfreeze(dd); 6882 6883 /* do send egress unfreeze steps - nothing to do */ 6884 6885 /* do receive unfreeze steps for kernel contexts */ 6886 rxe_kernel_unfreeze(dd); 6887 6888 /* 6889 * The unfreeze procedure touches global device registers when 6890 * it disables and re-enables RXE. Mark the device unfrozen 6891 * after all that is done so other parts of the driver waiting 6892 * for the device to unfreeze don't do things out of order. 6893 * 6894 * The above implies that the meaning of HFI1_FROZEN flag is 6895 * "Device has gone into freeze mode and freeze mode handling 6896 * is still in progress." 6897 * 6898 * The flag will be removed when freeze mode processing has 6899 * completed. 6900 */ 6901 dd->flags &= ~HFI1_FROZEN; 6902 wake_up(&dd->event_queue); 6903 6904 /* no longer frozen */ 6905 } 6906 6907 /* 6908 * Handle a link up interrupt from the 8051. 6909 * 6910 * This is a work-queue function outside of the interrupt. 6911 */ 6912 void handle_link_up(struct work_struct *work) 6913 { 6914 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6915 link_up_work); 6916 struct hfi1_devdata *dd = ppd->dd; 6917 6918 set_link_state(ppd, HLS_UP_INIT); 6919 6920 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */ 6921 read_ltp_rtt(dd); 6922 /* 6923 * OPA specifies that certain counters are cleared on a transition 6924 * to link up, so do that. 6925 */ 6926 clear_linkup_counters(dd); 6927 /* 6928 * And (re)set link up default values. 6929 */ 6930 set_linkup_defaults(ppd); 6931 6932 /* 6933 * Set VL15 credits. Use cached value from verify cap interrupt. 6934 * In case of quick linkup or simulator, vl15 value will be set by 6935 * handle_linkup_change. VerifyCap interrupt handler will not be 6936 * called in those scenarios. 6937 */ 6938 if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) 6939 set_up_vl15(dd, dd->vl15buf_cached); 6940 6941 /* enforce link speed enabled */ 6942 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) { 6943 /* oops - current speed is not enabled, bounce */ 6944 dd_dev_err(dd, 6945 "Link speed active 0x%x is outside enabled 0x%x, downing link\n", 6946 ppd->link_speed_active, ppd->link_speed_enabled); 6947 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0, 6948 OPA_LINKDOWN_REASON_SPEED_POLICY); 6949 set_link_state(ppd, HLS_DN_OFFLINE); 6950 start_link(ppd); 6951 } 6952 } 6953 6954 /* 6955 * Several pieces of LNI information were cached for SMA in ppd. 6956 * Reset these on link down 6957 */ 6958 static void reset_neighbor_info(struct hfi1_pportdata *ppd) 6959 { 6960 ppd->neighbor_guid = 0; 6961 ppd->neighbor_port_number = 0; 6962 ppd->neighbor_type = 0; 6963 ppd->neighbor_fm_security = 0; 6964 } 6965 6966 static const char * const link_down_reason_strs[] = { 6967 [OPA_LINKDOWN_REASON_NONE] = "None", 6968 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0", 6969 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length", 6970 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long", 6971 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short", 6972 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID", 6973 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID", 6974 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2", 6975 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC", 6976 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8", 6977 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail", 6978 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10", 6979 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error", 6980 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15", 6981 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker", 6982 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14", 6983 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15", 6984 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance", 6985 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance", 6986 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance", 6987 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack", 6988 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker", 6989 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt", 6990 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit", 6991 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit", 6992 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24", 6993 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25", 6994 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26", 6995 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27", 6996 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28", 6997 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29", 6998 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30", 6999 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] = 7000 "Excessive buffer overrun", 7001 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown", 7002 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot", 7003 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown", 7004 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce", 7005 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy", 7006 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy", 7007 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected", 7008 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] = 7009 "Local media not installed", 7010 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed", 7011 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config", 7012 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] = 7013 "End to end not installed", 7014 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy", 7015 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy", 7016 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy", 7017 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management", 7018 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled", 7019 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient" 7020 }; 7021 7022 /* return the neighbor link down reason string */ 7023 static const char *link_down_reason_str(u8 reason) 7024 { 7025 const char *str = NULL; 7026 7027 if (reason < ARRAY_SIZE(link_down_reason_strs)) 7028 str = link_down_reason_strs[reason]; 7029 if (!str) 7030 str = "(invalid)"; 7031 7032 return str; 7033 } 7034 7035 /* 7036 * Handle a link down interrupt from the 8051. 7037 * 7038 * This is a work-queue function outside of the interrupt. 7039 */ 7040 void handle_link_down(struct work_struct *work) 7041 { 7042 u8 lcl_reason, neigh_reason = 0; 7043 u8 link_down_reason; 7044 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7045 link_down_work); 7046 int was_up; 7047 static const char ldr_str[] = "Link down reason: "; 7048 7049 if ((ppd->host_link_state & 7050 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) && 7051 ppd->port_type == PORT_TYPE_FIXED) 7052 ppd->offline_disabled_reason = 7053 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED); 7054 7055 /* Go offline first, then deal with reading/writing through 8051 */ 7056 was_up = !!(ppd->host_link_state & HLS_UP); 7057 set_link_state(ppd, HLS_DN_OFFLINE); 7058 xchg(&ppd->is_link_down_queued, 0); 7059 7060 if (was_up) { 7061 lcl_reason = 0; 7062 /* link down reason is only valid if the link was up */ 7063 read_link_down_reason(ppd->dd, &link_down_reason); 7064 switch (link_down_reason) { 7065 case LDR_LINK_TRANSFER_ACTIVE_LOW: 7066 /* the link went down, no idle message reason */ 7067 dd_dev_info(ppd->dd, "%sUnexpected link down\n", 7068 ldr_str); 7069 break; 7070 case LDR_RECEIVED_LINKDOWN_IDLE_MSG: 7071 /* 7072 * The neighbor reason is only valid if an idle message 7073 * was received for it. 7074 */ 7075 read_planned_down_reason_code(ppd->dd, &neigh_reason); 7076 dd_dev_info(ppd->dd, 7077 "%sNeighbor link down message %d, %s\n", 7078 ldr_str, neigh_reason, 7079 link_down_reason_str(neigh_reason)); 7080 break; 7081 case LDR_RECEIVED_HOST_OFFLINE_REQ: 7082 dd_dev_info(ppd->dd, 7083 "%sHost requested link to go offline\n", 7084 ldr_str); 7085 break; 7086 default: 7087 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n", 7088 ldr_str, link_down_reason); 7089 break; 7090 } 7091 7092 /* 7093 * If no reason, assume peer-initiated but missed 7094 * LinkGoingDown idle flits. 7095 */ 7096 if (neigh_reason == 0) 7097 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN; 7098 } else { 7099 /* went down while polling or going up */ 7100 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT; 7101 } 7102 7103 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0); 7104 7105 /* inform the SMA when the link transitions from up to down */ 7106 if (was_up && ppd->local_link_down_reason.sma == 0 && 7107 ppd->neigh_link_down_reason.sma == 0) { 7108 ppd->local_link_down_reason.sma = 7109 ppd->local_link_down_reason.latest; 7110 ppd->neigh_link_down_reason.sma = 7111 ppd->neigh_link_down_reason.latest; 7112 } 7113 7114 reset_neighbor_info(ppd); 7115 7116 /* disable the port */ 7117 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 7118 7119 /* 7120 * If there is no cable attached, turn the DC off. Otherwise, 7121 * start the link bring up. 7122 */ 7123 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) 7124 dc_shutdown(ppd->dd); 7125 else 7126 start_link(ppd); 7127 } 7128 7129 void handle_link_bounce(struct work_struct *work) 7130 { 7131 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7132 link_bounce_work); 7133 7134 /* 7135 * Only do something if the link is currently up. 7136 */ 7137 if (ppd->host_link_state & HLS_UP) { 7138 set_link_state(ppd, HLS_DN_OFFLINE); 7139 start_link(ppd); 7140 } else { 7141 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n", 7142 __func__, link_state_name(ppd->host_link_state)); 7143 } 7144 } 7145 7146 /* 7147 * Mask conversion: Capability exchange to Port LTP. The capability 7148 * exchange has an implicit 16b CRC that is mandatory. 7149 */ 7150 static int cap_to_port_ltp(int cap) 7151 { 7152 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */ 7153 7154 if (cap & CAP_CRC_14B) 7155 port_ltp |= PORT_LTP_CRC_MODE_14; 7156 if (cap & CAP_CRC_48B) 7157 port_ltp |= PORT_LTP_CRC_MODE_48; 7158 if (cap & CAP_CRC_12B_16B_PER_LANE) 7159 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE; 7160 7161 return port_ltp; 7162 } 7163 7164 /* 7165 * Convert an OPA Port LTP mask to capability mask 7166 */ 7167 int port_ltp_to_cap(int port_ltp) 7168 { 7169 int cap_mask = 0; 7170 7171 if (port_ltp & PORT_LTP_CRC_MODE_14) 7172 cap_mask |= CAP_CRC_14B; 7173 if (port_ltp & PORT_LTP_CRC_MODE_48) 7174 cap_mask |= CAP_CRC_48B; 7175 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE) 7176 cap_mask |= CAP_CRC_12B_16B_PER_LANE; 7177 7178 return cap_mask; 7179 } 7180 7181 /* 7182 * Convert a single DC LCB CRC mode to an OPA Port LTP mask. 7183 */ 7184 static int lcb_to_port_ltp(int lcb_crc) 7185 { 7186 int port_ltp = 0; 7187 7188 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE) 7189 port_ltp = PORT_LTP_CRC_MODE_PER_LANE; 7190 else if (lcb_crc == LCB_CRC_48B) 7191 port_ltp = PORT_LTP_CRC_MODE_48; 7192 else if (lcb_crc == LCB_CRC_14B) 7193 port_ltp = PORT_LTP_CRC_MODE_14; 7194 else 7195 port_ltp = PORT_LTP_CRC_MODE_16; 7196 7197 return port_ltp; 7198 } 7199 7200 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd) 7201 { 7202 if (ppd->pkeys[2] != 0) { 7203 ppd->pkeys[2] = 0; 7204 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); 7205 hfi1_event_pkey_change(ppd->dd, ppd->port); 7206 } 7207 } 7208 7209 /* 7210 * Convert the given link width to the OPA link width bitmask. 7211 */ 7212 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width) 7213 { 7214 switch (width) { 7215 case 0: 7216 /* 7217 * Simulator and quick linkup do not set the width. 7218 * Just set it to 4x without complaint. 7219 */ 7220 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup) 7221 return OPA_LINK_WIDTH_4X; 7222 return 0; /* no lanes up */ 7223 case 1: return OPA_LINK_WIDTH_1X; 7224 case 2: return OPA_LINK_WIDTH_2X; 7225 case 3: return OPA_LINK_WIDTH_3X; 7226 default: 7227 dd_dev_info(dd, "%s: invalid width %d, using 4\n", 7228 __func__, width); 7229 /* fall through */ 7230 case 4: return OPA_LINK_WIDTH_4X; 7231 } 7232 } 7233 7234 /* 7235 * Do a population count on the bottom nibble. 7236 */ 7237 static const u8 bit_counts[16] = { 7238 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 7239 }; 7240 7241 static inline u8 nibble_to_count(u8 nibble) 7242 { 7243 return bit_counts[nibble & 0xf]; 7244 } 7245 7246 /* 7247 * Read the active lane information from the 8051 registers and return 7248 * their widths. 7249 * 7250 * Active lane information is found in these 8051 registers: 7251 * enable_lane_tx 7252 * enable_lane_rx 7253 */ 7254 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width, 7255 u16 *rx_width) 7256 { 7257 u16 tx, rx; 7258 u8 enable_lane_rx; 7259 u8 enable_lane_tx; 7260 u8 tx_polarity_inversion; 7261 u8 rx_polarity_inversion; 7262 u8 max_rate; 7263 7264 /* read the active lanes */ 7265 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 7266 &rx_polarity_inversion, &max_rate); 7267 read_local_lni(dd, &enable_lane_rx); 7268 7269 /* convert to counts */ 7270 tx = nibble_to_count(enable_lane_tx); 7271 rx = nibble_to_count(enable_lane_rx); 7272 7273 /* 7274 * Set link_speed_active here, overriding what was set in 7275 * handle_verify_cap(). The ASIC 8051 firmware does not correctly 7276 * set the max_rate field in handle_verify_cap until v0.19. 7277 */ 7278 if ((dd->icode == ICODE_RTL_SILICON) && 7279 (dd->dc8051_ver < dc8051_ver(0, 19, 0))) { 7280 /* max_rate: 0 = 12.5G, 1 = 25G */ 7281 switch (max_rate) { 7282 case 0: 7283 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G; 7284 break; 7285 default: 7286 dd_dev_err(dd, 7287 "%s: unexpected max rate %d, using 25Gb\n", 7288 __func__, (int)max_rate); 7289 /* fall through */ 7290 case 1: 7291 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; 7292 break; 7293 } 7294 } 7295 7296 dd_dev_info(dd, 7297 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n", 7298 enable_lane_tx, tx, enable_lane_rx, rx); 7299 *tx_width = link_width_to_bits(dd, tx); 7300 *rx_width = link_width_to_bits(dd, rx); 7301 } 7302 7303 /* 7304 * Read verify_cap_local_fm_link_width[1] to obtain the link widths. 7305 * Valid after the end of VerifyCap and during LinkUp. Does not change 7306 * after link up. I.e. look elsewhere for downgrade information. 7307 * 7308 * Bits are: 7309 * + bits [7:4] contain the number of active transmitters 7310 * + bits [3:0] contain the number of active receivers 7311 * These are numbers 1 through 4 and can be different values if the 7312 * link is asymmetric. 7313 * 7314 * verify_cap_local_fm_link_width[0] retains its original value. 7315 */ 7316 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width, 7317 u16 *rx_width) 7318 { 7319 u16 widths, tx, rx; 7320 u8 misc_bits, local_flags; 7321 u16 active_tx, active_rx; 7322 7323 read_vc_local_link_width(dd, &misc_bits, &local_flags, &widths); 7324 tx = widths >> 12; 7325 rx = (widths >> 8) & 0xf; 7326 7327 *tx_width = link_width_to_bits(dd, tx); 7328 *rx_width = link_width_to_bits(dd, rx); 7329 7330 /* print the active widths */ 7331 get_link_widths(dd, &active_tx, &active_rx); 7332 } 7333 7334 /* 7335 * Set ppd->link_width_active and ppd->link_width_downgrade_active using 7336 * hardware information when the link first comes up. 7337 * 7338 * The link width is not available until after VerifyCap.AllFramesReceived 7339 * (the trigger for handle_verify_cap), so this is outside that routine 7340 * and should be called when the 8051 signals linkup. 7341 */ 7342 void get_linkup_link_widths(struct hfi1_pportdata *ppd) 7343 { 7344 u16 tx_width, rx_width; 7345 7346 /* get end-of-LNI link widths */ 7347 get_linkup_widths(ppd->dd, &tx_width, &rx_width); 7348 7349 /* use tx_width as the link is supposed to be symmetric on link up */ 7350 ppd->link_width_active = tx_width; 7351 /* link width downgrade active (LWD.A) starts out matching LW.A */ 7352 ppd->link_width_downgrade_tx_active = ppd->link_width_active; 7353 ppd->link_width_downgrade_rx_active = ppd->link_width_active; 7354 /* per OPA spec, on link up LWD.E resets to LWD.S */ 7355 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported; 7356 /* cache the active egress rate (units {10^6 bits/sec]) */ 7357 ppd->current_egress_rate = active_egress_rate(ppd); 7358 } 7359 7360 /* 7361 * Handle a verify capabilities interrupt from the 8051. 7362 * 7363 * This is a work-queue function outside of the interrupt. 7364 */ 7365 void handle_verify_cap(struct work_struct *work) 7366 { 7367 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7368 link_vc_work); 7369 struct hfi1_devdata *dd = ppd->dd; 7370 u64 reg; 7371 u8 power_management; 7372 u8 continuous; 7373 u8 vcu; 7374 u8 vau; 7375 u8 z; 7376 u16 vl15buf; 7377 u16 link_widths; 7378 u16 crc_mask; 7379 u16 crc_val; 7380 u16 device_id; 7381 u16 active_tx, active_rx; 7382 u8 partner_supported_crc; 7383 u8 remote_tx_rate; 7384 u8 device_rev; 7385 7386 set_link_state(ppd, HLS_VERIFY_CAP); 7387 7388 lcb_shutdown(dd, 0); 7389 adjust_lcb_for_fpga_serdes(dd); 7390 7391 read_vc_remote_phy(dd, &power_management, &continuous); 7392 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf, 7393 &partner_supported_crc); 7394 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths); 7395 read_remote_device_id(dd, &device_id, &device_rev); 7396 7397 /* print the active widths */ 7398 get_link_widths(dd, &active_tx, &active_rx); 7399 dd_dev_info(dd, 7400 "Peer PHY: power management 0x%x, continuous updates 0x%x\n", 7401 (int)power_management, (int)continuous); 7402 dd_dev_info(dd, 7403 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n", 7404 (int)vau, (int)z, (int)vcu, (int)vl15buf, 7405 (int)partner_supported_crc); 7406 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n", 7407 (u32)remote_tx_rate, (u32)link_widths); 7408 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n", 7409 (u32)device_id, (u32)device_rev); 7410 /* 7411 * The peer vAU value just read is the peer receiver value. HFI does 7412 * not support a transmit vAU of 0 (AU == 8). We advertised that 7413 * with Z=1 in the fabric capabilities sent to the peer. The peer 7414 * will see our Z=1, and, if it advertised a vAU of 0, will move its 7415 * receive to vAU of 1 (AU == 16). Do the same here. We do not care 7416 * about the peer Z value - our sent vAU is 3 (hardwired) and is not 7417 * subject to the Z value exception. 7418 */ 7419 if (vau == 0) 7420 vau = 1; 7421 set_up_vau(dd, vau); 7422 7423 /* 7424 * Set VL15 credits to 0 in global credit register. Cache remote VL15 7425 * credits value and wait for link-up interrupt ot set it. 7426 */ 7427 set_up_vl15(dd, 0); 7428 dd->vl15buf_cached = vl15buf; 7429 7430 /* set up the LCB CRC mode */ 7431 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc; 7432 7433 /* order is important: use the lowest bit in common */ 7434 if (crc_mask & CAP_CRC_14B) 7435 crc_val = LCB_CRC_14B; 7436 else if (crc_mask & CAP_CRC_48B) 7437 crc_val = LCB_CRC_48B; 7438 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE) 7439 crc_val = LCB_CRC_12B_16B_PER_LANE; 7440 else 7441 crc_val = LCB_CRC_16B; 7442 7443 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val); 7444 write_csr(dd, DC_LCB_CFG_CRC_MODE, 7445 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT); 7446 7447 /* set (14b only) or clear sideband credit */ 7448 reg = read_csr(dd, SEND_CM_CTRL); 7449 if (crc_val == LCB_CRC_14B && crc_14b_sideband) { 7450 write_csr(dd, SEND_CM_CTRL, 7451 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7452 } else { 7453 write_csr(dd, SEND_CM_CTRL, 7454 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7455 } 7456 7457 ppd->link_speed_active = 0; /* invalid value */ 7458 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) { 7459 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */ 7460 switch (remote_tx_rate) { 7461 case 0: 7462 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7463 break; 7464 case 1: 7465 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7466 break; 7467 } 7468 } else { 7469 /* actual rate is highest bit of the ANDed rates */ 7470 u8 rate = remote_tx_rate & ppd->local_tx_rate; 7471 7472 if (rate & 2) 7473 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7474 else if (rate & 1) 7475 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7476 } 7477 if (ppd->link_speed_active == 0) { 7478 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n", 7479 __func__, (int)remote_tx_rate); 7480 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7481 } 7482 7483 /* 7484 * Cache the values of the supported, enabled, and active 7485 * LTP CRC modes to return in 'portinfo' queries. But the bit 7486 * flags that are returned in the portinfo query differ from 7487 * what's in the link_crc_mask, crc_sizes, and crc_val 7488 * variables. Convert these here. 7489 */ 7490 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 7491 /* supported crc modes */ 7492 ppd->port_ltp_crc_mode |= 7493 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4; 7494 /* enabled crc modes */ 7495 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val); 7496 /* active crc mode */ 7497 7498 /* set up the remote credit return table */ 7499 assign_remote_cm_au_table(dd, vcu); 7500 7501 /* 7502 * The LCB is reset on entry to handle_verify_cap(), so this must 7503 * be applied on every link up. 7504 * 7505 * Adjust LCB error kill enable to kill the link if 7506 * these RBUF errors are seen: 7507 * REPLAY_BUF_MBE_SMASK 7508 * FLIT_INPUT_BUF_MBE_SMASK 7509 */ 7510 if (is_ax(dd)) { /* fixed in B0 */ 7511 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN); 7512 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK 7513 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK; 7514 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg); 7515 } 7516 7517 /* pull LCB fifos out of reset - all fifo clocks must be stable */ 7518 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 7519 7520 /* give 8051 access to the LCB CSRs */ 7521 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 7522 set_8051_lcb_access(dd); 7523 7524 /* tell the 8051 to go to LinkUp */ 7525 set_link_state(ppd, HLS_GOING_UP); 7526 } 7527 7528 /* 7529 * Apply the link width downgrade enabled policy against the current active 7530 * link widths. 7531 * 7532 * Called when the enabled policy changes or the active link widths change. 7533 */ 7534 void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths) 7535 { 7536 int do_bounce = 0; 7537 int tries; 7538 u16 lwde; 7539 u16 tx, rx; 7540 7541 /* use the hls lock to avoid a race with actual link up */ 7542 tries = 0; 7543 retry: 7544 mutex_lock(&ppd->hls_lock); 7545 /* only apply if the link is up */ 7546 if (ppd->host_link_state & HLS_DOWN) { 7547 /* still going up..wait and retry */ 7548 if (ppd->host_link_state & HLS_GOING_UP) { 7549 if (++tries < 1000) { 7550 mutex_unlock(&ppd->hls_lock); 7551 usleep_range(100, 120); /* arbitrary */ 7552 goto retry; 7553 } 7554 dd_dev_err(ppd->dd, 7555 "%s: giving up waiting for link state change\n", 7556 __func__); 7557 } 7558 goto done; 7559 } 7560 7561 lwde = ppd->link_width_downgrade_enabled; 7562 7563 if (refresh_widths) { 7564 get_link_widths(ppd->dd, &tx, &rx); 7565 ppd->link_width_downgrade_tx_active = tx; 7566 ppd->link_width_downgrade_rx_active = rx; 7567 } 7568 7569 if (ppd->link_width_downgrade_tx_active == 0 || 7570 ppd->link_width_downgrade_rx_active == 0) { 7571 /* the 8051 reported a dead link as a downgrade */ 7572 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n"); 7573 } else if (lwde == 0) { 7574 /* downgrade is disabled */ 7575 7576 /* bounce if not at starting active width */ 7577 if ((ppd->link_width_active != 7578 ppd->link_width_downgrade_tx_active) || 7579 (ppd->link_width_active != 7580 ppd->link_width_downgrade_rx_active)) { 7581 dd_dev_err(ppd->dd, 7582 "Link downgrade is disabled and link has downgraded, downing link\n"); 7583 dd_dev_err(ppd->dd, 7584 " original 0x%x, tx active 0x%x, rx active 0x%x\n", 7585 ppd->link_width_active, 7586 ppd->link_width_downgrade_tx_active, 7587 ppd->link_width_downgrade_rx_active); 7588 do_bounce = 1; 7589 } 7590 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 || 7591 (lwde & ppd->link_width_downgrade_rx_active) == 0) { 7592 /* Tx or Rx is outside the enabled policy */ 7593 dd_dev_err(ppd->dd, 7594 "Link is outside of downgrade allowed, downing link\n"); 7595 dd_dev_err(ppd->dd, 7596 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n", 7597 lwde, ppd->link_width_downgrade_tx_active, 7598 ppd->link_width_downgrade_rx_active); 7599 do_bounce = 1; 7600 } 7601 7602 done: 7603 mutex_unlock(&ppd->hls_lock); 7604 7605 if (do_bounce) { 7606 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0, 7607 OPA_LINKDOWN_REASON_WIDTH_POLICY); 7608 set_link_state(ppd, HLS_DN_OFFLINE); 7609 start_link(ppd); 7610 } 7611 } 7612 7613 /* 7614 * Handle a link downgrade interrupt from the 8051. 7615 * 7616 * This is a work-queue function outside of the interrupt. 7617 */ 7618 void handle_link_downgrade(struct work_struct *work) 7619 { 7620 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7621 link_downgrade_work); 7622 7623 dd_dev_info(ppd->dd, "8051: Link width downgrade\n"); 7624 apply_link_downgrade_policy(ppd, 1); 7625 } 7626 7627 static char *dcc_err_string(char *buf, int buf_len, u64 flags) 7628 { 7629 return flag_string(buf, buf_len, flags, dcc_err_flags, 7630 ARRAY_SIZE(dcc_err_flags)); 7631 } 7632 7633 static char *lcb_err_string(char *buf, int buf_len, u64 flags) 7634 { 7635 return flag_string(buf, buf_len, flags, lcb_err_flags, 7636 ARRAY_SIZE(lcb_err_flags)); 7637 } 7638 7639 static char *dc8051_err_string(char *buf, int buf_len, u64 flags) 7640 { 7641 return flag_string(buf, buf_len, flags, dc8051_err_flags, 7642 ARRAY_SIZE(dc8051_err_flags)); 7643 } 7644 7645 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags) 7646 { 7647 return flag_string(buf, buf_len, flags, dc8051_info_err_flags, 7648 ARRAY_SIZE(dc8051_info_err_flags)); 7649 } 7650 7651 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags) 7652 { 7653 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags, 7654 ARRAY_SIZE(dc8051_info_host_msg_flags)); 7655 } 7656 7657 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg) 7658 { 7659 struct hfi1_pportdata *ppd = dd->pport; 7660 u64 info, err, host_msg; 7661 int queue_link_down = 0; 7662 char buf[96]; 7663 7664 /* look at the flags */ 7665 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) { 7666 /* 8051 information set by firmware */ 7667 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */ 7668 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051); 7669 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT) 7670 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK; 7671 host_msg = (info >> 7672 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT) 7673 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK; 7674 7675 /* 7676 * Handle error flags. 7677 */ 7678 if (err & FAILED_LNI) { 7679 /* 7680 * LNI error indications are cleared by the 8051 7681 * only when starting polling. Only pay attention 7682 * to them when in the states that occur during 7683 * LNI. 7684 */ 7685 if (ppd->host_link_state 7686 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 7687 queue_link_down = 1; 7688 dd_dev_info(dd, "Link error: %s\n", 7689 dc8051_info_err_string(buf, 7690 sizeof(buf), 7691 err & 7692 FAILED_LNI)); 7693 } 7694 err &= ~(u64)FAILED_LNI; 7695 } 7696 /* unknown frames can happen durning LNI, just count */ 7697 if (err & UNKNOWN_FRAME) { 7698 ppd->unknown_frame_count++; 7699 err &= ~(u64)UNKNOWN_FRAME; 7700 } 7701 if (err) { 7702 /* report remaining errors, but do not do anything */ 7703 dd_dev_err(dd, "8051 info error: %s\n", 7704 dc8051_info_err_string(buf, sizeof(buf), 7705 err)); 7706 } 7707 7708 /* 7709 * Handle host message flags. 7710 */ 7711 if (host_msg & HOST_REQ_DONE) { 7712 /* 7713 * Presently, the driver does a busy wait for 7714 * host requests to complete. This is only an 7715 * informational message. 7716 * NOTE: The 8051 clears the host message 7717 * information *on the next 8051 command*. 7718 * Therefore, when linkup is achieved, 7719 * this flag will still be set. 7720 */ 7721 host_msg &= ~(u64)HOST_REQ_DONE; 7722 } 7723 if (host_msg & BC_SMA_MSG) { 7724 queue_work(ppd->link_wq, &ppd->sma_message_work); 7725 host_msg &= ~(u64)BC_SMA_MSG; 7726 } 7727 if (host_msg & LINKUP_ACHIEVED) { 7728 dd_dev_info(dd, "8051: Link up\n"); 7729 queue_work(ppd->link_wq, &ppd->link_up_work); 7730 host_msg &= ~(u64)LINKUP_ACHIEVED; 7731 } 7732 if (host_msg & EXT_DEVICE_CFG_REQ) { 7733 handle_8051_request(ppd); 7734 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ; 7735 } 7736 if (host_msg & VERIFY_CAP_FRAME) { 7737 queue_work(ppd->link_wq, &ppd->link_vc_work); 7738 host_msg &= ~(u64)VERIFY_CAP_FRAME; 7739 } 7740 if (host_msg & LINK_GOING_DOWN) { 7741 const char *extra = ""; 7742 /* no downgrade action needed if going down */ 7743 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7744 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7745 extra = " (ignoring downgrade)"; 7746 } 7747 dd_dev_info(dd, "8051: Link down%s\n", extra); 7748 queue_link_down = 1; 7749 host_msg &= ~(u64)LINK_GOING_DOWN; 7750 } 7751 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7752 queue_work(ppd->link_wq, &ppd->link_downgrade_work); 7753 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7754 } 7755 if (host_msg) { 7756 /* report remaining messages, but do not do anything */ 7757 dd_dev_info(dd, "8051 info host message: %s\n", 7758 dc8051_info_host_msg_string(buf, 7759 sizeof(buf), 7760 host_msg)); 7761 } 7762 7763 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK; 7764 } 7765 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) { 7766 /* 7767 * Lost the 8051 heartbeat. If this happens, we 7768 * receive constant interrupts about it. Disable 7769 * the interrupt after the first. 7770 */ 7771 dd_dev_err(dd, "Lost 8051 heartbeat\n"); 7772 write_csr(dd, DC_DC8051_ERR_EN, 7773 read_csr(dd, DC_DC8051_ERR_EN) & 7774 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK); 7775 7776 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK; 7777 } 7778 if (reg) { 7779 /* report the error, but do not do anything */ 7780 dd_dev_err(dd, "8051 error: %s\n", 7781 dc8051_err_string(buf, sizeof(buf), reg)); 7782 } 7783 7784 if (queue_link_down) { 7785 /* 7786 * if the link is already going down or disabled, do not 7787 * queue another. If there's a link down entry already 7788 * queued, don't queue another one. 7789 */ 7790 if ((ppd->host_link_state & 7791 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) || 7792 ppd->link_enabled == 0) { 7793 dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n", 7794 __func__, ppd->host_link_state, 7795 ppd->link_enabled); 7796 } else { 7797 if (xchg(&ppd->is_link_down_queued, 1) == 1) 7798 dd_dev_info(dd, 7799 "%s: link down request already queued\n", 7800 __func__); 7801 else 7802 queue_work(ppd->link_wq, &ppd->link_down_work); 7803 } 7804 } 7805 } 7806 7807 static const char * const fm_config_txt[] = { 7808 [0] = 7809 "BadHeadDist: Distance violation between two head flits", 7810 [1] = 7811 "BadTailDist: Distance violation between two tail flits", 7812 [2] = 7813 "BadCtrlDist: Distance violation between two credit control flits", 7814 [3] = 7815 "BadCrdAck: Credits return for unsupported VL", 7816 [4] = 7817 "UnsupportedVLMarker: Received VL Marker", 7818 [5] = 7819 "BadPreempt: Exceeded the preemption nesting level", 7820 [6] = 7821 "BadControlFlit: Received unsupported control flit", 7822 /* no 7 */ 7823 [8] = 7824 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL", 7825 }; 7826 7827 static const char * const port_rcv_txt[] = { 7828 [1] = 7829 "BadPktLen: Illegal PktLen", 7830 [2] = 7831 "PktLenTooLong: Packet longer than PktLen", 7832 [3] = 7833 "PktLenTooShort: Packet shorter than PktLen", 7834 [4] = 7835 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)", 7836 [5] = 7837 "BadDLID: Illegal DLID (0, doesn't match HFI)", 7838 [6] = 7839 "BadL2: Illegal L2 opcode", 7840 [7] = 7841 "BadSC: Unsupported SC", 7842 [9] = 7843 "BadRC: Illegal RC", 7844 [11] = 7845 "PreemptError: Preempting with same VL", 7846 [12] = 7847 "PreemptVL15: Preempting a VL15 packet", 7848 }; 7849 7850 #define OPA_LDR_FMCONFIG_OFFSET 16 7851 #define OPA_LDR_PORTRCV_OFFSET 0 7852 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 7853 { 7854 u64 info, hdr0, hdr1; 7855 const char *extra; 7856 char buf[96]; 7857 struct hfi1_pportdata *ppd = dd->pport; 7858 u8 lcl_reason = 0; 7859 int do_bounce = 0; 7860 7861 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) { 7862 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) { 7863 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE); 7864 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK; 7865 /* set status bit */ 7866 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK; 7867 } 7868 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK; 7869 } 7870 7871 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) { 7872 struct hfi1_pportdata *ppd = dd->pport; 7873 /* this counter saturates at (2^32) - 1 */ 7874 if (ppd->link_downed < (u32)UINT_MAX) 7875 ppd->link_downed++; 7876 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK; 7877 } 7878 7879 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) { 7880 u8 reason_valid = 1; 7881 7882 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG); 7883 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) { 7884 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK; 7885 /* set status bit */ 7886 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK; 7887 } 7888 switch (info) { 7889 case 0: 7890 case 1: 7891 case 2: 7892 case 3: 7893 case 4: 7894 case 5: 7895 case 6: 7896 extra = fm_config_txt[info]; 7897 break; 7898 case 8: 7899 extra = fm_config_txt[info]; 7900 if (ppd->port_error_action & 7901 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) { 7902 do_bounce = 1; 7903 /* 7904 * lcl_reason cannot be derived from info 7905 * for this error 7906 */ 7907 lcl_reason = 7908 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER; 7909 } 7910 break; 7911 default: 7912 reason_valid = 0; 7913 snprintf(buf, sizeof(buf), "reserved%lld", info); 7914 extra = buf; 7915 break; 7916 } 7917 7918 if (reason_valid && !do_bounce) { 7919 do_bounce = ppd->port_error_action & 7920 (1 << (OPA_LDR_FMCONFIG_OFFSET + info)); 7921 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST; 7922 } 7923 7924 /* just report this */ 7925 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n", 7926 extra); 7927 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK; 7928 } 7929 7930 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) { 7931 u8 reason_valid = 1; 7932 7933 info = read_csr(dd, DCC_ERR_INFO_PORTRCV); 7934 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0); 7935 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1); 7936 if (!(dd->err_info_rcvport.status_and_code & 7937 OPA_EI_STATUS_SMASK)) { 7938 dd->err_info_rcvport.status_and_code = 7939 info & OPA_EI_CODE_SMASK; 7940 /* set status bit */ 7941 dd->err_info_rcvport.status_and_code |= 7942 OPA_EI_STATUS_SMASK; 7943 /* 7944 * save first 2 flits in the packet that caused 7945 * the error 7946 */ 7947 dd->err_info_rcvport.packet_flit1 = hdr0; 7948 dd->err_info_rcvport.packet_flit2 = hdr1; 7949 } 7950 switch (info) { 7951 case 1: 7952 case 2: 7953 case 3: 7954 case 4: 7955 case 5: 7956 case 6: 7957 case 7: 7958 case 9: 7959 case 11: 7960 case 12: 7961 extra = port_rcv_txt[info]; 7962 break; 7963 default: 7964 reason_valid = 0; 7965 snprintf(buf, sizeof(buf), "reserved%lld", info); 7966 extra = buf; 7967 break; 7968 } 7969 7970 if (reason_valid && !do_bounce) { 7971 do_bounce = ppd->port_error_action & 7972 (1 << (OPA_LDR_PORTRCV_OFFSET + info)); 7973 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0; 7974 } 7975 7976 /* just report this */ 7977 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n" 7978 " hdr0 0x%llx, hdr1 0x%llx\n", 7979 extra, hdr0, hdr1); 7980 7981 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK; 7982 } 7983 7984 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) { 7985 /* informative only */ 7986 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n"); 7987 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK; 7988 } 7989 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) { 7990 /* informative only */ 7991 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n"); 7992 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK; 7993 } 7994 7995 if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev))) 7996 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK; 7997 7998 /* report any remaining errors */ 7999 if (reg) 8000 dd_dev_info_ratelimited(dd, "DCC Error: %s\n", 8001 dcc_err_string(buf, sizeof(buf), reg)); 8002 8003 if (lcl_reason == 0) 8004 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN; 8005 8006 if (do_bounce) { 8007 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n", 8008 __func__); 8009 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason); 8010 queue_work(ppd->link_wq, &ppd->link_bounce_work); 8011 } 8012 } 8013 8014 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 8015 { 8016 char buf[96]; 8017 8018 dd_dev_info(dd, "LCB Error: %s\n", 8019 lcb_err_string(buf, sizeof(buf), reg)); 8020 } 8021 8022 /* 8023 * CCE block DC interrupt. Source is < 8. 8024 */ 8025 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source) 8026 { 8027 const struct err_reg_info *eri = &dc_errs[source]; 8028 8029 if (eri->handler) { 8030 interrupt_clear_down(dd, 0, eri); 8031 } else if (source == 3 /* dc_lbm_int */) { 8032 /* 8033 * This indicates that a parity error has occurred on the 8034 * address/control lines presented to the LBM. The error 8035 * is a single pulse, there is no associated error flag, 8036 * and it is non-maskable. This is because if a parity 8037 * error occurs on the request the request is dropped. 8038 * This should never occur, but it is nice to know if it 8039 * ever does. 8040 */ 8041 dd_dev_err(dd, "Parity error in DC LBM block\n"); 8042 } else { 8043 dd_dev_err(dd, "Invalid DC interrupt %u\n", source); 8044 } 8045 } 8046 8047 /* 8048 * TX block send credit interrupt. Source is < 160. 8049 */ 8050 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source) 8051 { 8052 sc_group_release_update(dd, source); 8053 } 8054 8055 /* 8056 * TX block SDMA interrupt. Source is < 48. 8057 * 8058 * SDMA interrupts are grouped by type: 8059 * 8060 * 0 - N-1 = SDma 8061 * N - 2N-1 = SDmaProgress 8062 * 2N - 3N-1 = SDmaIdle 8063 */ 8064 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source) 8065 { 8066 /* what interrupt */ 8067 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 8068 /* which engine */ 8069 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 8070 8071 #ifdef CONFIG_SDMA_VERBOSITY 8072 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which, 8073 slashstrip(__FILE__), __LINE__, __func__); 8074 sdma_dumpstate(&dd->per_sdma[which]); 8075 #endif 8076 8077 if (likely(what < 3 && which < dd->num_sdma)) { 8078 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source); 8079 } else { 8080 /* should not happen */ 8081 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source); 8082 } 8083 } 8084 8085 /* 8086 * RX block receive available interrupt. Source is < 160. 8087 */ 8088 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source) 8089 { 8090 struct hfi1_ctxtdata *rcd; 8091 char *err_detail; 8092 8093 if (likely(source < dd->num_rcv_contexts)) { 8094 rcd = hfi1_rcd_get_by_index(dd, source); 8095 if (rcd) { 8096 /* Check for non-user contexts, including vnic */ 8097 if (source < dd->first_dyn_alloc_ctxt || rcd->is_vnic) 8098 rcd->do_interrupt(rcd, 0); 8099 else 8100 handle_user_interrupt(rcd); 8101 8102 hfi1_rcd_put(rcd); 8103 return; /* OK */ 8104 } 8105 /* received an interrupt, but no rcd */ 8106 err_detail = "dataless"; 8107 } else { 8108 /* received an interrupt, but are not using that context */ 8109 err_detail = "out of range"; 8110 } 8111 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n", 8112 err_detail, source); 8113 } 8114 8115 /* 8116 * RX block receive urgent interrupt. Source is < 160. 8117 */ 8118 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source) 8119 { 8120 struct hfi1_ctxtdata *rcd; 8121 char *err_detail; 8122 8123 if (likely(source < dd->num_rcv_contexts)) { 8124 rcd = hfi1_rcd_get_by_index(dd, source); 8125 if (rcd) { 8126 /* only pay attention to user urgent interrupts */ 8127 if (source >= dd->first_dyn_alloc_ctxt && 8128 !rcd->is_vnic) 8129 handle_user_interrupt(rcd); 8130 8131 hfi1_rcd_put(rcd); 8132 return; /* OK */ 8133 } 8134 /* received an interrupt, but no rcd */ 8135 err_detail = "dataless"; 8136 } else { 8137 /* received an interrupt, but are not using that context */ 8138 err_detail = "out of range"; 8139 } 8140 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n", 8141 err_detail, source); 8142 } 8143 8144 /* 8145 * Reserved range interrupt. Should not be called in normal operation. 8146 */ 8147 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source) 8148 { 8149 char name[64]; 8150 8151 dd_dev_err(dd, "unexpected %s interrupt\n", 8152 is_reserved_name(name, sizeof(name), source)); 8153 } 8154 8155 static const struct is_table is_table[] = { 8156 /* 8157 * start end 8158 * name func interrupt func 8159 */ 8160 { IS_GENERAL_ERR_START, IS_GENERAL_ERR_END, 8161 is_misc_err_name, is_misc_err_int }, 8162 { IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END, 8163 is_sdma_eng_err_name, is_sdma_eng_err_int }, 8164 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END, 8165 is_sendctxt_err_name, is_sendctxt_err_int }, 8166 { IS_SDMA_START, IS_SDMA_END, 8167 is_sdma_eng_name, is_sdma_eng_int }, 8168 { IS_VARIOUS_START, IS_VARIOUS_END, 8169 is_various_name, is_various_int }, 8170 { IS_DC_START, IS_DC_END, 8171 is_dc_name, is_dc_int }, 8172 { IS_RCVAVAIL_START, IS_RCVAVAIL_END, 8173 is_rcv_avail_name, is_rcv_avail_int }, 8174 { IS_RCVURGENT_START, IS_RCVURGENT_END, 8175 is_rcv_urgent_name, is_rcv_urgent_int }, 8176 { IS_SENDCREDIT_START, IS_SENDCREDIT_END, 8177 is_send_credit_name, is_send_credit_int}, 8178 { IS_RESERVED_START, IS_RESERVED_END, 8179 is_reserved_name, is_reserved_int}, 8180 }; 8181 8182 /* 8183 * Interrupt source interrupt - called when the given source has an interrupt. 8184 * Source is a bit index into an array of 64-bit integers. 8185 */ 8186 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source) 8187 { 8188 const struct is_table *entry; 8189 8190 /* avoids a double compare by walking the table in-order */ 8191 for (entry = &is_table[0]; entry->is_name; entry++) { 8192 if (source < entry->end) { 8193 trace_hfi1_interrupt(dd, entry, source); 8194 entry->is_int(dd, source - entry->start); 8195 return; 8196 } 8197 } 8198 /* fell off the end */ 8199 dd_dev_err(dd, "invalid interrupt source %u\n", source); 8200 } 8201 8202 /* 8203 * General interrupt handler. This is able to correctly handle 8204 * all interrupts in case INTx is used. 8205 */ 8206 static irqreturn_t general_interrupt(int irq, void *data) 8207 { 8208 struct hfi1_devdata *dd = data; 8209 u64 regs[CCE_NUM_INT_CSRS]; 8210 u32 bit; 8211 int i; 8212 irqreturn_t handled = IRQ_NONE; 8213 8214 this_cpu_inc(*dd->int_counter); 8215 8216 /* phase 1: scan and clear all handled interrupts */ 8217 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 8218 if (dd->gi_mask[i] == 0) { 8219 regs[i] = 0; /* used later */ 8220 continue; 8221 } 8222 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) & 8223 dd->gi_mask[i]; 8224 /* only clear if anything is set */ 8225 if (regs[i]) 8226 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]); 8227 } 8228 8229 /* phase 2: call the appropriate handler */ 8230 for_each_set_bit(bit, (unsigned long *)®s[0], 8231 CCE_NUM_INT_CSRS * 64) { 8232 is_interrupt(dd, bit); 8233 handled = IRQ_HANDLED; 8234 } 8235 8236 return handled; 8237 } 8238 8239 static irqreturn_t sdma_interrupt(int irq, void *data) 8240 { 8241 struct sdma_engine *sde = data; 8242 struct hfi1_devdata *dd = sde->dd; 8243 u64 status; 8244 8245 #ifdef CONFIG_SDMA_VERBOSITY 8246 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 8247 slashstrip(__FILE__), __LINE__, __func__); 8248 sdma_dumpstate(sde); 8249 #endif 8250 8251 this_cpu_inc(*dd->int_counter); 8252 8253 /* This read_csr is really bad in the hot path */ 8254 status = read_csr(dd, 8255 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64))) 8256 & sde->imask; 8257 if (likely(status)) { 8258 /* clear the interrupt(s) */ 8259 write_csr(dd, 8260 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)), 8261 status); 8262 8263 /* handle the interrupt(s) */ 8264 sdma_engine_interrupt(sde, status); 8265 } else { 8266 dd_dev_err_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", 8267 sde->this_idx); 8268 } 8269 return IRQ_HANDLED; 8270 } 8271 8272 /* 8273 * Clear the receive interrupt. Use a read of the interrupt clear CSR 8274 * to insure that the write completed. This does NOT guarantee that 8275 * queued DMA writes to memory from the chip are pushed. 8276 */ 8277 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd) 8278 { 8279 struct hfi1_devdata *dd = rcd->dd; 8280 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg); 8281 8282 mmiowb(); /* make sure everything before is written */ 8283 write_csr(dd, addr, rcd->imask); 8284 /* force the above write on the chip and get a value back */ 8285 (void)read_csr(dd, addr); 8286 } 8287 8288 /* force the receive interrupt */ 8289 void force_recv_intr(struct hfi1_ctxtdata *rcd) 8290 { 8291 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask); 8292 } 8293 8294 /* 8295 * Return non-zero if a packet is present. 8296 * 8297 * This routine is called when rechecking for packets after the RcvAvail 8298 * interrupt has been cleared down. First, do a quick check of memory for 8299 * a packet present. If not found, use an expensive CSR read of the context 8300 * tail to determine the actual tail. The CSR read is necessary because there 8301 * is no method to push pending DMAs to memory other than an interrupt and we 8302 * are trying to determine if we need to force an interrupt. 8303 */ 8304 static inline int check_packet_present(struct hfi1_ctxtdata *rcd) 8305 { 8306 u32 tail; 8307 int present; 8308 8309 if (!HFI1_CAP_IS_KSET(DMA_RTAIL)) 8310 present = (rcd->seq_cnt == 8311 rhf_rcv_seq(rhf_to_cpu(get_rhf_addr(rcd)))); 8312 else /* is RDMA rtail */ 8313 present = (rcd->head != get_rcvhdrtail(rcd)); 8314 8315 if (present) 8316 return 1; 8317 8318 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */ 8319 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 8320 return rcd->head != tail; 8321 } 8322 8323 /* 8324 * Receive packet IRQ handler. This routine expects to be on its own IRQ. 8325 * This routine will try to handle packets immediately (latency), but if 8326 * it finds too many, it will invoke the thread handler (bandwitdh). The 8327 * chip receive interrupt is *not* cleared down until this or the thread (if 8328 * invoked) is finished. The intent is to avoid extra interrupts while we 8329 * are processing packets anyway. 8330 */ 8331 static irqreturn_t receive_context_interrupt(int irq, void *data) 8332 { 8333 struct hfi1_ctxtdata *rcd = data; 8334 struct hfi1_devdata *dd = rcd->dd; 8335 int disposition; 8336 int present; 8337 8338 trace_hfi1_receive_interrupt(dd, rcd); 8339 this_cpu_inc(*dd->int_counter); 8340 aspm_ctx_disable(rcd); 8341 8342 /* receive interrupt remains blocked while processing packets */ 8343 disposition = rcd->do_interrupt(rcd, 0); 8344 8345 /* 8346 * Too many packets were seen while processing packets in this 8347 * IRQ handler. Invoke the handler thread. The receive interrupt 8348 * remains blocked. 8349 */ 8350 if (disposition == RCV_PKT_LIMIT) 8351 return IRQ_WAKE_THREAD; 8352 8353 /* 8354 * The packet processor detected no more packets. Clear the receive 8355 * interrupt and recheck for a packet packet that may have arrived 8356 * after the previous check and interrupt clear. If a packet arrived, 8357 * force another interrupt. 8358 */ 8359 clear_recv_intr(rcd); 8360 present = check_packet_present(rcd); 8361 if (present) 8362 force_recv_intr(rcd); 8363 8364 return IRQ_HANDLED; 8365 } 8366 8367 /* 8368 * Receive packet thread handler. This expects to be invoked with the 8369 * receive interrupt still blocked. 8370 */ 8371 static irqreturn_t receive_context_thread(int irq, void *data) 8372 { 8373 struct hfi1_ctxtdata *rcd = data; 8374 int present; 8375 8376 /* receive interrupt is still blocked from the IRQ handler */ 8377 (void)rcd->do_interrupt(rcd, 1); 8378 8379 /* 8380 * The packet processor will only return if it detected no more 8381 * packets. Hold IRQs here so we can safely clear the interrupt and 8382 * recheck for a packet that may have arrived after the previous 8383 * check and the interrupt clear. If a packet arrived, force another 8384 * interrupt. 8385 */ 8386 local_irq_disable(); 8387 clear_recv_intr(rcd); 8388 present = check_packet_present(rcd); 8389 if (present) 8390 force_recv_intr(rcd); 8391 local_irq_enable(); 8392 8393 return IRQ_HANDLED; 8394 } 8395 8396 /* ========================================================================= */ 8397 8398 u32 read_physical_state(struct hfi1_devdata *dd) 8399 { 8400 u64 reg; 8401 8402 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE); 8403 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT) 8404 & DC_DC8051_STS_CUR_STATE_PORT_MASK; 8405 } 8406 8407 u32 read_logical_state(struct hfi1_devdata *dd) 8408 { 8409 u64 reg; 8410 8411 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8412 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT) 8413 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK; 8414 } 8415 8416 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate) 8417 { 8418 u64 reg; 8419 8420 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8421 /* clear current state, set new state */ 8422 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK; 8423 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT; 8424 write_csr(dd, DCC_CFG_PORT_CONFIG, reg); 8425 } 8426 8427 /* 8428 * Use the 8051 to read a LCB CSR. 8429 */ 8430 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data) 8431 { 8432 u32 regno; 8433 int ret; 8434 8435 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 8436 if (acquire_lcb_access(dd, 0) == 0) { 8437 *data = read_csr(dd, addr); 8438 release_lcb_access(dd, 0); 8439 return 0; 8440 } 8441 return -EBUSY; 8442 } 8443 8444 /* register is an index of LCB registers: (offset - base) / 8 */ 8445 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8446 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data); 8447 if (ret != HCMD_SUCCESS) 8448 return -EBUSY; 8449 return 0; 8450 } 8451 8452 /* 8453 * Provide a cache for some of the LCB registers in case the LCB is 8454 * unavailable. 8455 * (The LCB is unavailable in certain link states, for example.) 8456 */ 8457 struct lcb_datum { 8458 u32 off; 8459 u64 val; 8460 }; 8461 8462 static struct lcb_datum lcb_cache[] = { 8463 { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0}, 8464 { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 }, 8465 { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 }, 8466 }; 8467 8468 static void update_lcb_cache(struct hfi1_devdata *dd) 8469 { 8470 int i; 8471 int ret; 8472 u64 val; 8473 8474 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) { 8475 ret = read_lcb_csr(dd, lcb_cache[i].off, &val); 8476 8477 /* Update if we get good data */ 8478 if (likely(ret != -EBUSY)) 8479 lcb_cache[i].val = val; 8480 } 8481 } 8482 8483 static int read_lcb_cache(u32 off, u64 *val) 8484 { 8485 int i; 8486 8487 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) { 8488 if (lcb_cache[i].off == off) { 8489 *val = lcb_cache[i].val; 8490 return 0; 8491 } 8492 } 8493 8494 pr_warn("%s bad offset 0x%x\n", __func__, off); 8495 return -1; 8496 } 8497 8498 /* 8499 * Read an LCB CSR. Access may not be in host control, so check. 8500 * Return 0 on success, -EBUSY on failure. 8501 */ 8502 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data) 8503 { 8504 struct hfi1_pportdata *ppd = dd->pport; 8505 8506 /* if up, go through the 8051 for the value */ 8507 if (ppd->host_link_state & HLS_UP) 8508 return read_lcb_via_8051(dd, addr, data); 8509 /* if going up or down, check the cache, otherwise, no access */ 8510 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) { 8511 if (read_lcb_cache(addr, data)) 8512 return -EBUSY; 8513 return 0; 8514 } 8515 8516 /* otherwise, host has access */ 8517 *data = read_csr(dd, addr); 8518 return 0; 8519 } 8520 8521 /* 8522 * Use the 8051 to write a LCB CSR. 8523 */ 8524 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data) 8525 { 8526 u32 regno; 8527 int ret; 8528 8529 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || 8530 (dd->dc8051_ver < dc8051_ver(0, 20, 0))) { 8531 if (acquire_lcb_access(dd, 0) == 0) { 8532 write_csr(dd, addr, data); 8533 release_lcb_access(dd, 0); 8534 return 0; 8535 } 8536 return -EBUSY; 8537 } 8538 8539 /* register is an index of LCB registers: (offset - base) / 8 */ 8540 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8541 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data); 8542 if (ret != HCMD_SUCCESS) 8543 return -EBUSY; 8544 return 0; 8545 } 8546 8547 /* 8548 * Write an LCB CSR. Access may not be in host control, so check. 8549 * Return 0 on success, -EBUSY on failure. 8550 */ 8551 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data) 8552 { 8553 struct hfi1_pportdata *ppd = dd->pport; 8554 8555 /* if up, go through the 8051 for the value */ 8556 if (ppd->host_link_state & HLS_UP) 8557 return write_lcb_via_8051(dd, addr, data); 8558 /* if going up or down, no access */ 8559 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) 8560 return -EBUSY; 8561 /* otherwise, host has access */ 8562 write_csr(dd, addr, data); 8563 return 0; 8564 } 8565 8566 /* 8567 * If the 8051 is in reset mode (dd->dc_shutdown == 1), this function 8568 * will still continue executing. 8569 * 8570 * Returns: 8571 * < 0 = Linux error, not able to get access 8572 * > 0 = 8051 command RETURN_CODE 8573 */ 8574 static int _do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 8575 u64 *out_data) 8576 { 8577 u64 reg, completed; 8578 int return_code; 8579 unsigned long timeout; 8580 8581 lockdep_assert_held(&dd->dc8051_lock); 8582 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data); 8583 8584 /* 8585 * If an 8051 host command timed out previously, then the 8051 is 8586 * stuck. 8587 * 8588 * On first timeout, attempt to reset and restart the entire DC 8589 * block (including 8051). (Is this too big of a hammer?) 8590 * 8591 * If the 8051 times out a second time, the reset did not bring it 8592 * back to healthy life. In that case, fail any subsequent commands. 8593 */ 8594 if (dd->dc8051_timed_out) { 8595 if (dd->dc8051_timed_out > 1) { 8596 dd_dev_err(dd, 8597 "Previous 8051 host command timed out, skipping command %u\n", 8598 type); 8599 return_code = -ENXIO; 8600 goto fail; 8601 } 8602 _dc_shutdown(dd); 8603 _dc_start(dd); 8604 } 8605 8606 /* 8607 * If there is no timeout, then the 8051 command interface is 8608 * waiting for a command. 8609 */ 8610 8611 /* 8612 * When writing a LCB CSR, out_data contains the full value to 8613 * to be written, while in_data contains the relative LCB 8614 * address in 7:0. Do the work here, rather than the caller, 8615 * of distrubting the write data to where it needs to go: 8616 * 8617 * Write data 8618 * 39:00 -> in_data[47:8] 8619 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE 8620 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA 8621 */ 8622 if (type == HCMD_WRITE_LCB_CSR) { 8623 in_data |= ((*out_data) & 0xffffffffffull) << 8; 8624 /* must preserve COMPLETED - it is tied to hardware */ 8625 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0); 8626 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK; 8627 reg |= ((((*out_data) >> 40) & 0xff) << 8628 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT) 8629 | ((((*out_data) >> 48) & 0xffff) << 8630 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 8631 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg); 8632 } 8633 8634 /* 8635 * Do two writes: the first to stabilize the type and req_data, the 8636 * second to activate. 8637 */ 8638 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK) 8639 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT 8640 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK) 8641 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT; 8642 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8643 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK; 8644 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8645 8646 /* wait for completion, alternate: interrupt */ 8647 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT); 8648 while (1) { 8649 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1); 8650 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK; 8651 if (completed) 8652 break; 8653 if (time_after(jiffies, timeout)) { 8654 dd->dc8051_timed_out++; 8655 dd_dev_err(dd, "8051 host command %u timeout\n", type); 8656 if (out_data) 8657 *out_data = 0; 8658 return_code = -ETIMEDOUT; 8659 goto fail; 8660 } 8661 udelay(2); 8662 } 8663 8664 if (out_data) { 8665 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT) 8666 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK; 8667 if (type == HCMD_READ_LCB_CSR) { 8668 /* top 16 bits are in a different register */ 8669 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1) 8670 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK) 8671 << (48 8672 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT); 8673 } 8674 } 8675 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT) 8676 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK; 8677 dd->dc8051_timed_out = 0; 8678 /* 8679 * Clear command for next user. 8680 */ 8681 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0); 8682 8683 fail: 8684 return return_code; 8685 } 8686 8687 /* 8688 * Returns: 8689 * < 0 = Linux error, not able to get access 8690 * > 0 = 8051 command RETURN_CODE 8691 */ 8692 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 8693 u64 *out_data) 8694 { 8695 int return_code; 8696 8697 mutex_lock(&dd->dc8051_lock); 8698 /* We can't send any commands to the 8051 if it's in reset */ 8699 if (dd->dc_shutdown) { 8700 return_code = -ENODEV; 8701 goto fail; 8702 } 8703 8704 return_code = _do_8051_command(dd, type, in_data, out_data); 8705 8706 fail: 8707 mutex_unlock(&dd->dc8051_lock); 8708 return return_code; 8709 } 8710 8711 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state) 8712 { 8713 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL); 8714 } 8715 8716 static int _load_8051_config(struct hfi1_devdata *dd, u8 field_id, 8717 u8 lane_id, u32 config_data) 8718 { 8719 u64 data; 8720 int ret; 8721 8722 lockdep_assert_held(&dd->dc8051_lock); 8723 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT 8724 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT 8725 | (u64)config_data << LOAD_DATA_DATA_SHIFT; 8726 ret = _do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL); 8727 if (ret != HCMD_SUCCESS) { 8728 dd_dev_err(dd, 8729 "load 8051 config: field id %d, lane %d, err %d\n", 8730 (int)field_id, (int)lane_id, ret); 8731 } 8732 return ret; 8733 } 8734 8735 int load_8051_config(struct hfi1_devdata *dd, u8 field_id, 8736 u8 lane_id, u32 config_data) 8737 { 8738 int return_code; 8739 8740 mutex_lock(&dd->dc8051_lock); 8741 return_code = _load_8051_config(dd, field_id, lane_id, config_data); 8742 mutex_unlock(&dd->dc8051_lock); 8743 8744 return return_code; 8745 } 8746 8747 /* 8748 * Read the 8051 firmware "registers". Use the RAM directly. Always 8749 * set the result, even on error. 8750 * Return 0 on success, -errno on failure 8751 */ 8752 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id, 8753 u32 *result) 8754 { 8755 u64 big_data; 8756 u32 addr; 8757 int ret; 8758 8759 /* address start depends on the lane_id */ 8760 if (lane_id < 4) 8761 addr = (4 * NUM_GENERAL_FIELDS) 8762 + (lane_id * 4 * NUM_LANE_FIELDS); 8763 else 8764 addr = 0; 8765 addr += field_id * 4; 8766 8767 /* read is in 8-byte chunks, hardware will truncate the address down */ 8768 ret = read_8051_data(dd, addr, 8, &big_data); 8769 8770 if (ret == 0) { 8771 /* extract the 4 bytes we want */ 8772 if (addr & 0x4) 8773 *result = (u32)(big_data >> 32); 8774 else 8775 *result = (u32)big_data; 8776 } else { 8777 *result = 0; 8778 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n", 8779 __func__, lane_id, field_id); 8780 } 8781 8782 return ret; 8783 } 8784 8785 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management, 8786 u8 continuous) 8787 { 8788 u32 frame; 8789 8790 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT 8791 | power_management << POWER_MANAGEMENT_SHIFT; 8792 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY, 8793 GENERAL_CONFIG, frame); 8794 } 8795 8796 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu, 8797 u16 vl15buf, u8 crc_sizes) 8798 { 8799 u32 frame; 8800 8801 frame = (u32)vau << VAU_SHIFT 8802 | (u32)z << Z_SHIFT 8803 | (u32)vcu << VCU_SHIFT 8804 | (u32)vl15buf << VL15BUF_SHIFT 8805 | (u32)crc_sizes << CRC_SIZES_SHIFT; 8806 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC, 8807 GENERAL_CONFIG, frame); 8808 } 8809 8810 static void read_vc_local_link_width(struct hfi1_devdata *dd, u8 *misc_bits, 8811 u8 *flag_bits, u16 *link_widths) 8812 { 8813 u32 frame; 8814 8815 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG, 8816 &frame); 8817 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK; 8818 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK; 8819 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 8820 } 8821 8822 static int write_vc_local_link_width(struct hfi1_devdata *dd, 8823 u8 misc_bits, 8824 u8 flag_bits, 8825 u16 link_widths) 8826 { 8827 u32 frame; 8828 8829 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT 8830 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT 8831 | (u32)link_widths << LINK_WIDTH_SHIFT; 8832 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_WIDTH, GENERAL_CONFIG, 8833 frame); 8834 } 8835 8836 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id, 8837 u8 device_rev) 8838 { 8839 u32 frame; 8840 8841 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT) 8842 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT); 8843 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame); 8844 } 8845 8846 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 8847 u8 *device_rev) 8848 { 8849 u32 frame; 8850 8851 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame); 8852 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK; 8853 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT) 8854 & REMOTE_DEVICE_REV_MASK; 8855 } 8856 8857 int write_host_interface_version(struct hfi1_devdata *dd, u8 version) 8858 { 8859 u32 frame; 8860 u32 mask; 8861 8862 lockdep_assert_held(&dd->dc8051_lock); 8863 mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT); 8864 read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame); 8865 /* Clear, then set field */ 8866 frame &= ~mask; 8867 frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT); 8868 return _load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, 8869 frame); 8870 } 8871 8872 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor, 8873 u8 *ver_patch) 8874 { 8875 u32 frame; 8876 8877 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame); 8878 *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) & 8879 STS_FM_VERSION_MAJOR_MASK; 8880 *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) & 8881 STS_FM_VERSION_MINOR_MASK; 8882 8883 read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame); 8884 *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) & 8885 STS_FM_VERSION_PATCH_MASK; 8886 } 8887 8888 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 8889 u8 *continuous) 8890 { 8891 u32 frame; 8892 8893 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame); 8894 *power_management = (frame >> POWER_MANAGEMENT_SHIFT) 8895 & POWER_MANAGEMENT_MASK; 8896 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT) 8897 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK; 8898 } 8899 8900 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 8901 u8 *vcu, u16 *vl15buf, u8 *crc_sizes) 8902 { 8903 u32 frame; 8904 8905 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame); 8906 *vau = (frame >> VAU_SHIFT) & VAU_MASK; 8907 *z = (frame >> Z_SHIFT) & Z_MASK; 8908 *vcu = (frame >> VCU_SHIFT) & VCU_MASK; 8909 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK; 8910 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK; 8911 } 8912 8913 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 8914 u8 *remote_tx_rate, 8915 u16 *link_widths) 8916 { 8917 u32 frame; 8918 8919 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG, 8920 &frame); 8921 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT) 8922 & REMOTE_TX_RATE_MASK; 8923 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 8924 } 8925 8926 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx) 8927 { 8928 u32 frame; 8929 8930 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame); 8931 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK; 8932 } 8933 8934 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls) 8935 { 8936 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls); 8937 } 8938 8939 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs) 8940 { 8941 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs); 8942 } 8943 8944 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality) 8945 { 8946 u32 frame; 8947 int ret; 8948 8949 *link_quality = 0; 8950 if (dd->pport->host_link_state & HLS_UP) { 8951 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, 8952 &frame); 8953 if (ret == 0) 8954 *link_quality = (frame >> LINK_QUALITY_SHIFT) 8955 & LINK_QUALITY_MASK; 8956 } 8957 } 8958 8959 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc) 8960 { 8961 u32 frame; 8962 8963 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame); 8964 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK; 8965 } 8966 8967 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr) 8968 { 8969 u32 frame; 8970 8971 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame); 8972 *ldr = (frame & 0xff); 8973 } 8974 8975 static int read_tx_settings(struct hfi1_devdata *dd, 8976 u8 *enable_lane_tx, 8977 u8 *tx_polarity_inversion, 8978 u8 *rx_polarity_inversion, 8979 u8 *max_rate) 8980 { 8981 u32 frame; 8982 int ret; 8983 8984 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame); 8985 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT) 8986 & ENABLE_LANE_TX_MASK; 8987 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT) 8988 & TX_POLARITY_INVERSION_MASK; 8989 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT) 8990 & RX_POLARITY_INVERSION_MASK; 8991 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK; 8992 return ret; 8993 } 8994 8995 static int write_tx_settings(struct hfi1_devdata *dd, 8996 u8 enable_lane_tx, 8997 u8 tx_polarity_inversion, 8998 u8 rx_polarity_inversion, 8999 u8 max_rate) 9000 { 9001 u32 frame; 9002 9003 /* no need to mask, all variable sizes match field widths */ 9004 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT 9005 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT 9006 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT 9007 | max_rate << MAX_RATE_SHIFT; 9008 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame); 9009 } 9010 9011 /* 9012 * Read an idle LCB message. 9013 * 9014 * Returns 0 on success, -EINVAL on error 9015 */ 9016 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out) 9017 { 9018 int ret; 9019 9020 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out); 9021 if (ret != HCMD_SUCCESS) { 9022 dd_dev_err(dd, "read idle message: type %d, err %d\n", 9023 (u32)type, ret); 9024 return -EINVAL; 9025 } 9026 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out); 9027 /* return only the payload as we already know the type */ 9028 *data_out >>= IDLE_PAYLOAD_SHIFT; 9029 return 0; 9030 } 9031 9032 /* 9033 * Read an idle SMA message. To be done in response to a notification from 9034 * the 8051. 9035 * 9036 * Returns 0 on success, -EINVAL on error 9037 */ 9038 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data) 9039 { 9040 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT, 9041 data); 9042 } 9043 9044 /* 9045 * Send an idle LCB message. 9046 * 9047 * Returns 0 on success, -EINVAL on error 9048 */ 9049 static int send_idle_message(struct hfi1_devdata *dd, u64 data) 9050 { 9051 int ret; 9052 9053 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data); 9054 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL); 9055 if (ret != HCMD_SUCCESS) { 9056 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n", 9057 data, ret); 9058 return -EINVAL; 9059 } 9060 return 0; 9061 } 9062 9063 /* 9064 * Send an idle SMA message. 9065 * 9066 * Returns 0 on success, -EINVAL on error 9067 */ 9068 int send_idle_sma(struct hfi1_devdata *dd, u64 message) 9069 { 9070 u64 data; 9071 9072 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) | 9073 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT); 9074 return send_idle_message(dd, data); 9075 } 9076 9077 /* 9078 * Initialize the LCB then do a quick link up. This may or may not be 9079 * in loopback. 9080 * 9081 * return 0 on success, -errno on error 9082 */ 9083 static int do_quick_linkup(struct hfi1_devdata *dd) 9084 { 9085 int ret; 9086 9087 lcb_shutdown(dd, 0); 9088 9089 if (loopback) { 9090 /* LCB_CFG_LOOPBACK.VAL = 2 */ 9091 /* LCB_CFG_LANE_WIDTH.VAL = 0 */ 9092 write_csr(dd, DC_LCB_CFG_LOOPBACK, 9093 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT); 9094 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 9095 } 9096 9097 /* start the LCBs */ 9098 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */ 9099 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 9100 9101 /* simulator only loopback steps */ 9102 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 9103 /* LCB_CFG_RUN.EN = 1 */ 9104 write_csr(dd, DC_LCB_CFG_RUN, 9105 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 9106 9107 ret = wait_link_transfer_active(dd, 10); 9108 if (ret) 9109 return ret; 9110 9111 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 9112 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT); 9113 } 9114 9115 if (!loopback) { 9116 /* 9117 * When doing quick linkup and not in loopback, both 9118 * sides must be done with LCB set-up before either 9119 * starts the quick linkup. Put a delay here so that 9120 * both sides can be started and have a chance to be 9121 * done with LCB set up before resuming. 9122 */ 9123 dd_dev_err(dd, 9124 "Pausing for peer to be finished with LCB set up\n"); 9125 msleep(5000); 9126 dd_dev_err(dd, "Continuing with quick linkup\n"); 9127 } 9128 9129 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 9130 set_8051_lcb_access(dd); 9131 9132 /* 9133 * State "quick" LinkUp request sets the physical link state to 9134 * LinkUp without a verify capability sequence. 9135 * This state is in simulator v37 and later. 9136 */ 9137 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP); 9138 if (ret != HCMD_SUCCESS) { 9139 dd_dev_err(dd, 9140 "%s: set physical link state to quick LinkUp failed with return %d\n", 9141 __func__, ret); 9142 9143 set_host_lcb_access(dd); 9144 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 9145 9146 if (ret >= 0) 9147 ret = -EINVAL; 9148 return ret; 9149 } 9150 9151 return 0; /* success */ 9152 } 9153 9154 /* 9155 * Do all special steps to set up loopback. 9156 */ 9157 static int init_loopback(struct hfi1_devdata *dd) 9158 { 9159 dd_dev_info(dd, "Entering loopback mode\n"); 9160 9161 /* all loopbacks should disable self GUID check */ 9162 write_csr(dd, DC_DC8051_CFG_MODE, 9163 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK)); 9164 9165 /* 9166 * The simulator has only one loopback option - LCB. Switch 9167 * to that option, which includes quick link up. 9168 * 9169 * Accept all valid loopback values. 9170 */ 9171 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) && 9172 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB || 9173 loopback == LOOPBACK_CABLE)) { 9174 loopback = LOOPBACK_LCB; 9175 quick_linkup = 1; 9176 return 0; 9177 } 9178 9179 /* 9180 * SerDes loopback init sequence is handled in set_local_link_attributes 9181 */ 9182 if (loopback == LOOPBACK_SERDES) 9183 return 0; 9184 9185 /* LCB loopback - handled at poll time */ 9186 if (loopback == LOOPBACK_LCB) { 9187 quick_linkup = 1; /* LCB is always quick linkup */ 9188 9189 /* not supported in emulation due to emulation RTL changes */ 9190 if (dd->icode == ICODE_FPGA_EMULATION) { 9191 dd_dev_err(dd, 9192 "LCB loopback not supported in emulation\n"); 9193 return -EINVAL; 9194 } 9195 return 0; 9196 } 9197 9198 /* external cable loopback requires no extra steps */ 9199 if (loopback == LOOPBACK_CABLE) 9200 return 0; 9201 9202 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback); 9203 return -EINVAL; 9204 } 9205 9206 /* 9207 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits 9208 * used in the Verify Capability link width attribute. 9209 */ 9210 static u16 opa_to_vc_link_widths(u16 opa_widths) 9211 { 9212 int i; 9213 u16 result = 0; 9214 9215 static const struct link_bits { 9216 u16 from; 9217 u16 to; 9218 } opa_link_xlate[] = { 9219 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) }, 9220 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) }, 9221 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) }, 9222 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) }, 9223 }; 9224 9225 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) { 9226 if (opa_widths & opa_link_xlate[i].from) 9227 result |= opa_link_xlate[i].to; 9228 } 9229 return result; 9230 } 9231 9232 /* 9233 * Set link attributes before moving to polling. 9234 */ 9235 static int set_local_link_attributes(struct hfi1_pportdata *ppd) 9236 { 9237 struct hfi1_devdata *dd = ppd->dd; 9238 u8 enable_lane_tx; 9239 u8 tx_polarity_inversion; 9240 u8 rx_polarity_inversion; 9241 int ret; 9242 u32 misc_bits = 0; 9243 /* reset our fabric serdes to clear any lingering problems */ 9244 fabric_serdes_reset(dd); 9245 9246 /* set the local tx rate - need to read-modify-write */ 9247 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 9248 &rx_polarity_inversion, &ppd->local_tx_rate); 9249 if (ret) 9250 goto set_local_link_attributes_fail; 9251 9252 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) { 9253 /* set the tx rate to the fastest enabled */ 9254 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9255 ppd->local_tx_rate = 1; 9256 else 9257 ppd->local_tx_rate = 0; 9258 } else { 9259 /* set the tx rate to all enabled */ 9260 ppd->local_tx_rate = 0; 9261 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9262 ppd->local_tx_rate |= 2; 9263 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G) 9264 ppd->local_tx_rate |= 1; 9265 } 9266 9267 enable_lane_tx = 0xF; /* enable all four lanes */ 9268 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion, 9269 rx_polarity_inversion, ppd->local_tx_rate); 9270 if (ret != HCMD_SUCCESS) 9271 goto set_local_link_attributes_fail; 9272 9273 /* 9274 * DC supports continuous updates. 9275 */ 9276 ret = write_vc_local_phy(dd, 9277 0 /* no power management */, 9278 1 /* continuous updates */); 9279 if (ret != HCMD_SUCCESS) 9280 goto set_local_link_attributes_fail; 9281 9282 /* z=1 in the next call: AU of 0 is not supported by the hardware */ 9283 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init, 9284 ppd->port_crc_mode_enabled); 9285 if (ret != HCMD_SUCCESS) 9286 goto set_local_link_attributes_fail; 9287 9288 /* 9289 * SerDes loopback init sequence requires 9290 * setting bit 0 of MISC_CONFIG_BITS 9291 */ 9292 if (loopback == LOOPBACK_SERDES) 9293 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT; 9294 9295 ret = write_vc_local_link_width(dd, misc_bits, 0, 9296 opa_to_vc_link_widths( 9297 ppd->link_width_enabled)); 9298 if (ret != HCMD_SUCCESS) 9299 goto set_local_link_attributes_fail; 9300 9301 /* let peer know who we are */ 9302 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev); 9303 if (ret == HCMD_SUCCESS) 9304 return 0; 9305 9306 set_local_link_attributes_fail: 9307 dd_dev_err(dd, 9308 "Failed to set local link attributes, return 0x%x\n", 9309 ret); 9310 return ret; 9311 } 9312 9313 /* 9314 * Call this to start the link. 9315 * Do not do anything if the link is disabled. 9316 * Returns 0 if link is disabled, moved to polling, or the driver is not ready. 9317 */ 9318 int start_link(struct hfi1_pportdata *ppd) 9319 { 9320 /* 9321 * Tune the SerDes to a ballpark setting for optimal signal and bit 9322 * error rate. Needs to be done before starting the link. 9323 */ 9324 tune_serdes(ppd); 9325 9326 if (!ppd->driver_link_ready) { 9327 dd_dev_info(ppd->dd, 9328 "%s: stopping link start because driver is not ready\n", 9329 __func__); 9330 return 0; 9331 } 9332 9333 /* 9334 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the 9335 * pkey table can be configured properly if the HFI unit is connected 9336 * to switch port with MgmtAllowed=NO 9337 */ 9338 clear_full_mgmt_pkey(ppd); 9339 9340 return set_link_state(ppd, HLS_DN_POLL); 9341 } 9342 9343 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd) 9344 { 9345 struct hfi1_devdata *dd = ppd->dd; 9346 u64 mask; 9347 unsigned long timeout; 9348 9349 /* 9350 * Some QSFP cables have a quirk that asserts the IntN line as a side 9351 * effect of power up on plug-in. We ignore this false positive 9352 * interrupt until the module has finished powering up by waiting for 9353 * a minimum timeout of the module inrush initialization time of 9354 * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the 9355 * module have stabilized. 9356 */ 9357 msleep(500); 9358 9359 /* 9360 * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1) 9361 */ 9362 timeout = jiffies + msecs_to_jiffies(2000); 9363 while (1) { 9364 mask = read_csr(dd, dd->hfi1_id ? 9365 ASIC_QSFP2_IN : ASIC_QSFP1_IN); 9366 if (!(mask & QSFP_HFI0_INT_N)) 9367 break; 9368 if (time_after(jiffies, timeout)) { 9369 dd_dev_info(dd, "%s: No IntN detected, reset complete\n", 9370 __func__); 9371 break; 9372 } 9373 udelay(2); 9374 } 9375 } 9376 9377 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable) 9378 { 9379 struct hfi1_devdata *dd = ppd->dd; 9380 u64 mask; 9381 9382 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK); 9383 if (enable) { 9384 /* 9385 * Clear the status register to avoid an immediate interrupt 9386 * when we re-enable the IntN pin 9387 */ 9388 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9389 QSFP_HFI0_INT_N); 9390 mask |= (u64)QSFP_HFI0_INT_N; 9391 } else { 9392 mask &= ~(u64)QSFP_HFI0_INT_N; 9393 } 9394 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask); 9395 } 9396 9397 int reset_qsfp(struct hfi1_pportdata *ppd) 9398 { 9399 struct hfi1_devdata *dd = ppd->dd; 9400 u64 mask, qsfp_mask; 9401 9402 /* Disable INT_N from triggering QSFP interrupts */ 9403 set_qsfp_int_n(ppd, 0); 9404 9405 /* Reset the QSFP */ 9406 mask = (u64)QSFP_HFI0_RESET_N; 9407 9408 qsfp_mask = read_csr(dd, 9409 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT); 9410 qsfp_mask &= ~mask; 9411 write_csr(dd, 9412 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9413 9414 udelay(10); 9415 9416 qsfp_mask |= mask; 9417 write_csr(dd, 9418 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9419 9420 wait_for_qsfp_init(ppd); 9421 9422 /* 9423 * Allow INT_N to trigger the QSFP interrupt to watch 9424 * for alarms and warnings 9425 */ 9426 set_qsfp_int_n(ppd, 1); 9427 9428 /* 9429 * After the reset, AOC transmitters are enabled by default. They need 9430 * to be turned off to complete the QSFP setup before they can be 9431 * enabled again. 9432 */ 9433 return set_qsfp_tx(ppd, 0); 9434 } 9435 9436 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd, 9437 u8 *qsfp_interrupt_status) 9438 { 9439 struct hfi1_devdata *dd = ppd->dd; 9440 9441 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) || 9442 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING)) 9443 dd_dev_err(dd, "%s: QSFP cable temperature too high\n", 9444 __func__); 9445 9446 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) || 9447 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING)) 9448 dd_dev_err(dd, "%s: QSFP cable temperature too low\n", 9449 __func__); 9450 9451 /* 9452 * The remaining alarms/warnings don't matter if the link is down. 9453 */ 9454 if (ppd->host_link_state & HLS_DOWN) 9455 return 0; 9456 9457 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) || 9458 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING)) 9459 dd_dev_err(dd, "%s: QSFP supply voltage too high\n", 9460 __func__); 9461 9462 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) || 9463 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING)) 9464 dd_dev_err(dd, "%s: QSFP supply voltage too low\n", 9465 __func__); 9466 9467 /* Byte 2 is vendor specific */ 9468 9469 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) || 9470 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING)) 9471 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n", 9472 __func__); 9473 9474 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) || 9475 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING)) 9476 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n", 9477 __func__); 9478 9479 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) || 9480 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING)) 9481 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n", 9482 __func__); 9483 9484 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) || 9485 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING)) 9486 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n", 9487 __func__); 9488 9489 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) || 9490 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING)) 9491 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n", 9492 __func__); 9493 9494 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) || 9495 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING)) 9496 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n", 9497 __func__); 9498 9499 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) || 9500 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING)) 9501 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n", 9502 __func__); 9503 9504 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) || 9505 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING)) 9506 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n", 9507 __func__); 9508 9509 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) || 9510 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING)) 9511 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n", 9512 __func__); 9513 9514 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) || 9515 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING)) 9516 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n", 9517 __func__); 9518 9519 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) || 9520 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING)) 9521 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n", 9522 __func__); 9523 9524 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) || 9525 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING)) 9526 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n", 9527 __func__); 9528 9529 /* Bytes 9-10 and 11-12 are reserved */ 9530 /* Bytes 13-15 are vendor specific */ 9531 9532 return 0; 9533 } 9534 9535 /* This routine will only be scheduled if the QSFP module present is asserted */ 9536 void qsfp_event(struct work_struct *work) 9537 { 9538 struct qsfp_data *qd; 9539 struct hfi1_pportdata *ppd; 9540 struct hfi1_devdata *dd; 9541 9542 qd = container_of(work, struct qsfp_data, qsfp_work); 9543 ppd = qd->ppd; 9544 dd = ppd->dd; 9545 9546 /* Sanity check */ 9547 if (!qsfp_mod_present(ppd)) 9548 return; 9549 9550 if (ppd->host_link_state == HLS_DN_DISABLE) { 9551 dd_dev_info(ppd->dd, 9552 "%s: stopping link start because link is disabled\n", 9553 __func__); 9554 return; 9555 } 9556 9557 /* 9558 * Turn DC back on after cable has been re-inserted. Up until 9559 * now, the DC has been in reset to save power. 9560 */ 9561 dc_start(dd); 9562 9563 if (qd->cache_refresh_required) { 9564 set_qsfp_int_n(ppd, 0); 9565 9566 wait_for_qsfp_init(ppd); 9567 9568 /* 9569 * Allow INT_N to trigger the QSFP interrupt to watch 9570 * for alarms and warnings 9571 */ 9572 set_qsfp_int_n(ppd, 1); 9573 9574 start_link(ppd); 9575 } 9576 9577 if (qd->check_interrupt_flags) { 9578 u8 qsfp_interrupt_status[16] = {0,}; 9579 9580 if (one_qsfp_read(ppd, dd->hfi1_id, 6, 9581 &qsfp_interrupt_status[0], 16) != 16) { 9582 dd_dev_info(dd, 9583 "%s: Failed to read status of QSFP module\n", 9584 __func__); 9585 } else { 9586 unsigned long flags; 9587 9588 handle_qsfp_error_conditions( 9589 ppd, qsfp_interrupt_status); 9590 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 9591 ppd->qsfp_info.check_interrupt_flags = 0; 9592 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 9593 flags); 9594 } 9595 } 9596 } 9597 9598 static void init_qsfp_int(struct hfi1_devdata *dd) 9599 { 9600 struct hfi1_pportdata *ppd = dd->pport; 9601 u64 qsfp_mask, cce_int_mask; 9602 const int qsfp1_int_smask = QSFP1_INT % 64; 9603 const int qsfp2_int_smask = QSFP2_INT % 64; 9604 9605 /* 9606 * disable QSFP1 interrupts for HFI1, QSFP2 interrupts for HFI0 9607 * Qsfp1Int and Qsfp2Int are adjacent bits in the same CSR, 9608 * therefore just one of QSFP1_INT/QSFP2_INT can be used to find 9609 * the index of the appropriate CSR in the CCEIntMask CSR array 9610 */ 9611 cce_int_mask = read_csr(dd, CCE_INT_MASK + 9612 (8 * (QSFP1_INT / 64))); 9613 if (dd->hfi1_id) { 9614 cce_int_mask &= ~((u64)1 << qsfp1_int_smask); 9615 write_csr(dd, CCE_INT_MASK + (8 * (QSFP1_INT / 64)), 9616 cce_int_mask); 9617 } else { 9618 cce_int_mask &= ~((u64)1 << qsfp2_int_smask); 9619 write_csr(dd, CCE_INT_MASK + (8 * (QSFP2_INT / 64)), 9620 cce_int_mask); 9621 } 9622 9623 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 9624 /* Clear current status to avoid spurious interrupts */ 9625 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9626 qsfp_mask); 9627 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, 9628 qsfp_mask); 9629 9630 set_qsfp_int_n(ppd, 0); 9631 9632 /* Handle active low nature of INT_N and MODPRST_N pins */ 9633 if (qsfp_mod_present(ppd)) 9634 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N; 9635 write_csr(dd, 9636 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT, 9637 qsfp_mask); 9638 } 9639 9640 /* 9641 * Do a one-time initialize of the LCB block. 9642 */ 9643 static void init_lcb(struct hfi1_devdata *dd) 9644 { 9645 /* simulator does not correctly handle LCB cclk loopback, skip */ 9646 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 9647 return; 9648 9649 /* the DC has been reset earlier in the driver load */ 9650 9651 /* set LCB for cclk loopback on the port */ 9652 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01); 9653 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00); 9654 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00); 9655 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 9656 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08); 9657 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02); 9658 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00); 9659 } 9660 9661 /* 9662 * Perform a test read on the QSFP. Return 0 on success, -ERRNO 9663 * on error. 9664 */ 9665 static int test_qsfp_read(struct hfi1_pportdata *ppd) 9666 { 9667 int ret; 9668 u8 status; 9669 9670 /* 9671 * Report success if not a QSFP or, if it is a QSFP, but the cable is 9672 * not present 9673 */ 9674 if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd)) 9675 return 0; 9676 9677 /* read byte 2, the status byte */ 9678 ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1); 9679 if (ret < 0) 9680 return ret; 9681 if (ret != 1) 9682 return -EIO; 9683 9684 return 0; /* success */ 9685 } 9686 9687 /* 9688 * Values for QSFP retry. 9689 * 9690 * Give up after 10s (20 x 500ms). The overall timeout was empirically 9691 * arrived at from experience on a large cluster. 9692 */ 9693 #define MAX_QSFP_RETRIES 20 9694 #define QSFP_RETRY_WAIT 500 /* msec */ 9695 9696 /* 9697 * Try a QSFP read. If it fails, schedule a retry for later. 9698 * Called on first link activation after driver load. 9699 */ 9700 static void try_start_link(struct hfi1_pportdata *ppd) 9701 { 9702 if (test_qsfp_read(ppd)) { 9703 /* read failed */ 9704 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) { 9705 dd_dev_err(ppd->dd, "QSFP not responding, giving up\n"); 9706 return; 9707 } 9708 dd_dev_info(ppd->dd, 9709 "QSFP not responding, waiting and retrying %d\n", 9710 (int)ppd->qsfp_retry_count); 9711 ppd->qsfp_retry_count++; 9712 queue_delayed_work(ppd->link_wq, &ppd->start_link_work, 9713 msecs_to_jiffies(QSFP_RETRY_WAIT)); 9714 return; 9715 } 9716 ppd->qsfp_retry_count = 0; 9717 9718 start_link(ppd); 9719 } 9720 9721 /* 9722 * Workqueue function to start the link after a delay. 9723 */ 9724 void handle_start_link(struct work_struct *work) 9725 { 9726 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 9727 start_link_work.work); 9728 try_start_link(ppd); 9729 } 9730 9731 int bringup_serdes(struct hfi1_pportdata *ppd) 9732 { 9733 struct hfi1_devdata *dd = ppd->dd; 9734 u64 guid; 9735 int ret; 9736 9737 if (HFI1_CAP_IS_KSET(EXTENDED_PSN)) 9738 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK); 9739 9740 guid = ppd->guids[HFI1_PORT_GUID_INDEX]; 9741 if (!guid) { 9742 if (dd->base_guid) 9743 guid = dd->base_guid + ppd->port - 1; 9744 ppd->guids[HFI1_PORT_GUID_INDEX] = guid; 9745 } 9746 9747 /* Set linkinit_reason on power up per OPA spec */ 9748 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP; 9749 9750 /* one-time init of the LCB */ 9751 init_lcb(dd); 9752 9753 if (loopback) { 9754 ret = init_loopback(dd); 9755 if (ret < 0) 9756 return ret; 9757 } 9758 9759 get_port_type(ppd); 9760 if (ppd->port_type == PORT_TYPE_QSFP) { 9761 set_qsfp_int_n(ppd, 0); 9762 wait_for_qsfp_init(ppd); 9763 set_qsfp_int_n(ppd, 1); 9764 } 9765 9766 try_start_link(ppd); 9767 return 0; 9768 } 9769 9770 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd) 9771 { 9772 struct hfi1_devdata *dd = ppd->dd; 9773 9774 /* 9775 * Shut down the link and keep it down. First turn off that the 9776 * driver wants to allow the link to be up (driver_link_ready). 9777 * Then make sure the link is not automatically restarted 9778 * (link_enabled). Cancel any pending restart. And finally 9779 * go offline. 9780 */ 9781 ppd->driver_link_ready = 0; 9782 ppd->link_enabled = 0; 9783 9784 ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */ 9785 flush_delayed_work(&ppd->start_link_work); 9786 cancel_delayed_work_sync(&ppd->start_link_work); 9787 9788 ppd->offline_disabled_reason = 9789 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT); 9790 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0, 9791 OPA_LINKDOWN_REASON_REBOOT); 9792 set_link_state(ppd, HLS_DN_OFFLINE); 9793 9794 /* disable the port */ 9795 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 9796 } 9797 9798 static inline int init_cpu_counters(struct hfi1_devdata *dd) 9799 { 9800 struct hfi1_pportdata *ppd; 9801 int i; 9802 9803 ppd = (struct hfi1_pportdata *)(dd + 1); 9804 for (i = 0; i < dd->num_pports; i++, ppd++) { 9805 ppd->ibport_data.rvp.rc_acks = NULL; 9806 ppd->ibport_data.rvp.rc_qacks = NULL; 9807 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64); 9808 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64); 9809 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64); 9810 if (!ppd->ibport_data.rvp.rc_acks || 9811 !ppd->ibport_data.rvp.rc_delayed_comp || 9812 !ppd->ibport_data.rvp.rc_qacks) 9813 return -ENOMEM; 9814 } 9815 9816 return 0; 9817 } 9818 9819 /* 9820 * index is the index into the receive array 9821 */ 9822 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index, 9823 u32 type, unsigned long pa, u16 order) 9824 { 9825 u64 reg; 9826 9827 if (!(dd->flags & HFI1_PRESENT)) 9828 goto done; 9829 9830 if (type == PT_INVALID || type == PT_INVALID_FLUSH) { 9831 pa = 0; 9832 order = 0; 9833 } else if (type > PT_INVALID) { 9834 dd_dev_err(dd, 9835 "unexpected receive array type %u for index %u, not handled\n", 9836 type, index); 9837 goto done; 9838 } 9839 trace_hfi1_put_tid(dd, index, type, pa, order); 9840 9841 #define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */ 9842 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK 9843 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT 9844 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK) 9845 << RCV_ARRAY_RT_ADDR_SHIFT; 9846 trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg); 9847 writeq(reg, dd->rcvarray_wc + (index * 8)); 9848 9849 if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3) 9850 /* 9851 * Eager entries are written and flushed 9852 * 9853 * Expected entries are flushed every 4 writes 9854 */ 9855 flush_wc(); 9856 done: 9857 return; 9858 } 9859 9860 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd) 9861 { 9862 struct hfi1_devdata *dd = rcd->dd; 9863 u32 i; 9864 9865 /* this could be optimized */ 9866 for (i = rcd->eager_base; i < rcd->eager_base + 9867 rcd->egrbufs.alloced; i++) 9868 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9869 9870 for (i = rcd->expected_base; 9871 i < rcd->expected_base + rcd->expected_count; i++) 9872 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9873 } 9874 9875 static const char * const ib_cfg_name_strings[] = { 9876 "HFI1_IB_CFG_LIDLMC", 9877 "HFI1_IB_CFG_LWID_DG_ENB", 9878 "HFI1_IB_CFG_LWID_ENB", 9879 "HFI1_IB_CFG_LWID", 9880 "HFI1_IB_CFG_SPD_ENB", 9881 "HFI1_IB_CFG_SPD", 9882 "HFI1_IB_CFG_RXPOL_ENB", 9883 "HFI1_IB_CFG_LREV_ENB", 9884 "HFI1_IB_CFG_LINKLATENCY", 9885 "HFI1_IB_CFG_HRTBT", 9886 "HFI1_IB_CFG_OP_VLS", 9887 "HFI1_IB_CFG_VL_HIGH_CAP", 9888 "HFI1_IB_CFG_VL_LOW_CAP", 9889 "HFI1_IB_CFG_OVERRUN_THRESH", 9890 "HFI1_IB_CFG_PHYERR_THRESH", 9891 "HFI1_IB_CFG_LINKDEFAULT", 9892 "HFI1_IB_CFG_PKEYS", 9893 "HFI1_IB_CFG_MTU", 9894 "HFI1_IB_CFG_LSTATE", 9895 "HFI1_IB_CFG_VL_HIGH_LIMIT", 9896 "HFI1_IB_CFG_PMA_TICKS", 9897 "HFI1_IB_CFG_PORT" 9898 }; 9899 9900 static const char *ib_cfg_name(int which) 9901 { 9902 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings)) 9903 return "invalid"; 9904 return ib_cfg_name_strings[which]; 9905 } 9906 9907 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which) 9908 { 9909 struct hfi1_devdata *dd = ppd->dd; 9910 int val = 0; 9911 9912 switch (which) { 9913 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */ 9914 val = ppd->link_width_enabled; 9915 break; 9916 case HFI1_IB_CFG_LWID: /* currently active Link-width */ 9917 val = ppd->link_width_active; 9918 break; 9919 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 9920 val = ppd->link_speed_enabled; 9921 break; 9922 case HFI1_IB_CFG_SPD: /* current Link speed */ 9923 val = ppd->link_speed_active; 9924 break; 9925 9926 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */ 9927 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */ 9928 case HFI1_IB_CFG_LINKLATENCY: 9929 goto unimplemented; 9930 9931 case HFI1_IB_CFG_OP_VLS: 9932 val = ppd->actual_vls_operational; 9933 break; 9934 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */ 9935 val = VL_ARB_HIGH_PRIO_TABLE_SIZE; 9936 break; 9937 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */ 9938 val = VL_ARB_LOW_PRIO_TABLE_SIZE; 9939 break; 9940 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 9941 val = ppd->overrun_threshold; 9942 break; 9943 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 9944 val = ppd->phy_error_threshold; 9945 break; 9946 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 9947 val = HLS_DEFAULT; 9948 break; 9949 9950 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */ 9951 case HFI1_IB_CFG_PMA_TICKS: 9952 default: 9953 unimplemented: 9954 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 9955 dd_dev_info( 9956 dd, 9957 "%s: which %s: not implemented\n", 9958 __func__, 9959 ib_cfg_name(which)); 9960 break; 9961 } 9962 9963 return val; 9964 } 9965 9966 /* 9967 * The largest MAD packet size. 9968 */ 9969 #define MAX_MAD_PACKET 2048 9970 9971 /* 9972 * Return the maximum header bytes that can go on the _wire_ 9973 * for this device. This count includes the ICRC which is 9974 * not part of the packet held in memory but it is appended 9975 * by the HW. 9976 * This is dependent on the device's receive header entry size. 9977 * HFI allows this to be set per-receive context, but the 9978 * driver presently enforces a global value. 9979 */ 9980 u32 lrh_max_header_bytes(struct hfi1_devdata *dd) 9981 { 9982 /* 9983 * The maximum non-payload (MTU) bytes in LRH.PktLen are 9984 * the Receive Header Entry Size minus the PBC (or RHF) size 9985 * plus one DW for the ICRC appended by HW. 9986 * 9987 * dd->rcd[0].rcvhdrqentsize is in DW. 9988 * We use rcd[0] as all context will have the same value. Also, 9989 * the first kernel context would have been allocated by now so 9990 * we are guaranteed a valid value. 9991 */ 9992 return (dd->rcd[0]->rcvhdrqentsize - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2; 9993 } 9994 9995 /* 9996 * Set Send Length 9997 * @ppd - per port data 9998 * 9999 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck* 10000 * registers compare against LRH.PktLen, so use the max bytes included 10001 * in the LRH. 10002 * 10003 * This routine changes all VL values except VL15, which it maintains at 10004 * the same value. 10005 */ 10006 static void set_send_length(struct hfi1_pportdata *ppd) 10007 { 10008 struct hfi1_devdata *dd = ppd->dd; 10009 u32 max_hb = lrh_max_header_bytes(dd), dcmtu; 10010 u32 maxvlmtu = dd->vld[15].mtu; 10011 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2) 10012 & SEND_LEN_CHECK1_LEN_VL15_MASK) << 10013 SEND_LEN_CHECK1_LEN_VL15_SHIFT; 10014 int i, j; 10015 u32 thres; 10016 10017 for (i = 0; i < ppd->vls_supported; i++) { 10018 if (dd->vld[i].mtu > maxvlmtu) 10019 maxvlmtu = dd->vld[i].mtu; 10020 if (i <= 3) 10021 len1 |= (((dd->vld[i].mtu + max_hb) >> 2) 10022 & SEND_LEN_CHECK0_LEN_VL0_MASK) << 10023 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT); 10024 else 10025 len2 |= (((dd->vld[i].mtu + max_hb) >> 2) 10026 & SEND_LEN_CHECK1_LEN_VL4_MASK) << 10027 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT); 10028 } 10029 write_csr(dd, SEND_LEN_CHECK0, len1); 10030 write_csr(dd, SEND_LEN_CHECK1, len2); 10031 /* adjust kernel credit return thresholds based on new MTUs */ 10032 /* all kernel receive contexts have the same hdrqentsize */ 10033 for (i = 0; i < ppd->vls_supported; i++) { 10034 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50), 10035 sc_mtu_to_threshold(dd->vld[i].sc, 10036 dd->vld[i].mtu, 10037 dd->rcd[0]->rcvhdrqentsize)); 10038 for (j = 0; j < INIT_SC_PER_VL; j++) 10039 sc_set_cr_threshold( 10040 pio_select_send_context_vl(dd, j, i), 10041 thres); 10042 } 10043 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50), 10044 sc_mtu_to_threshold(dd->vld[15].sc, 10045 dd->vld[15].mtu, 10046 dd->rcd[0]->rcvhdrqentsize)); 10047 sc_set_cr_threshold(dd->vld[15].sc, thres); 10048 10049 /* Adjust maximum MTU for the port in DC */ 10050 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 : 10051 (ilog2(maxvlmtu >> 8) + 1); 10052 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG); 10053 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK; 10054 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) << 10055 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT; 10056 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1); 10057 } 10058 10059 static void set_lidlmc(struct hfi1_pportdata *ppd) 10060 { 10061 int i; 10062 u64 sreg = 0; 10063 struct hfi1_devdata *dd = ppd->dd; 10064 u32 mask = ~((1U << ppd->lmc) - 1); 10065 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1); 10066 u32 lid; 10067 10068 /* 10069 * Program 0 in CSR if port lid is extended. This prevents 10070 * 9B packets being sent out for large lids. 10071 */ 10072 lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid; 10073 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK 10074 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK); 10075 c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK) 10076 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) | 10077 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK) 10078 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT); 10079 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1); 10080 10081 /* 10082 * Iterate over all the send contexts and set their SLID check 10083 */ 10084 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) << 10085 SEND_CTXT_CHECK_SLID_MASK_SHIFT) | 10086 (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) << 10087 SEND_CTXT_CHECK_SLID_VALUE_SHIFT); 10088 10089 for (i = 0; i < dd->chip_send_contexts; i++) { 10090 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x", 10091 i, (u32)sreg); 10092 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg); 10093 } 10094 10095 /* Now we have to do the same thing for the sdma engines */ 10096 sdma_update_lmc(dd, mask, lid); 10097 } 10098 10099 static const char *state_completed_string(u32 completed) 10100 { 10101 static const char * const state_completed[] = { 10102 "EstablishComm", 10103 "OptimizeEQ", 10104 "VerifyCap" 10105 }; 10106 10107 if (completed < ARRAY_SIZE(state_completed)) 10108 return state_completed[completed]; 10109 10110 return "unknown"; 10111 } 10112 10113 static const char all_lanes_dead_timeout_expired[] = 10114 "All lanes were inactive – was the interconnect media removed?"; 10115 static const char tx_out_of_policy[] = 10116 "Passing lanes on local port do not meet the local link width policy"; 10117 static const char no_state_complete[] = 10118 "State timeout occurred before link partner completed the state"; 10119 static const char * const state_complete_reasons[] = { 10120 [0x00] = "Reason unknown", 10121 [0x01] = "Link was halted by driver, refer to LinkDownReason", 10122 [0x02] = "Link partner reported failure", 10123 [0x10] = "Unable to achieve frame sync on any lane", 10124 [0x11] = 10125 "Unable to find a common bit rate with the link partner", 10126 [0x12] = 10127 "Unable to achieve frame sync on sufficient lanes to meet the local link width policy", 10128 [0x13] = 10129 "Unable to identify preset equalization on sufficient lanes to meet the local link width policy", 10130 [0x14] = no_state_complete, 10131 [0x15] = 10132 "State timeout occurred before link partner identified equalization presets", 10133 [0x16] = 10134 "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy", 10135 [0x17] = tx_out_of_policy, 10136 [0x20] = all_lanes_dead_timeout_expired, 10137 [0x21] = 10138 "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy", 10139 [0x22] = no_state_complete, 10140 [0x23] = 10141 "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy", 10142 [0x24] = tx_out_of_policy, 10143 [0x30] = all_lanes_dead_timeout_expired, 10144 [0x31] = 10145 "State timeout occurred waiting for host to process received frames", 10146 [0x32] = no_state_complete, 10147 [0x33] = 10148 "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy", 10149 [0x34] = tx_out_of_policy, 10150 [0x35] = "Negotiated link width is mutually exclusive", 10151 [0x36] = 10152 "Timed out before receiving verifycap frames in VerifyCap.Exchange", 10153 [0x37] = "Unable to resolve secure data exchange", 10154 }; 10155 10156 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd, 10157 u32 code) 10158 { 10159 const char *str = NULL; 10160 10161 if (code < ARRAY_SIZE(state_complete_reasons)) 10162 str = state_complete_reasons[code]; 10163 10164 if (str) 10165 return str; 10166 return "Reserved"; 10167 } 10168 10169 /* describe the given last state complete frame */ 10170 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame, 10171 const char *prefix) 10172 { 10173 struct hfi1_devdata *dd = ppd->dd; 10174 u32 success; 10175 u32 state; 10176 u32 reason; 10177 u32 lanes; 10178 10179 /* 10180 * Decode frame: 10181 * [ 0: 0] - success 10182 * [ 3: 1] - state 10183 * [ 7: 4] - next state timeout 10184 * [15: 8] - reason code 10185 * [31:16] - lanes 10186 */ 10187 success = frame & 0x1; 10188 state = (frame >> 1) & 0x7; 10189 reason = (frame >> 8) & 0xff; 10190 lanes = (frame >> 16) & 0xffff; 10191 10192 dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n", 10193 prefix, frame); 10194 dd_dev_err(dd, " last reported state state: %s (0x%x)\n", 10195 state_completed_string(state), state); 10196 dd_dev_err(dd, " state successfully completed: %s\n", 10197 success ? "yes" : "no"); 10198 dd_dev_err(dd, " fail reason 0x%x: %s\n", 10199 reason, state_complete_reason_code_string(ppd, reason)); 10200 dd_dev_err(dd, " passing lane mask: 0x%x", lanes); 10201 } 10202 10203 /* 10204 * Read the last state complete frames and explain them. This routine 10205 * expects to be called if the link went down during link negotiation 10206 * and initialization (LNI). That is, anywhere between polling and link up. 10207 */ 10208 static void check_lni_states(struct hfi1_pportdata *ppd) 10209 { 10210 u32 last_local_state; 10211 u32 last_remote_state; 10212 10213 read_last_local_state(ppd->dd, &last_local_state); 10214 read_last_remote_state(ppd->dd, &last_remote_state); 10215 10216 /* 10217 * Don't report anything if there is nothing to report. A value of 10218 * 0 means the link was taken down while polling and there was no 10219 * training in-process. 10220 */ 10221 if (last_local_state == 0 && last_remote_state == 0) 10222 return; 10223 10224 decode_state_complete(ppd, last_local_state, "transmitted"); 10225 decode_state_complete(ppd, last_remote_state, "received"); 10226 } 10227 10228 /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */ 10229 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms) 10230 { 10231 u64 reg; 10232 unsigned long timeout; 10233 10234 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */ 10235 timeout = jiffies + msecs_to_jiffies(wait_ms); 10236 while (1) { 10237 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE); 10238 if (reg) 10239 break; 10240 if (time_after(jiffies, timeout)) { 10241 dd_dev_err(dd, 10242 "timeout waiting for LINK_TRANSFER_ACTIVE\n"); 10243 return -ETIMEDOUT; 10244 } 10245 udelay(2); 10246 } 10247 return 0; 10248 } 10249 10250 /* called when the logical link state is not down as it should be */ 10251 static void force_logical_link_state_down(struct hfi1_pportdata *ppd) 10252 { 10253 struct hfi1_devdata *dd = ppd->dd; 10254 10255 /* 10256 * Bring link up in LCB loopback 10257 */ 10258 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); 10259 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 10260 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 10261 10262 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 10263 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0); 10264 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 10265 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2); 10266 10267 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 10268 (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET); 10269 udelay(3); 10270 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1); 10271 write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 10272 10273 wait_link_transfer_active(dd, 100); 10274 10275 /* 10276 * Bring the link down again. 10277 */ 10278 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); 10279 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0); 10280 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0); 10281 10282 dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n"); 10283 } 10284 10285 /* 10286 * Helper for set_link_state(). Do not call except from that routine. 10287 * Expects ppd->hls_mutex to be held. 10288 * 10289 * @rem_reason value to be sent to the neighbor 10290 * 10291 * LinkDownReasons only set if transition succeeds. 10292 */ 10293 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) 10294 { 10295 struct hfi1_devdata *dd = ppd->dd; 10296 u32 previous_state; 10297 int offline_state_ret; 10298 int ret; 10299 10300 update_lcb_cache(dd); 10301 10302 previous_state = ppd->host_link_state; 10303 ppd->host_link_state = HLS_GOING_OFFLINE; 10304 10305 /* start offline transition */ 10306 ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE); 10307 10308 if (ret != HCMD_SUCCESS) { 10309 dd_dev_err(dd, 10310 "Failed to transition to Offline link state, return %d\n", 10311 ret); 10312 return -EINVAL; 10313 } 10314 if (ppd->offline_disabled_reason == 10315 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)) 10316 ppd->offline_disabled_reason = 10317 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 10318 10319 offline_state_ret = wait_phys_link_offline_substates(ppd, 10000); 10320 if (offline_state_ret < 0) 10321 return offline_state_ret; 10322 10323 /* Disabling AOC transmitters */ 10324 if (ppd->port_type == PORT_TYPE_QSFP && 10325 ppd->qsfp_info.limiting_active && 10326 qsfp_mod_present(ppd)) { 10327 int ret; 10328 10329 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT); 10330 if (ret == 0) { 10331 set_qsfp_tx(ppd, 0); 10332 release_chip_resource(dd, qsfp_resource(dd)); 10333 } else { 10334 /* not fatal, but should warn */ 10335 dd_dev_err(dd, 10336 "Unable to acquire lock to turn off QSFP TX\n"); 10337 } 10338 } 10339 10340 /* 10341 * Wait for the offline.Quiet transition if it hasn't happened yet. It 10342 * can take a while for the link to go down. 10343 */ 10344 if (offline_state_ret != PLS_OFFLINE_QUIET) { 10345 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000); 10346 if (ret < 0) 10347 return ret; 10348 } 10349 10350 /* 10351 * Now in charge of LCB - must be after the physical state is 10352 * offline.quiet and before host_link_state is changed. 10353 */ 10354 set_host_lcb_access(dd); 10355 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 10356 10357 /* make sure the logical state is also down */ 10358 ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000); 10359 if (ret) 10360 force_logical_link_state_down(ppd); 10361 10362 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */ 10363 update_statusp(ppd, IB_PORT_DOWN); 10364 10365 /* 10366 * The LNI has a mandatory wait time after the physical state 10367 * moves to Offline.Quiet. The wait time may be different 10368 * depending on how the link went down. The 8051 firmware 10369 * will observe the needed wait time and only move to ready 10370 * when that is completed. The largest of the quiet timeouts 10371 * is 6s, so wait that long and then at least 0.5s more for 10372 * other transitions, and another 0.5s for a buffer. 10373 */ 10374 ret = wait_fm_ready(dd, 7000); 10375 if (ret) { 10376 dd_dev_err(dd, 10377 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n"); 10378 /* state is really offline, so make it so */ 10379 ppd->host_link_state = HLS_DN_OFFLINE; 10380 return ret; 10381 } 10382 10383 /* 10384 * The state is now offline and the 8051 is ready to accept host 10385 * requests. 10386 * - change our state 10387 * - notify others if we were previously in a linkup state 10388 */ 10389 ppd->host_link_state = HLS_DN_OFFLINE; 10390 if (previous_state & HLS_UP) { 10391 /* went down while link was up */ 10392 handle_linkup_change(dd, 0); 10393 } else if (previous_state 10394 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 10395 /* went down while attempting link up */ 10396 check_lni_states(ppd); 10397 10398 /* The QSFP doesn't need to be reset on LNI failure */ 10399 ppd->qsfp_info.reset_needed = 0; 10400 } 10401 10402 /* the active link width (downgrade) is 0 on link down */ 10403 ppd->link_width_active = 0; 10404 ppd->link_width_downgrade_tx_active = 0; 10405 ppd->link_width_downgrade_rx_active = 0; 10406 ppd->current_egress_rate = 0; 10407 return 0; 10408 } 10409 10410 /* return the link state name */ 10411 static const char *link_state_name(u32 state) 10412 { 10413 const char *name; 10414 int n = ilog2(state); 10415 static const char * const names[] = { 10416 [__HLS_UP_INIT_BP] = "INIT", 10417 [__HLS_UP_ARMED_BP] = "ARMED", 10418 [__HLS_UP_ACTIVE_BP] = "ACTIVE", 10419 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF", 10420 [__HLS_DN_POLL_BP] = "POLL", 10421 [__HLS_DN_DISABLE_BP] = "DISABLE", 10422 [__HLS_DN_OFFLINE_BP] = "OFFLINE", 10423 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP", 10424 [__HLS_GOING_UP_BP] = "GOING_UP", 10425 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE", 10426 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN" 10427 }; 10428 10429 name = n < ARRAY_SIZE(names) ? names[n] : NULL; 10430 return name ? name : "unknown"; 10431 } 10432 10433 /* return the link state reason name */ 10434 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state) 10435 { 10436 if (state == HLS_UP_INIT) { 10437 switch (ppd->linkinit_reason) { 10438 case OPA_LINKINIT_REASON_LINKUP: 10439 return "(LINKUP)"; 10440 case OPA_LINKINIT_REASON_FLAPPING: 10441 return "(FLAPPING)"; 10442 case OPA_LINKINIT_OUTSIDE_POLICY: 10443 return "(OUTSIDE_POLICY)"; 10444 case OPA_LINKINIT_QUARANTINED: 10445 return "(QUARANTINED)"; 10446 case OPA_LINKINIT_INSUFIC_CAPABILITY: 10447 return "(INSUFIC_CAPABILITY)"; 10448 default: 10449 break; 10450 } 10451 } 10452 return ""; 10453 } 10454 10455 /* 10456 * driver_pstate - convert the driver's notion of a port's 10457 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*). 10458 * Return -1 (converted to a u32) to indicate error. 10459 */ 10460 u32 driver_pstate(struct hfi1_pportdata *ppd) 10461 { 10462 switch (ppd->host_link_state) { 10463 case HLS_UP_INIT: 10464 case HLS_UP_ARMED: 10465 case HLS_UP_ACTIVE: 10466 return IB_PORTPHYSSTATE_LINKUP; 10467 case HLS_DN_POLL: 10468 return IB_PORTPHYSSTATE_POLLING; 10469 case HLS_DN_DISABLE: 10470 return IB_PORTPHYSSTATE_DISABLED; 10471 case HLS_DN_OFFLINE: 10472 return OPA_PORTPHYSSTATE_OFFLINE; 10473 case HLS_VERIFY_CAP: 10474 return IB_PORTPHYSSTATE_POLLING; 10475 case HLS_GOING_UP: 10476 return IB_PORTPHYSSTATE_POLLING; 10477 case HLS_GOING_OFFLINE: 10478 return OPA_PORTPHYSSTATE_OFFLINE; 10479 case HLS_LINK_COOLDOWN: 10480 return OPA_PORTPHYSSTATE_OFFLINE; 10481 case HLS_DN_DOWNDEF: 10482 default: 10483 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10484 ppd->host_link_state); 10485 return -1; 10486 } 10487 } 10488 10489 /* 10490 * driver_lstate - convert the driver's notion of a port's 10491 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1 10492 * (converted to a u32) to indicate error. 10493 */ 10494 u32 driver_lstate(struct hfi1_pportdata *ppd) 10495 { 10496 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN)) 10497 return IB_PORT_DOWN; 10498 10499 switch (ppd->host_link_state & HLS_UP) { 10500 case HLS_UP_INIT: 10501 return IB_PORT_INIT; 10502 case HLS_UP_ARMED: 10503 return IB_PORT_ARMED; 10504 case HLS_UP_ACTIVE: 10505 return IB_PORT_ACTIVE; 10506 default: 10507 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10508 ppd->host_link_state); 10509 return -1; 10510 } 10511 } 10512 10513 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason, 10514 u8 neigh_reason, u8 rem_reason) 10515 { 10516 if (ppd->local_link_down_reason.latest == 0 && 10517 ppd->neigh_link_down_reason.latest == 0) { 10518 ppd->local_link_down_reason.latest = lcl_reason; 10519 ppd->neigh_link_down_reason.latest = neigh_reason; 10520 ppd->remote_link_down_reason = rem_reason; 10521 } 10522 } 10523 10524 /* 10525 * Verify if BCT for data VLs is non-zero. 10526 */ 10527 static inline bool data_vls_operational(struct hfi1_pportdata *ppd) 10528 { 10529 return !!ppd->actual_vls_operational; 10530 } 10531 10532 /* 10533 * Change the physical and/or logical link state. 10534 * 10535 * Do not call this routine while inside an interrupt. It contains 10536 * calls to routines that can take multiple seconds to finish. 10537 * 10538 * Returns 0 on success, -errno on failure. 10539 */ 10540 int set_link_state(struct hfi1_pportdata *ppd, u32 state) 10541 { 10542 struct hfi1_devdata *dd = ppd->dd; 10543 struct ib_event event = {.device = NULL}; 10544 int ret1, ret = 0; 10545 int orig_new_state, poll_bounce; 10546 10547 mutex_lock(&ppd->hls_lock); 10548 10549 orig_new_state = state; 10550 if (state == HLS_DN_DOWNDEF) 10551 state = HLS_DEFAULT; 10552 10553 /* interpret poll -> poll as a link bounce */ 10554 poll_bounce = ppd->host_link_state == HLS_DN_POLL && 10555 state == HLS_DN_POLL; 10556 10557 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__, 10558 link_state_name(ppd->host_link_state), 10559 link_state_name(orig_new_state), 10560 poll_bounce ? "(bounce) " : "", 10561 link_state_reason_name(ppd, state)); 10562 10563 /* 10564 * If we're going to a (HLS_*) link state that implies the logical 10565 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then 10566 * reset is_sm_config_started to 0. 10567 */ 10568 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE))) 10569 ppd->is_sm_config_started = 0; 10570 10571 /* 10572 * Do nothing if the states match. Let a poll to poll link bounce 10573 * go through. 10574 */ 10575 if (ppd->host_link_state == state && !poll_bounce) 10576 goto done; 10577 10578 switch (state) { 10579 case HLS_UP_INIT: 10580 if (ppd->host_link_state == HLS_DN_POLL && 10581 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) { 10582 /* 10583 * Quick link up jumps from polling to here. 10584 * 10585 * Whether in normal or loopback mode, the 10586 * simulator jumps from polling to link up. 10587 * Accept that here. 10588 */ 10589 /* OK */ 10590 } else if (ppd->host_link_state != HLS_GOING_UP) { 10591 goto unexpected; 10592 } 10593 10594 /* 10595 * Wait for Link_Up physical state. 10596 * Physical and Logical states should already be 10597 * be transitioned to LinkUp and LinkInit respectively. 10598 */ 10599 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000); 10600 if (ret) { 10601 dd_dev_err(dd, 10602 "%s: physical state did not change to LINK-UP\n", 10603 __func__); 10604 break; 10605 } 10606 10607 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000); 10608 if (ret) { 10609 dd_dev_err(dd, 10610 "%s: logical state did not change to INIT\n", 10611 __func__); 10612 break; 10613 } 10614 10615 /* clear old transient LINKINIT_REASON code */ 10616 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR) 10617 ppd->linkinit_reason = 10618 OPA_LINKINIT_REASON_LINKUP; 10619 10620 /* enable the port */ 10621 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 10622 10623 handle_linkup_change(dd, 1); 10624 ppd->host_link_state = HLS_UP_INIT; 10625 update_statusp(ppd, IB_PORT_INIT); 10626 break; 10627 case HLS_UP_ARMED: 10628 if (ppd->host_link_state != HLS_UP_INIT) 10629 goto unexpected; 10630 10631 if (!data_vls_operational(ppd)) { 10632 dd_dev_err(dd, 10633 "%s: data VLs not operational\n", __func__); 10634 ret = -EINVAL; 10635 break; 10636 } 10637 10638 set_logical_state(dd, LSTATE_ARMED); 10639 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000); 10640 if (ret) { 10641 dd_dev_err(dd, 10642 "%s: logical state did not change to ARMED\n", 10643 __func__); 10644 break; 10645 } 10646 ppd->host_link_state = HLS_UP_ARMED; 10647 update_statusp(ppd, IB_PORT_ARMED); 10648 /* 10649 * The simulator does not currently implement SMA messages, 10650 * so neighbor_normal is not set. Set it here when we first 10651 * move to Armed. 10652 */ 10653 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 10654 ppd->neighbor_normal = 1; 10655 break; 10656 case HLS_UP_ACTIVE: 10657 if (ppd->host_link_state != HLS_UP_ARMED) 10658 goto unexpected; 10659 10660 set_logical_state(dd, LSTATE_ACTIVE); 10661 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000); 10662 if (ret) { 10663 dd_dev_err(dd, 10664 "%s: logical state did not change to ACTIVE\n", 10665 __func__); 10666 } else { 10667 /* tell all engines to go running */ 10668 sdma_all_running(dd); 10669 ppd->host_link_state = HLS_UP_ACTIVE; 10670 update_statusp(ppd, IB_PORT_ACTIVE); 10671 10672 /* Signal the IB layer that the port has went active */ 10673 event.device = &dd->verbs_dev.rdi.ibdev; 10674 event.element.port_num = ppd->port; 10675 event.event = IB_EVENT_PORT_ACTIVE; 10676 } 10677 break; 10678 case HLS_DN_POLL: 10679 if ((ppd->host_link_state == HLS_DN_DISABLE || 10680 ppd->host_link_state == HLS_DN_OFFLINE) && 10681 dd->dc_shutdown) 10682 dc_start(dd); 10683 /* Hand LED control to the DC */ 10684 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 10685 10686 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10687 u8 tmp = ppd->link_enabled; 10688 10689 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10690 if (ret) { 10691 ppd->link_enabled = tmp; 10692 break; 10693 } 10694 ppd->remote_link_down_reason = 0; 10695 10696 if (ppd->driver_link_ready) 10697 ppd->link_enabled = 1; 10698 } 10699 10700 set_all_slowpath(ppd->dd); 10701 ret = set_local_link_attributes(ppd); 10702 if (ret) 10703 break; 10704 10705 ppd->port_error_action = 0; 10706 ppd->host_link_state = HLS_DN_POLL; 10707 10708 if (quick_linkup) { 10709 /* quick linkup does not go into polling */ 10710 ret = do_quick_linkup(dd); 10711 } else { 10712 ret1 = set_physical_link_state(dd, PLS_POLLING); 10713 if (ret1 != HCMD_SUCCESS) { 10714 dd_dev_err(dd, 10715 "Failed to transition to Polling link state, return 0x%x\n", 10716 ret1); 10717 ret = -EINVAL; 10718 } 10719 } 10720 ppd->offline_disabled_reason = 10721 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE); 10722 /* 10723 * If an error occurred above, go back to offline. The 10724 * caller may reschedule another attempt. 10725 */ 10726 if (ret) 10727 goto_offline(ppd, 0); 10728 else 10729 log_physical_state(ppd, PLS_POLLING); 10730 break; 10731 case HLS_DN_DISABLE: 10732 /* link is disabled */ 10733 ppd->link_enabled = 0; 10734 10735 /* allow any state to transition to disabled */ 10736 10737 /* must transition to offline first */ 10738 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10739 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10740 if (ret) 10741 break; 10742 ppd->remote_link_down_reason = 0; 10743 } 10744 10745 if (!dd->dc_shutdown) { 10746 ret1 = set_physical_link_state(dd, PLS_DISABLED); 10747 if (ret1 != HCMD_SUCCESS) { 10748 dd_dev_err(dd, 10749 "Failed to transition to Disabled link state, return 0x%x\n", 10750 ret1); 10751 ret = -EINVAL; 10752 break; 10753 } 10754 ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000); 10755 if (ret) { 10756 dd_dev_err(dd, 10757 "%s: physical state did not change to DISABLED\n", 10758 __func__); 10759 break; 10760 } 10761 dc_shutdown(dd); 10762 } 10763 ppd->host_link_state = HLS_DN_DISABLE; 10764 break; 10765 case HLS_DN_OFFLINE: 10766 if (ppd->host_link_state == HLS_DN_DISABLE) 10767 dc_start(dd); 10768 10769 /* allow any state to transition to offline */ 10770 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10771 if (!ret) 10772 ppd->remote_link_down_reason = 0; 10773 break; 10774 case HLS_VERIFY_CAP: 10775 if (ppd->host_link_state != HLS_DN_POLL) 10776 goto unexpected; 10777 ppd->host_link_state = HLS_VERIFY_CAP; 10778 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP); 10779 break; 10780 case HLS_GOING_UP: 10781 if (ppd->host_link_state != HLS_VERIFY_CAP) 10782 goto unexpected; 10783 10784 ret1 = set_physical_link_state(dd, PLS_LINKUP); 10785 if (ret1 != HCMD_SUCCESS) { 10786 dd_dev_err(dd, 10787 "Failed to transition to link up state, return 0x%x\n", 10788 ret1); 10789 ret = -EINVAL; 10790 break; 10791 } 10792 ppd->host_link_state = HLS_GOING_UP; 10793 break; 10794 10795 case HLS_GOING_OFFLINE: /* transient within goto_offline() */ 10796 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */ 10797 default: 10798 dd_dev_info(dd, "%s: state 0x%x: not supported\n", 10799 __func__, state); 10800 ret = -EINVAL; 10801 break; 10802 } 10803 10804 goto done; 10805 10806 unexpected: 10807 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n", 10808 __func__, link_state_name(ppd->host_link_state), 10809 link_state_name(state)); 10810 ret = -EINVAL; 10811 10812 done: 10813 mutex_unlock(&ppd->hls_lock); 10814 10815 if (event.device) 10816 ib_dispatch_event(&event); 10817 10818 return ret; 10819 } 10820 10821 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val) 10822 { 10823 u64 reg; 10824 int ret = 0; 10825 10826 switch (which) { 10827 case HFI1_IB_CFG_LIDLMC: 10828 set_lidlmc(ppd); 10829 break; 10830 case HFI1_IB_CFG_VL_HIGH_LIMIT: 10831 /* 10832 * The VL Arbitrator high limit is sent in units of 4k 10833 * bytes, while HFI stores it in units of 64 bytes. 10834 */ 10835 val *= 4096 / 64; 10836 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK) 10837 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT; 10838 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg); 10839 break; 10840 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 10841 /* HFI only supports POLL as the default link down state */ 10842 if (val != HLS_DN_POLL) 10843 ret = -EINVAL; 10844 break; 10845 case HFI1_IB_CFG_OP_VLS: 10846 if (ppd->vls_operational != val) { 10847 ppd->vls_operational = val; 10848 if (!ppd->port) 10849 ret = -EINVAL; 10850 } 10851 break; 10852 /* 10853 * For link width, link width downgrade, and speed enable, always AND 10854 * the setting with what is actually supported. This has two benefits. 10855 * First, enabled can't have unsupported values, no matter what the 10856 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean 10857 * "fill in with your supported value" have all the bits in the 10858 * field set, so simply ANDing with supported has the desired result. 10859 */ 10860 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */ 10861 ppd->link_width_enabled = val & ppd->link_width_supported; 10862 break; 10863 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */ 10864 ppd->link_width_downgrade_enabled = 10865 val & ppd->link_width_downgrade_supported; 10866 break; 10867 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 10868 ppd->link_speed_enabled = val & ppd->link_speed_supported; 10869 break; 10870 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 10871 /* 10872 * HFI does not follow IB specs, save this value 10873 * so we can report it, if asked. 10874 */ 10875 ppd->overrun_threshold = val; 10876 break; 10877 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 10878 /* 10879 * HFI does not follow IB specs, save this value 10880 * so we can report it, if asked. 10881 */ 10882 ppd->phy_error_threshold = val; 10883 break; 10884 10885 case HFI1_IB_CFG_MTU: 10886 set_send_length(ppd); 10887 break; 10888 10889 case HFI1_IB_CFG_PKEYS: 10890 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 10891 set_partition_keys(ppd); 10892 break; 10893 10894 default: 10895 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 10896 dd_dev_info(ppd->dd, 10897 "%s: which %s, val 0x%x: not implemented\n", 10898 __func__, ib_cfg_name(which), val); 10899 break; 10900 } 10901 return ret; 10902 } 10903 10904 /* begin functions related to vl arbitration table caching */ 10905 static void init_vl_arb_caches(struct hfi1_pportdata *ppd) 10906 { 10907 int i; 10908 10909 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 10910 VL_ARB_LOW_PRIO_TABLE_SIZE); 10911 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 10912 VL_ARB_HIGH_PRIO_TABLE_SIZE); 10913 10914 /* 10915 * Note that we always return values directly from the 10916 * 'vl_arb_cache' (and do no CSR reads) in response to a 10917 * 'Get(VLArbTable)'. This is obviously correct after a 10918 * 'Set(VLArbTable)', since the cache will then be up to 10919 * date. But it's also correct prior to any 'Set(VLArbTable)' 10920 * since then both the cache, and the relevant h/w registers 10921 * will be zeroed. 10922 */ 10923 10924 for (i = 0; i < MAX_PRIO_TABLE; i++) 10925 spin_lock_init(&ppd->vl_arb_cache[i].lock); 10926 } 10927 10928 /* 10929 * vl_arb_lock_cache 10930 * 10931 * All other vl_arb_* functions should be called only after locking 10932 * the cache. 10933 */ 10934 static inline struct vl_arb_cache * 10935 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx) 10936 { 10937 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE) 10938 return NULL; 10939 spin_lock(&ppd->vl_arb_cache[idx].lock); 10940 return &ppd->vl_arb_cache[idx]; 10941 } 10942 10943 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx) 10944 { 10945 spin_unlock(&ppd->vl_arb_cache[idx].lock); 10946 } 10947 10948 static void vl_arb_get_cache(struct vl_arb_cache *cache, 10949 struct ib_vl_weight_elem *vl) 10950 { 10951 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl)); 10952 } 10953 10954 static void vl_arb_set_cache(struct vl_arb_cache *cache, 10955 struct ib_vl_weight_elem *vl) 10956 { 10957 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 10958 } 10959 10960 static int vl_arb_match_cache(struct vl_arb_cache *cache, 10961 struct ib_vl_weight_elem *vl) 10962 { 10963 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 10964 } 10965 10966 /* end functions related to vl arbitration table caching */ 10967 10968 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target, 10969 u32 size, struct ib_vl_weight_elem *vl) 10970 { 10971 struct hfi1_devdata *dd = ppd->dd; 10972 u64 reg; 10973 unsigned int i, is_up = 0; 10974 int drain, ret = 0; 10975 10976 mutex_lock(&ppd->hls_lock); 10977 10978 if (ppd->host_link_state & HLS_UP) 10979 is_up = 1; 10980 10981 drain = !is_ax(dd) && is_up; 10982 10983 if (drain) 10984 /* 10985 * Before adjusting VL arbitration weights, empty per-VL 10986 * FIFOs, otherwise a packet whose VL weight is being 10987 * set to 0 could get stuck in a FIFO with no chance to 10988 * egress. 10989 */ 10990 ret = stop_drain_data_vls(dd); 10991 10992 if (ret) { 10993 dd_dev_err( 10994 dd, 10995 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n", 10996 __func__); 10997 goto err; 10998 } 10999 11000 for (i = 0; i < size; i++, vl++) { 11001 /* 11002 * NOTE: The low priority shift and mask are used here, but 11003 * they are the same for both the low and high registers. 11004 */ 11005 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK) 11006 << SEND_LOW_PRIORITY_LIST_VL_SHIFT) 11007 | (((u64)vl->weight 11008 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK) 11009 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT); 11010 write_csr(dd, target + (i * 8), reg); 11011 } 11012 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE); 11013 11014 if (drain) 11015 open_fill_data_vls(dd); /* reopen all VLs */ 11016 11017 err: 11018 mutex_unlock(&ppd->hls_lock); 11019 11020 return ret; 11021 } 11022 11023 /* 11024 * Read one credit merge VL register. 11025 */ 11026 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr, 11027 struct vl_limit *vll) 11028 { 11029 u64 reg = read_csr(dd, csr); 11030 11031 vll->dedicated = cpu_to_be16( 11032 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT) 11033 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK); 11034 vll->shared = cpu_to_be16( 11035 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT) 11036 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK); 11037 } 11038 11039 /* 11040 * Read the current credit merge limits. 11041 */ 11042 static int get_buffer_control(struct hfi1_devdata *dd, 11043 struct buffer_control *bc, u16 *overall_limit) 11044 { 11045 u64 reg; 11046 int i; 11047 11048 /* not all entries are filled in */ 11049 memset(bc, 0, sizeof(*bc)); 11050 11051 /* OPA and HFI have a 1-1 mapping */ 11052 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11053 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]); 11054 11055 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */ 11056 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]); 11057 11058 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11059 bc->overall_shared_limit = cpu_to_be16( 11060 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) 11061 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK); 11062 if (overall_limit) 11063 *overall_limit = (reg 11064 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) 11065 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK; 11066 return sizeof(struct buffer_control); 11067 } 11068 11069 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 11070 { 11071 u64 reg; 11072 int i; 11073 11074 /* each register contains 16 SC->VLnt mappings, 4 bits each */ 11075 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0); 11076 for (i = 0; i < sizeof(u64); i++) { 11077 u8 byte = *(((u8 *)®) + i); 11078 11079 dp->vlnt[2 * i] = byte & 0xf; 11080 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4; 11081 } 11082 11083 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16); 11084 for (i = 0; i < sizeof(u64); i++) { 11085 u8 byte = *(((u8 *)®) + i); 11086 11087 dp->vlnt[16 + (2 * i)] = byte & 0xf; 11088 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4; 11089 } 11090 return sizeof(struct sc2vlnt); 11091 } 11092 11093 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems, 11094 struct ib_vl_weight_elem *vl) 11095 { 11096 unsigned int i; 11097 11098 for (i = 0; i < nelems; i++, vl++) { 11099 vl->vl = 0xf; 11100 vl->weight = 0; 11101 } 11102 } 11103 11104 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 11105 { 11106 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, 11107 DC_SC_VL_VAL(15_0, 11108 0, dp->vlnt[0] & 0xf, 11109 1, dp->vlnt[1] & 0xf, 11110 2, dp->vlnt[2] & 0xf, 11111 3, dp->vlnt[3] & 0xf, 11112 4, dp->vlnt[4] & 0xf, 11113 5, dp->vlnt[5] & 0xf, 11114 6, dp->vlnt[6] & 0xf, 11115 7, dp->vlnt[7] & 0xf, 11116 8, dp->vlnt[8] & 0xf, 11117 9, dp->vlnt[9] & 0xf, 11118 10, dp->vlnt[10] & 0xf, 11119 11, dp->vlnt[11] & 0xf, 11120 12, dp->vlnt[12] & 0xf, 11121 13, dp->vlnt[13] & 0xf, 11122 14, dp->vlnt[14] & 0xf, 11123 15, dp->vlnt[15] & 0xf)); 11124 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, 11125 DC_SC_VL_VAL(31_16, 11126 16, dp->vlnt[16] & 0xf, 11127 17, dp->vlnt[17] & 0xf, 11128 18, dp->vlnt[18] & 0xf, 11129 19, dp->vlnt[19] & 0xf, 11130 20, dp->vlnt[20] & 0xf, 11131 21, dp->vlnt[21] & 0xf, 11132 22, dp->vlnt[22] & 0xf, 11133 23, dp->vlnt[23] & 0xf, 11134 24, dp->vlnt[24] & 0xf, 11135 25, dp->vlnt[25] & 0xf, 11136 26, dp->vlnt[26] & 0xf, 11137 27, dp->vlnt[27] & 0xf, 11138 28, dp->vlnt[28] & 0xf, 11139 29, dp->vlnt[29] & 0xf, 11140 30, dp->vlnt[30] & 0xf, 11141 31, dp->vlnt[31] & 0xf)); 11142 } 11143 11144 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what, 11145 u16 limit) 11146 { 11147 if (limit != 0) 11148 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n", 11149 what, (int)limit, idx); 11150 } 11151 11152 /* change only the shared limit portion of SendCmGLobalCredit */ 11153 static void set_global_shared(struct hfi1_devdata *dd, u16 limit) 11154 { 11155 u64 reg; 11156 11157 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11158 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK; 11159 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT; 11160 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 11161 } 11162 11163 /* change only the total credit limit portion of SendCmGLobalCredit */ 11164 static void set_global_limit(struct hfi1_devdata *dd, u16 limit) 11165 { 11166 u64 reg; 11167 11168 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11169 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK; 11170 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 11171 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 11172 } 11173 11174 /* set the given per-VL shared limit */ 11175 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit) 11176 { 11177 u64 reg; 11178 u32 addr; 11179 11180 if (vl < TXE_NUM_DATA_VL) 11181 addr = SEND_CM_CREDIT_VL + (8 * vl); 11182 else 11183 addr = SEND_CM_CREDIT_VL15; 11184 11185 reg = read_csr(dd, addr); 11186 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK; 11187 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT; 11188 write_csr(dd, addr, reg); 11189 } 11190 11191 /* set the given per-VL dedicated limit */ 11192 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit) 11193 { 11194 u64 reg; 11195 u32 addr; 11196 11197 if (vl < TXE_NUM_DATA_VL) 11198 addr = SEND_CM_CREDIT_VL + (8 * vl); 11199 else 11200 addr = SEND_CM_CREDIT_VL15; 11201 11202 reg = read_csr(dd, addr); 11203 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK; 11204 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT; 11205 write_csr(dd, addr, reg); 11206 } 11207 11208 /* spin until the given per-VL status mask bits clear */ 11209 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask, 11210 const char *which) 11211 { 11212 unsigned long timeout; 11213 u64 reg; 11214 11215 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT); 11216 while (1) { 11217 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask; 11218 11219 if (reg == 0) 11220 return; /* success */ 11221 if (time_after(jiffies, timeout)) 11222 break; /* timed out */ 11223 udelay(1); 11224 } 11225 11226 dd_dev_err(dd, 11227 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n", 11228 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg); 11229 /* 11230 * If this occurs, it is likely there was a credit loss on the link. 11231 * The only recovery from that is a link bounce. 11232 */ 11233 dd_dev_err(dd, 11234 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n"); 11235 } 11236 11237 /* 11238 * The number of credits on the VLs may be changed while everything 11239 * is "live", but the following algorithm must be followed due to 11240 * how the hardware is actually implemented. In particular, 11241 * Return_Credit_Status[] is the only correct status check. 11242 * 11243 * if (reducing Global_Shared_Credit_Limit or any shared limit changing) 11244 * set Global_Shared_Credit_Limit = 0 11245 * use_all_vl = 1 11246 * mask0 = all VLs that are changing either dedicated or shared limits 11247 * set Shared_Limit[mask0] = 0 11248 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0 11249 * if (changing any dedicated limit) 11250 * mask1 = all VLs that are lowering dedicated limits 11251 * lower Dedicated_Limit[mask1] 11252 * spin until Return_Credit_Status[mask1] == 0 11253 * raise Dedicated_Limits 11254 * raise Shared_Limits 11255 * raise Global_Shared_Credit_Limit 11256 * 11257 * lower = if the new limit is lower, set the limit to the new value 11258 * raise = if the new limit is higher than the current value (may be changed 11259 * earlier in the algorithm), set the new limit to the new value 11260 */ 11261 int set_buffer_control(struct hfi1_pportdata *ppd, 11262 struct buffer_control *new_bc) 11263 { 11264 struct hfi1_devdata *dd = ppd->dd; 11265 u64 changing_mask, ld_mask, stat_mask; 11266 int change_count; 11267 int i, use_all_mask; 11268 int this_shared_changing; 11269 int vl_count = 0, ret; 11270 /* 11271 * A0: add the variable any_shared_limit_changing below and in the 11272 * algorithm above. If removing A0 support, it can be removed. 11273 */ 11274 int any_shared_limit_changing; 11275 struct buffer_control cur_bc; 11276 u8 changing[OPA_MAX_VLS]; 11277 u8 lowering_dedicated[OPA_MAX_VLS]; 11278 u16 cur_total; 11279 u32 new_total = 0; 11280 const u64 all_mask = 11281 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK 11282 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK 11283 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK 11284 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK 11285 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK 11286 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK 11287 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK 11288 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK 11289 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK; 11290 11291 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15) 11292 #define NUM_USABLE_VLS 16 /* look at VL15 and less */ 11293 11294 /* find the new total credits, do sanity check on unused VLs */ 11295 for (i = 0; i < OPA_MAX_VLS; i++) { 11296 if (valid_vl(i)) { 11297 new_total += be16_to_cpu(new_bc->vl[i].dedicated); 11298 continue; 11299 } 11300 nonzero_msg(dd, i, "dedicated", 11301 be16_to_cpu(new_bc->vl[i].dedicated)); 11302 nonzero_msg(dd, i, "shared", 11303 be16_to_cpu(new_bc->vl[i].shared)); 11304 new_bc->vl[i].dedicated = 0; 11305 new_bc->vl[i].shared = 0; 11306 } 11307 new_total += be16_to_cpu(new_bc->overall_shared_limit); 11308 11309 /* fetch the current values */ 11310 get_buffer_control(dd, &cur_bc, &cur_total); 11311 11312 /* 11313 * Create the masks we will use. 11314 */ 11315 memset(changing, 0, sizeof(changing)); 11316 memset(lowering_dedicated, 0, sizeof(lowering_dedicated)); 11317 /* 11318 * NOTE: Assumes that the individual VL bits are adjacent and in 11319 * increasing order 11320 */ 11321 stat_mask = 11322 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK; 11323 changing_mask = 0; 11324 ld_mask = 0; 11325 change_count = 0; 11326 any_shared_limit_changing = 0; 11327 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) { 11328 if (!valid_vl(i)) 11329 continue; 11330 this_shared_changing = new_bc->vl[i].shared 11331 != cur_bc.vl[i].shared; 11332 if (this_shared_changing) 11333 any_shared_limit_changing = 1; 11334 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated || 11335 this_shared_changing) { 11336 changing[i] = 1; 11337 changing_mask |= stat_mask; 11338 change_count++; 11339 } 11340 if (be16_to_cpu(new_bc->vl[i].dedicated) < 11341 be16_to_cpu(cur_bc.vl[i].dedicated)) { 11342 lowering_dedicated[i] = 1; 11343 ld_mask |= stat_mask; 11344 } 11345 } 11346 11347 /* bracket the credit change with a total adjustment */ 11348 if (new_total > cur_total) 11349 set_global_limit(dd, new_total); 11350 11351 /* 11352 * Start the credit change algorithm. 11353 */ 11354 use_all_mask = 0; 11355 if ((be16_to_cpu(new_bc->overall_shared_limit) < 11356 be16_to_cpu(cur_bc.overall_shared_limit)) || 11357 (is_ax(dd) && any_shared_limit_changing)) { 11358 set_global_shared(dd, 0); 11359 cur_bc.overall_shared_limit = 0; 11360 use_all_mask = 1; 11361 } 11362 11363 for (i = 0; i < NUM_USABLE_VLS; i++) { 11364 if (!valid_vl(i)) 11365 continue; 11366 11367 if (changing[i]) { 11368 set_vl_shared(dd, i, 0); 11369 cur_bc.vl[i].shared = 0; 11370 } 11371 } 11372 11373 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask, 11374 "shared"); 11375 11376 if (change_count > 0) { 11377 for (i = 0; i < NUM_USABLE_VLS; i++) { 11378 if (!valid_vl(i)) 11379 continue; 11380 11381 if (lowering_dedicated[i]) { 11382 set_vl_dedicated(dd, i, 11383 be16_to_cpu(new_bc-> 11384 vl[i].dedicated)); 11385 cur_bc.vl[i].dedicated = 11386 new_bc->vl[i].dedicated; 11387 } 11388 } 11389 11390 wait_for_vl_status_clear(dd, ld_mask, "dedicated"); 11391 11392 /* now raise all dedicated that are going up */ 11393 for (i = 0; i < NUM_USABLE_VLS; i++) { 11394 if (!valid_vl(i)) 11395 continue; 11396 11397 if (be16_to_cpu(new_bc->vl[i].dedicated) > 11398 be16_to_cpu(cur_bc.vl[i].dedicated)) 11399 set_vl_dedicated(dd, i, 11400 be16_to_cpu(new_bc-> 11401 vl[i].dedicated)); 11402 } 11403 } 11404 11405 /* next raise all shared that are going up */ 11406 for (i = 0; i < NUM_USABLE_VLS; i++) { 11407 if (!valid_vl(i)) 11408 continue; 11409 11410 if (be16_to_cpu(new_bc->vl[i].shared) > 11411 be16_to_cpu(cur_bc.vl[i].shared)) 11412 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared)); 11413 } 11414 11415 /* finally raise the global shared */ 11416 if (be16_to_cpu(new_bc->overall_shared_limit) > 11417 be16_to_cpu(cur_bc.overall_shared_limit)) 11418 set_global_shared(dd, 11419 be16_to_cpu(new_bc->overall_shared_limit)); 11420 11421 /* bracket the credit change with a total adjustment */ 11422 if (new_total < cur_total) 11423 set_global_limit(dd, new_total); 11424 11425 /* 11426 * Determine the actual number of operational VLS using the number of 11427 * dedicated and shared credits for each VL. 11428 */ 11429 if (change_count > 0) { 11430 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11431 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 || 11432 be16_to_cpu(new_bc->vl[i].shared) > 0) 11433 vl_count++; 11434 ppd->actual_vls_operational = vl_count; 11435 ret = sdma_map_init(dd, ppd->port - 1, vl_count ? 11436 ppd->actual_vls_operational : 11437 ppd->vls_operational, 11438 NULL); 11439 if (ret == 0) 11440 ret = pio_map_init(dd, ppd->port - 1, vl_count ? 11441 ppd->actual_vls_operational : 11442 ppd->vls_operational, NULL); 11443 if (ret) 11444 return ret; 11445 } 11446 return 0; 11447 } 11448 11449 /* 11450 * Read the given fabric manager table. Return the size of the 11451 * table (in bytes) on success, and a negative error code on 11452 * failure. 11453 */ 11454 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t) 11455 11456 { 11457 int size; 11458 struct vl_arb_cache *vlc; 11459 11460 switch (which) { 11461 case FM_TBL_VL_HIGH_ARB: 11462 size = 256; 11463 /* 11464 * OPA specifies 128 elements (of 2 bytes each), though 11465 * HFI supports only 16 elements in h/w. 11466 */ 11467 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11468 vl_arb_get_cache(vlc, t); 11469 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11470 break; 11471 case FM_TBL_VL_LOW_ARB: 11472 size = 256; 11473 /* 11474 * OPA specifies 128 elements (of 2 bytes each), though 11475 * HFI supports only 16 elements in h/w. 11476 */ 11477 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11478 vl_arb_get_cache(vlc, t); 11479 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11480 break; 11481 case FM_TBL_BUFFER_CONTROL: 11482 size = get_buffer_control(ppd->dd, t, NULL); 11483 break; 11484 case FM_TBL_SC2VLNT: 11485 size = get_sc2vlnt(ppd->dd, t); 11486 break; 11487 case FM_TBL_VL_PREEMPT_ELEMS: 11488 size = 256; 11489 /* OPA specifies 128 elements, of 2 bytes each */ 11490 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t); 11491 break; 11492 case FM_TBL_VL_PREEMPT_MATRIX: 11493 size = 256; 11494 /* 11495 * OPA specifies that this is the same size as the VL 11496 * arbitration tables (i.e., 256 bytes). 11497 */ 11498 break; 11499 default: 11500 return -EINVAL; 11501 } 11502 return size; 11503 } 11504 11505 /* 11506 * Write the given fabric manager table. 11507 */ 11508 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t) 11509 { 11510 int ret = 0; 11511 struct vl_arb_cache *vlc; 11512 11513 switch (which) { 11514 case FM_TBL_VL_HIGH_ARB: 11515 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11516 if (vl_arb_match_cache(vlc, t)) { 11517 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11518 break; 11519 } 11520 vl_arb_set_cache(vlc, t); 11521 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11522 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST, 11523 VL_ARB_HIGH_PRIO_TABLE_SIZE, t); 11524 break; 11525 case FM_TBL_VL_LOW_ARB: 11526 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11527 if (vl_arb_match_cache(vlc, t)) { 11528 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11529 break; 11530 } 11531 vl_arb_set_cache(vlc, t); 11532 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11533 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST, 11534 VL_ARB_LOW_PRIO_TABLE_SIZE, t); 11535 break; 11536 case FM_TBL_BUFFER_CONTROL: 11537 ret = set_buffer_control(ppd, t); 11538 break; 11539 case FM_TBL_SC2VLNT: 11540 set_sc2vlnt(ppd->dd, t); 11541 break; 11542 default: 11543 ret = -EINVAL; 11544 } 11545 return ret; 11546 } 11547 11548 /* 11549 * Disable all data VLs. 11550 * 11551 * Return 0 if disabled, non-zero if the VLs cannot be disabled. 11552 */ 11553 static int disable_data_vls(struct hfi1_devdata *dd) 11554 { 11555 if (is_ax(dd)) 11556 return 1; 11557 11558 pio_send_control(dd, PSC_DATA_VL_DISABLE); 11559 11560 return 0; 11561 } 11562 11563 /* 11564 * open_fill_data_vls() - the counterpart to stop_drain_data_vls(). 11565 * Just re-enables all data VLs (the "fill" part happens 11566 * automatically - the name was chosen for symmetry with 11567 * stop_drain_data_vls()). 11568 * 11569 * Return 0 if successful, non-zero if the VLs cannot be enabled. 11570 */ 11571 int open_fill_data_vls(struct hfi1_devdata *dd) 11572 { 11573 if (is_ax(dd)) 11574 return 1; 11575 11576 pio_send_control(dd, PSC_DATA_VL_ENABLE); 11577 11578 return 0; 11579 } 11580 11581 /* 11582 * drain_data_vls() - assumes that disable_data_vls() has been called, 11583 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA 11584 * engines to drop to 0. 11585 */ 11586 static void drain_data_vls(struct hfi1_devdata *dd) 11587 { 11588 sc_wait(dd); 11589 sdma_wait(dd); 11590 pause_for_credit_return(dd); 11591 } 11592 11593 /* 11594 * stop_drain_data_vls() - disable, then drain all per-VL fifos. 11595 * 11596 * Use open_fill_data_vls() to resume using data VLs. This pair is 11597 * meant to be used like this: 11598 * 11599 * stop_drain_data_vls(dd); 11600 * // do things with per-VL resources 11601 * open_fill_data_vls(dd); 11602 */ 11603 int stop_drain_data_vls(struct hfi1_devdata *dd) 11604 { 11605 int ret; 11606 11607 ret = disable_data_vls(dd); 11608 if (ret == 0) 11609 drain_data_vls(dd); 11610 11611 return ret; 11612 } 11613 11614 /* 11615 * Convert a nanosecond time to a cclock count. No matter how slow 11616 * the cclock, a non-zero ns will always have a non-zero result. 11617 */ 11618 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns) 11619 { 11620 u32 cclocks; 11621 11622 if (dd->icode == ICODE_FPGA_EMULATION) 11623 cclocks = (ns * 1000) / FPGA_CCLOCK_PS; 11624 else /* simulation pretends to be ASIC */ 11625 cclocks = (ns * 1000) / ASIC_CCLOCK_PS; 11626 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */ 11627 cclocks = 1; 11628 return cclocks; 11629 } 11630 11631 /* 11632 * Convert a cclock count to nanoseconds. Not matter how slow 11633 * the cclock, a non-zero cclocks will always have a non-zero result. 11634 */ 11635 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks) 11636 { 11637 u32 ns; 11638 11639 if (dd->icode == ICODE_FPGA_EMULATION) 11640 ns = (cclocks * FPGA_CCLOCK_PS) / 1000; 11641 else /* simulation pretends to be ASIC */ 11642 ns = (cclocks * ASIC_CCLOCK_PS) / 1000; 11643 if (cclocks && !ns) 11644 ns = 1; 11645 return ns; 11646 } 11647 11648 /* 11649 * Dynamically adjust the receive interrupt timeout for a context based on 11650 * incoming packet rate. 11651 * 11652 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero. 11653 */ 11654 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts) 11655 { 11656 struct hfi1_devdata *dd = rcd->dd; 11657 u32 timeout = rcd->rcvavail_timeout; 11658 11659 /* 11660 * This algorithm doubles or halves the timeout depending on whether 11661 * the number of packets received in this interrupt were less than or 11662 * greater equal the interrupt count. 11663 * 11664 * The calculations below do not allow a steady state to be achieved. 11665 * Only at the endpoints it is possible to have an unchanging 11666 * timeout. 11667 */ 11668 if (npkts < rcv_intr_count) { 11669 /* 11670 * Not enough packets arrived before the timeout, adjust 11671 * timeout downward. 11672 */ 11673 if (timeout < 2) /* already at minimum? */ 11674 return; 11675 timeout >>= 1; 11676 } else { 11677 /* 11678 * More than enough packets arrived before the timeout, adjust 11679 * timeout upward. 11680 */ 11681 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */ 11682 return; 11683 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr); 11684 } 11685 11686 rcd->rcvavail_timeout = timeout; 11687 /* 11688 * timeout cannot be larger than rcv_intr_timeout_csr which has already 11689 * been verified to be in range 11690 */ 11691 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT, 11692 (u64)timeout << 11693 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 11694 } 11695 11696 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, 11697 u32 intr_adjust, u32 npkts) 11698 { 11699 struct hfi1_devdata *dd = rcd->dd; 11700 u64 reg; 11701 u32 ctxt = rcd->ctxt; 11702 11703 /* 11704 * Need to write timeout register before updating RcvHdrHead to ensure 11705 * that a new value is used when the HW decides to restart counting. 11706 */ 11707 if (intr_adjust) 11708 adjust_rcv_timeout(rcd, npkts); 11709 if (updegr) { 11710 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK) 11711 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT; 11712 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg); 11713 } 11714 mmiowb(); 11715 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) | 11716 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK) 11717 << RCV_HDR_HEAD_HEAD_SHIFT); 11718 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 11719 mmiowb(); 11720 } 11721 11722 u32 hdrqempty(struct hfi1_ctxtdata *rcd) 11723 { 11724 u32 head, tail; 11725 11726 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) 11727 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT; 11728 11729 if (rcd->rcvhdrtail_kvaddr) 11730 tail = get_rcvhdrtail(rcd); 11731 else 11732 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 11733 11734 return head == tail; 11735 } 11736 11737 /* 11738 * Context Control and Receive Array encoding for buffer size: 11739 * 0x0 invalid 11740 * 0x1 4 KB 11741 * 0x2 8 KB 11742 * 0x3 16 KB 11743 * 0x4 32 KB 11744 * 0x5 64 KB 11745 * 0x6 128 KB 11746 * 0x7 256 KB 11747 * 0x8 512 KB (Receive Array only) 11748 * 0x9 1 MB (Receive Array only) 11749 * 0xa 2 MB (Receive Array only) 11750 * 11751 * 0xB-0xF - reserved (Receive Array only) 11752 * 11753 * 11754 * This routine assumes that the value has already been sanity checked. 11755 */ 11756 static u32 encoded_size(u32 size) 11757 { 11758 switch (size) { 11759 case 4 * 1024: return 0x1; 11760 case 8 * 1024: return 0x2; 11761 case 16 * 1024: return 0x3; 11762 case 32 * 1024: return 0x4; 11763 case 64 * 1024: return 0x5; 11764 case 128 * 1024: return 0x6; 11765 case 256 * 1024: return 0x7; 11766 case 512 * 1024: return 0x8; 11767 case 1 * 1024 * 1024: return 0x9; 11768 case 2 * 1024 * 1024: return 0xa; 11769 } 11770 return 0x1; /* if invalid, go with the minimum size */ 11771 } 11772 11773 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, 11774 struct hfi1_ctxtdata *rcd) 11775 { 11776 u64 rcvctrl, reg; 11777 int did_enable = 0; 11778 u16 ctxt; 11779 11780 if (!rcd) 11781 return; 11782 11783 ctxt = rcd->ctxt; 11784 11785 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op); 11786 11787 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL); 11788 /* if the context already enabled, don't do the extra steps */ 11789 if ((op & HFI1_RCVCTRL_CTXT_ENB) && 11790 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) { 11791 /* reset the tail and hdr addresses, and sequence count */ 11792 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR, 11793 rcd->rcvhdrq_dma); 11794 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 11795 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11796 rcd->rcvhdrqtailaddr_dma); 11797 rcd->seq_cnt = 1; 11798 11799 /* reset the cached receive header queue head value */ 11800 rcd->head = 0; 11801 11802 /* 11803 * Zero the receive header queue so we don't get false 11804 * positives when checking the sequence number. The 11805 * sequence numbers could land exactly on the same spot. 11806 * E.g. a rcd restart before the receive header wrapped. 11807 */ 11808 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size); 11809 11810 /* starting timeout */ 11811 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr; 11812 11813 /* enable the context */ 11814 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK; 11815 11816 /* clean the egr buffer size first */ 11817 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 11818 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size) 11819 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK) 11820 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT; 11821 11822 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */ 11823 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0); 11824 did_enable = 1; 11825 11826 /* zero RcvEgrIndexHead */ 11827 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0); 11828 11829 /* set eager count and base index */ 11830 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT) 11831 & RCV_EGR_CTRL_EGR_CNT_MASK) 11832 << RCV_EGR_CTRL_EGR_CNT_SHIFT) | 11833 (((rcd->eager_base >> RCV_SHIFT) 11834 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK) 11835 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT); 11836 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg); 11837 11838 /* 11839 * Set TID (expected) count and base index. 11840 * rcd->expected_count is set to individual RcvArray entries, 11841 * not pairs, and the CSR takes a pair-count in groups of 11842 * four, so divide by 8. 11843 */ 11844 reg = (((rcd->expected_count >> RCV_SHIFT) 11845 & RCV_TID_CTRL_TID_PAIR_CNT_MASK) 11846 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) | 11847 (((rcd->expected_base >> RCV_SHIFT) 11848 & RCV_TID_CTRL_TID_BASE_INDEX_MASK) 11849 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT); 11850 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg); 11851 if (ctxt == HFI1_CTRL_CTXT) 11852 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT); 11853 } 11854 if (op & HFI1_RCVCTRL_CTXT_DIS) { 11855 write_csr(dd, RCV_VL15, 0); 11856 /* 11857 * When receive context is being disabled turn on tail 11858 * update with a dummy tail address and then disable 11859 * receive context. 11860 */ 11861 if (dd->rcvhdrtail_dummy_dma) { 11862 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11863 dd->rcvhdrtail_dummy_dma); 11864 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */ 11865 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 11866 } 11867 11868 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK; 11869 } 11870 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) 11871 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 11872 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) 11873 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 11874 if (op & HFI1_RCVCTRL_TAILUPD_ENB && rcd->rcvhdrqtailaddr_dma) 11875 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 11876 if (op & HFI1_RCVCTRL_TAILUPD_DIS) { 11877 /* See comment on RcvCtxtCtrl.TailUpd above */ 11878 if (!(op & HFI1_RCVCTRL_CTXT_DIS)) 11879 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK; 11880 } 11881 if (op & HFI1_RCVCTRL_TIDFLOW_ENB) 11882 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 11883 if (op & HFI1_RCVCTRL_TIDFLOW_DIS) 11884 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 11885 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) { 11886 /* 11887 * In one-packet-per-eager mode, the size comes from 11888 * the RcvArray entry. 11889 */ 11890 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 11891 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 11892 } 11893 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS) 11894 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 11895 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB) 11896 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 11897 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS) 11898 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 11899 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB) 11900 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 11901 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS) 11902 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 11903 rcd->rcvctrl = rcvctrl; 11904 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx\n", ctxt, rcvctrl); 11905 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcd->rcvctrl); 11906 11907 /* work around sticky RcvCtxtStatus.BlockedRHQFull */ 11908 if (did_enable && 11909 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) { 11910 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 11911 if (reg != 0) { 11912 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n", 11913 ctxt, reg); 11914 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 11915 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10); 11916 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00); 11917 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 11918 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 11919 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n", 11920 ctxt, reg, reg == 0 ? "not" : "still"); 11921 } 11922 } 11923 11924 if (did_enable) { 11925 /* 11926 * The interrupt timeout and count must be set after 11927 * the context is enabled to take effect. 11928 */ 11929 /* set interrupt timeout */ 11930 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT, 11931 (u64)rcd->rcvavail_timeout << 11932 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 11933 11934 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */ 11935 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT; 11936 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 11937 } 11938 11939 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS)) 11940 /* 11941 * If the context has been disabled and the Tail Update has 11942 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address 11943 * so it doesn't contain an address that is invalid. 11944 */ 11945 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11946 dd->rcvhdrtail_dummy_dma); 11947 } 11948 11949 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp) 11950 { 11951 int ret; 11952 u64 val = 0; 11953 11954 if (namep) { 11955 ret = dd->cntrnameslen; 11956 *namep = dd->cntrnames; 11957 } else { 11958 const struct cntr_entry *entry; 11959 int i, j; 11960 11961 ret = (dd->ndevcntrs) * sizeof(u64); 11962 11963 /* Get the start of the block of counters */ 11964 *cntrp = dd->cntrs; 11965 11966 /* 11967 * Now go and fill in each counter in the block. 11968 */ 11969 for (i = 0; i < DEV_CNTR_LAST; i++) { 11970 entry = &dev_cntrs[i]; 11971 hfi1_cdbg(CNTR, "reading %s", entry->name); 11972 if (entry->flags & CNTR_DISABLED) { 11973 /* Nothing */ 11974 hfi1_cdbg(CNTR, "\tDisabled\n"); 11975 } else { 11976 if (entry->flags & CNTR_VL) { 11977 hfi1_cdbg(CNTR, "\tPer VL\n"); 11978 for (j = 0; j < C_VL_COUNT; j++) { 11979 val = entry->rw_cntr(entry, 11980 dd, j, 11981 CNTR_MODE_R, 11982 0); 11983 hfi1_cdbg( 11984 CNTR, 11985 "\t\tRead 0x%llx for %d\n", 11986 val, j); 11987 dd->cntrs[entry->offset + j] = 11988 val; 11989 } 11990 } else if (entry->flags & CNTR_SDMA) { 11991 hfi1_cdbg(CNTR, 11992 "\t Per SDMA Engine\n"); 11993 for (j = 0; j < dd->chip_sdma_engines; 11994 j++) { 11995 val = 11996 entry->rw_cntr(entry, dd, j, 11997 CNTR_MODE_R, 0); 11998 hfi1_cdbg(CNTR, 11999 "\t\tRead 0x%llx for %d\n", 12000 val, j); 12001 dd->cntrs[entry->offset + j] = 12002 val; 12003 } 12004 } else { 12005 val = entry->rw_cntr(entry, dd, 12006 CNTR_INVALID_VL, 12007 CNTR_MODE_R, 0); 12008 dd->cntrs[entry->offset] = val; 12009 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 12010 } 12011 } 12012 } 12013 } 12014 return ret; 12015 } 12016 12017 /* 12018 * Used by sysfs to create files for hfi stats to read 12019 */ 12020 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp) 12021 { 12022 int ret; 12023 u64 val = 0; 12024 12025 if (namep) { 12026 ret = ppd->dd->portcntrnameslen; 12027 *namep = ppd->dd->portcntrnames; 12028 } else { 12029 const struct cntr_entry *entry; 12030 int i, j; 12031 12032 ret = ppd->dd->nportcntrs * sizeof(u64); 12033 *cntrp = ppd->cntrs; 12034 12035 for (i = 0; i < PORT_CNTR_LAST; i++) { 12036 entry = &port_cntrs[i]; 12037 hfi1_cdbg(CNTR, "reading %s", entry->name); 12038 if (entry->flags & CNTR_DISABLED) { 12039 /* Nothing */ 12040 hfi1_cdbg(CNTR, "\tDisabled\n"); 12041 continue; 12042 } 12043 12044 if (entry->flags & CNTR_VL) { 12045 hfi1_cdbg(CNTR, "\tPer VL"); 12046 for (j = 0; j < C_VL_COUNT; j++) { 12047 val = entry->rw_cntr(entry, ppd, j, 12048 CNTR_MODE_R, 12049 0); 12050 hfi1_cdbg( 12051 CNTR, 12052 "\t\tRead 0x%llx for %d", 12053 val, j); 12054 ppd->cntrs[entry->offset + j] = val; 12055 } 12056 } else { 12057 val = entry->rw_cntr(entry, ppd, 12058 CNTR_INVALID_VL, 12059 CNTR_MODE_R, 12060 0); 12061 ppd->cntrs[entry->offset] = val; 12062 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 12063 } 12064 } 12065 } 12066 return ret; 12067 } 12068 12069 static void free_cntrs(struct hfi1_devdata *dd) 12070 { 12071 struct hfi1_pportdata *ppd; 12072 int i; 12073 12074 if (dd->synth_stats_timer.function) 12075 del_timer_sync(&dd->synth_stats_timer); 12076 ppd = (struct hfi1_pportdata *)(dd + 1); 12077 for (i = 0; i < dd->num_pports; i++, ppd++) { 12078 kfree(ppd->cntrs); 12079 kfree(ppd->scntrs); 12080 free_percpu(ppd->ibport_data.rvp.rc_acks); 12081 free_percpu(ppd->ibport_data.rvp.rc_qacks); 12082 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp); 12083 ppd->cntrs = NULL; 12084 ppd->scntrs = NULL; 12085 ppd->ibport_data.rvp.rc_acks = NULL; 12086 ppd->ibport_data.rvp.rc_qacks = NULL; 12087 ppd->ibport_data.rvp.rc_delayed_comp = NULL; 12088 } 12089 kfree(dd->portcntrnames); 12090 dd->portcntrnames = NULL; 12091 kfree(dd->cntrs); 12092 dd->cntrs = NULL; 12093 kfree(dd->scntrs); 12094 dd->scntrs = NULL; 12095 kfree(dd->cntrnames); 12096 dd->cntrnames = NULL; 12097 if (dd->update_cntr_wq) { 12098 destroy_workqueue(dd->update_cntr_wq); 12099 dd->update_cntr_wq = NULL; 12100 } 12101 } 12102 12103 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry, 12104 u64 *psval, void *context, int vl) 12105 { 12106 u64 val; 12107 u64 sval = *psval; 12108 12109 if (entry->flags & CNTR_DISABLED) { 12110 dd_dev_err(dd, "Counter %s not enabled", entry->name); 12111 return 0; 12112 } 12113 12114 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 12115 12116 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0); 12117 12118 /* If its a synthetic counter there is more work we need to do */ 12119 if (entry->flags & CNTR_SYNTH) { 12120 if (sval == CNTR_MAX) { 12121 /* No need to read already saturated */ 12122 return CNTR_MAX; 12123 } 12124 12125 if (entry->flags & CNTR_32BIT) { 12126 /* 32bit counters can wrap multiple times */ 12127 u64 upper = sval >> 32; 12128 u64 lower = (sval << 32) >> 32; 12129 12130 if (lower > val) { /* hw wrapped */ 12131 if (upper == CNTR_32BIT_MAX) 12132 val = CNTR_MAX; 12133 else 12134 upper++; 12135 } 12136 12137 if (val != CNTR_MAX) 12138 val = (upper << 32) | val; 12139 12140 } else { 12141 /* If we rolled we are saturated */ 12142 if ((val < sval) || (val > CNTR_MAX)) 12143 val = CNTR_MAX; 12144 } 12145 } 12146 12147 *psval = val; 12148 12149 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 12150 12151 return val; 12152 } 12153 12154 static u64 write_dev_port_cntr(struct hfi1_devdata *dd, 12155 struct cntr_entry *entry, 12156 u64 *psval, void *context, int vl, u64 data) 12157 { 12158 u64 val; 12159 12160 if (entry->flags & CNTR_DISABLED) { 12161 dd_dev_err(dd, "Counter %s not enabled", entry->name); 12162 return 0; 12163 } 12164 12165 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 12166 12167 if (entry->flags & CNTR_SYNTH) { 12168 *psval = data; 12169 if (entry->flags & CNTR_32BIT) { 12170 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 12171 (data << 32) >> 32); 12172 val = data; /* return the full 64bit value */ 12173 } else { 12174 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 12175 data); 12176 } 12177 } else { 12178 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data); 12179 } 12180 12181 *psval = val; 12182 12183 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 12184 12185 return val; 12186 } 12187 12188 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl) 12189 { 12190 struct cntr_entry *entry; 12191 u64 *sval; 12192 12193 entry = &dev_cntrs[index]; 12194 sval = dd->scntrs + entry->offset; 12195 12196 if (vl != CNTR_INVALID_VL) 12197 sval += vl; 12198 12199 return read_dev_port_cntr(dd, entry, sval, dd, vl); 12200 } 12201 12202 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data) 12203 { 12204 struct cntr_entry *entry; 12205 u64 *sval; 12206 12207 entry = &dev_cntrs[index]; 12208 sval = dd->scntrs + entry->offset; 12209 12210 if (vl != CNTR_INVALID_VL) 12211 sval += vl; 12212 12213 return write_dev_port_cntr(dd, entry, sval, dd, vl, data); 12214 } 12215 12216 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl) 12217 { 12218 struct cntr_entry *entry; 12219 u64 *sval; 12220 12221 entry = &port_cntrs[index]; 12222 sval = ppd->scntrs + entry->offset; 12223 12224 if (vl != CNTR_INVALID_VL) 12225 sval += vl; 12226 12227 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 12228 (index <= C_RCV_HDR_OVF_LAST)) { 12229 /* We do not want to bother for disabled contexts */ 12230 return 0; 12231 } 12232 12233 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl); 12234 } 12235 12236 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data) 12237 { 12238 struct cntr_entry *entry; 12239 u64 *sval; 12240 12241 entry = &port_cntrs[index]; 12242 sval = ppd->scntrs + entry->offset; 12243 12244 if (vl != CNTR_INVALID_VL) 12245 sval += vl; 12246 12247 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 12248 (index <= C_RCV_HDR_OVF_LAST)) { 12249 /* We do not want to bother for disabled contexts */ 12250 return 0; 12251 } 12252 12253 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data); 12254 } 12255 12256 static void do_update_synth_timer(struct work_struct *work) 12257 { 12258 u64 cur_tx; 12259 u64 cur_rx; 12260 u64 total_flits; 12261 u8 update = 0; 12262 int i, j, vl; 12263 struct hfi1_pportdata *ppd; 12264 struct cntr_entry *entry; 12265 struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata, 12266 update_cntr_work); 12267 12268 /* 12269 * Rather than keep beating on the CSRs pick a minimal set that we can 12270 * check to watch for potential roll over. We can do this by looking at 12271 * the number of flits sent/recv. If the total flits exceeds 32bits then 12272 * we have to iterate all the counters and update. 12273 */ 12274 entry = &dev_cntrs[C_DC_RCV_FLITS]; 12275 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 12276 12277 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 12278 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 12279 12280 hfi1_cdbg( 12281 CNTR, 12282 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx\n", 12283 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx); 12284 12285 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) { 12286 /* 12287 * May not be strictly necessary to update but it won't hurt and 12288 * simplifies the logic here. 12289 */ 12290 update = 1; 12291 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating", 12292 dd->unit); 12293 } else { 12294 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx); 12295 hfi1_cdbg(CNTR, 12296 "[%d] total flits 0x%llx limit 0x%llx\n", dd->unit, 12297 total_flits, (u64)CNTR_32BIT_MAX); 12298 if (total_flits >= CNTR_32BIT_MAX) { 12299 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating", 12300 dd->unit); 12301 update = 1; 12302 } 12303 } 12304 12305 if (update) { 12306 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit); 12307 for (i = 0; i < DEV_CNTR_LAST; i++) { 12308 entry = &dev_cntrs[i]; 12309 if (entry->flags & CNTR_VL) { 12310 for (vl = 0; vl < C_VL_COUNT; vl++) 12311 read_dev_cntr(dd, i, vl); 12312 } else { 12313 read_dev_cntr(dd, i, CNTR_INVALID_VL); 12314 } 12315 } 12316 ppd = (struct hfi1_pportdata *)(dd + 1); 12317 for (i = 0; i < dd->num_pports; i++, ppd++) { 12318 for (j = 0; j < PORT_CNTR_LAST; j++) { 12319 entry = &port_cntrs[j]; 12320 if (entry->flags & CNTR_VL) { 12321 for (vl = 0; vl < C_VL_COUNT; vl++) 12322 read_port_cntr(ppd, j, vl); 12323 } else { 12324 read_port_cntr(ppd, j, CNTR_INVALID_VL); 12325 } 12326 } 12327 } 12328 12329 /* 12330 * We want the value in the register. The goal is to keep track 12331 * of the number of "ticks" not the counter value. In other 12332 * words if the register rolls we want to notice it and go ahead 12333 * and force an update. 12334 */ 12335 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 12336 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 12337 CNTR_MODE_R, 0); 12338 12339 entry = &dev_cntrs[C_DC_RCV_FLITS]; 12340 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 12341 CNTR_MODE_R, 0); 12342 12343 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx", 12344 dd->unit, dd->last_tx, dd->last_rx); 12345 12346 } else { 12347 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit); 12348 } 12349 } 12350 12351 static void update_synth_timer(struct timer_list *t) 12352 { 12353 struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer); 12354 12355 queue_work(dd->update_cntr_wq, &dd->update_cntr_work); 12356 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12357 } 12358 12359 #define C_MAX_NAME 16 /* 15 chars + one for /0 */ 12360 static int init_cntrs(struct hfi1_devdata *dd) 12361 { 12362 int i, rcv_ctxts, j; 12363 size_t sz; 12364 char *p; 12365 char name[C_MAX_NAME]; 12366 struct hfi1_pportdata *ppd; 12367 const char *bit_type_32 = ",32"; 12368 const int bit_type_32_sz = strlen(bit_type_32); 12369 12370 /* set up the stats timer; the add_timer is done at the end */ 12371 timer_setup(&dd->synth_stats_timer, update_synth_timer, 0); 12372 12373 /***********************/ 12374 /* per device counters */ 12375 /***********************/ 12376 12377 /* size names and determine how many we have*/ 12378 dd->ndevcntrs = 0; 12379 sz = 0; 12380 12381 for (i = 0; i < DEV_CNTR_LAST; i++) { 12382 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12383 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name); 12384 continue; 12385 } 12386 12387 if (dev_cntrs[i].flags & CNTR_VL) { 12388 dev_cntrs[i].offset = dd->ndevcntrs; 12389 for (j = 0; j < C_VL_COUNT; j++) { 12390 snprintf(name, C_MAX_NAME, "%s%d", 12391 dev_cntrs[i].name, vl_from_idx(j)); 12392 sz += strlen(name); 12393 /* Add ",32" for 32-bit counters */ 12394 if (dev_cntrs[i].flags & CNTR_32BIT) 12395 sz += bit_type_32_sz; 12396 sz++; 12397 dd->ndevcntrs++; 12398 } 12399 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12400 dev_cntrs[i].offset = dd->ndevcntrs; 12401 for (j = 0; j < dd->chip_sdma_engines; j++) { 12402 snprintf(name, C_MAX_NAME, "%s%d", 12403 dev_cntrs[i].name, j); 12404 sz += strlen(name); 12405 /* Add ",32" for 32-bit counters */ 12406 if (dev_cntrs[i].flags & CNTR_32BIT) 12407 sz += bit_type_32_sz; 12408 sz++; 12409 dd->ndevcntrs++; 12410 } 12411 } else { 12412 /* +1 for newline. */ 12413 sz += strlen(dev_cntrs[i].name) + 1; 12414 /* Add ",32" for 32-bit counters */ 12415 if (dev_cntrs[i].flags & CNTR_32BIT) 12416 sz += bit_type_32_sz; 12417 dev_cntrs[i].offset = dd->ndevcntrs; 12418 dd->ndevcntrs++; 12419 } 12420 } 12421 12422 /* allocate space for the counter values */ 12423 dd->cntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL); 12424 if (!dd->cntrs) 12425 goto bail; 12426 12427 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL); 12428 if (!dd->scntrs) 12429 goto bail; 12430 12431 /* allocate space for the counter names */ 12432 dd->cntrnameslen = sz; 12433 dd->cntrnames = kmalloc(sz, GFP_KERNEL); 12434 if (!dd->cntrnames) 12435 goto bail; 12436 12437 /* fill in the names */ 12438 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) { 12439 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12440 /* Nothing */ 12441 } else if (dev_cntrs[i].flags & CNTR_VL) { 12442 for (j = 0; j < C_VL_COUNT; j++) { 12443 snprintf(name, C_MAX_NAME, "%s%d", 12444 dev_cntrs[i].name, 12445 vl_from_idx(j)); 12446 memcpy(p, name, strlen(name)); 12447 p += strlen(name); 12448 12449 /* Counter is 32 bits */ 12450 if (dev_cntrs[i].flags & CNTR_32BIT) { 12451 memcpy(p, bit_type_32, bit_type_32_sz); 12452 p += bit_type_32_sz; 12453 } 12454 12455 *p++ = '\n'; 12456 } 12457 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12458 for (j = 0; j < dd->chip_sdma_engines; j++) { 12459 snprintf(name, C_MAX_NAME, "%s%d", 12460 dev_cntrs[i].name, j); 12461 memcpy(p, name, strlen(name)); 12462 p += strlen(name); 12463 12464 /* Counter is 32 bits */ 12465 if (dev_cntrs[i].flags & CNTR_32BIT) { 12466 memcpy(p, bit_type_32, bit_type_32_sz); 12467 p += bit_type_32_sz; 12468 } 12469 12470 *p++ = '\n'; 12471 } 12472 } else { 12473 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name)); 12474 p += strlen(dev_cntrs[i].name); 12475 12476 /* Counter is 32 bits */ 12477 if (dev_cntrs[i].flags & CNTR_32BIT) { 12478 memcpy(p, bit_type_32, bit_type_32_sz); 12479 p += bit_type_32_sz; 12480 } 12481 12482 *p++ = '\n'; 12483 } 12484 } 12485 12486 /*********************/ 12487 /* per port counters */ 12488 /*********************/ 12489 12490 /* 12491 * Go through the counters for the overflows and disable the ones we 12492 * don't need. This varies based on platform so we need to do it 12493 * dynamically here. 12494 */ 12495 rcv_ctxts = dd->num_rcv_contexts; 12496 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts; 12497 i <= C_RCV_HDR_OVF_LAST; i++) { 12498 port_cntrs[i].flags |= CNTR_DISABLED; 12499 } 12500 12501 /* size port counter names and determine how many we have*/ 12502 sz = 0; 12503 dd->nportcntrs = 0; 12504 for (i = 0; i < PORT_CNTR_LAST; i++) { 12505 if (port_cntrs[i].flags & CNTR_DISABLED) { 12506 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name); 12507 continue; 12508 } 12509 12510 if (port_cntrs[i].flags & CNTR_VL) { 12511 port_cntrs[i].offset = dd->nportcntrs; 12512 for (j = 0; j < C_VL_COUNT; j++) { 12513 snprintf(name, C_MAX_NAME, "%s%d", 12514 port_cntrs[i].name, vl_from_idx(j)); 12515 sz += strlen(name); 12516 /* Add ",32" for 32-bit counters */ 12517 if (port_cntrs[i].flags & CNTR_32BIT) 12518 sz += bit_type_32_sz; 12519 sz++; 12520 dd->nportcntrs++; 12521 } 12522 } else { 12523 /* +1 for newline */ 12524 sz += strlen(port_cntrs[i].name) + 1; 12525 /* Add ",32" for 32-bit counters */ 12526 if (port_cntrs[i].flags & CNTR_32BIT) 12527 sz += bit_type_32_sz; 12528 port_cntrs[i].offset = dd->nportcntrs; 12529 dd->nportcntrs++; 12530 } 12531 } 12532 12533 /* allocate space for the counter names */ 12534 dd->portcntrnameslen = sz; 12535 dd->portcntrnames = kmalloc(sz, GFP_KERNEL); 12536 if (!dd->portcntrnames) 12537 goto bail; 12538 12539 /* fill in port cntr names */ 12540 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) { 12541 if (port_cntrs[i].flags & CNTR_DISABLED) 12542 continue; 12543 12544 if (port_cntrs[i].flags & CNTR_VL) { 12545 for (j = 0; j < C_VL_COUNT; j++) { 12546 snprintf(name, C_MAX_NAME, "%s%d", 12547 port_cntrs[i].name, vl_from_idx(j)); 12548 memcpy(p, name, strlen(name)); 12549 p += strlen(name); 12550 12551 /* Counter is 32 bits */ 12552 if (port_cntrs[i].flags & CNTR_32BIT) { 12553 memcpy(p, bit_type_32, bit_type_32_sz); 12554 p += bit_type_32_sz; 12555 } 12556 12557 *p++ = '\n'; 12558 } 12559 } else { 12560 memcpy(p, port_cntrs[i].name, 12561 strlen(port_cntrs[i].name)); 12562 p += strlen(port_cntrs[i].name); 12563 12564 /* Counter is 32 bits */ 12565 if (port_cntrs[i].flags & CNTR_32BIT) { 12566 memcpy(p, bit_type_32, bit_type_32_sz); 12567 p += bit_type_32_sz; 12568 } 12569 12570 *p++ = '\n'; 12571 } 12572 } 12573 12574 /* allocate per port storage for counter values */ 12575 ppd = (struct hfi1_pportdata *)(dd + 1); 12576 for (i = 0; i < dd->num_pports; i++, ppd++) { 12577 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12578 if (!ppd->cntrs) 12579 goto bail; 12580 12581 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12582 if (!ppd->scntrs) 12583 goto bail; 12584 } 12585 12586 /* CPU counters need to be allocated and zeroed */ 12587 if (init_cpu_counters(dd)) 12588 goto bail; 12589 12590 dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d", 12591 WQ_MEM_RECLAIM, dd->unit); 12592 if (!dd->update_cntr_wq) 12593 goto bail; 12594 12595 INIT_WORK(&dd->update_cntr_work, do_update_synth_timer); 12596 12597 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12598 return 0; 12599 bail: 12600 free_cntrs(dd); 12601 return -ENOMEM; 12602 } 12603 12604 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) 12605 { 12606 switch (chip_lstate) { 12607 default: 12608 dd_dev_err(dd, 12609 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n", 12610 chip_lstate); 12611 /* fall through */ 12612 case LSTATE_DOWN: 12613 return IB_PORT_DOWN; 12614 case LSTATE_INIT: 12615 return IB_PORT_INIT; 12616 case LSTATE_ARMED: 12617 return IB_PORT_ARMED; 12618 case LSTATE_ACTIVE: 12619 return IB_PORT_ACTIVE; 12620 } 12621 } 12622 12623 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) 12624 { 12625 /* look at the HFI meta-states only */ 12626 switch (chip_pstate & 0xf0) { 12627 default: 12628 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n", 12629 chip_pstate); 12630 /* fall through */ 12631 case PLS_DISABLED: 12632 return IB_PORTPHYSSTATE_DISABLED; 12633 case PLS_OFFLINE: 12634 return OPA_PORTPHYSSTATE_OFFLINE; 12635 case PLS_POLLING: 12636 return IB_PORTPHYSSTATE_POLLING; 12637 case PLS_CONFIGPHY: 12638 return IB_PORTPHYSSTATE_TRAINING; 12639 case PLS_LINKUP: 12640 return IB_PORTPHYSSTATE_LINKUP; 12641 case PLS_PHYTEST: 12642 return IB_PORTPHYSSTATE_PHY_TEST; 12643 } 12644 } 12645 12646 /* return the OPA port logical state name */ 12647 const char *opa_lstate_name(u32 lstate) 12648 { 12649 static const char * const port_logical_names[] = { 12650 "PORT_NOP", 12651 "PORT_DOWN", 12652 "PORT_INIT", 12653 "PORT_ARMED", 12654 "PORT_ACTIVE", 12655 "PORT_ACTIVE_DEFER", 12656 }; 12657 if (lstate < ARRAY_SIZE(port_logical_names)) 12658 return port_logical_names[lstate]; 12659 return "unknown"; 12660 } 12661 12662 /* return the OPA port physical state name */ 12663 const char *opa_pstate_name(u32 pstate) 12664 { 12665 static const char * const port_physical_names[] = { 12666 "PHYS_NOP", 12667 "reserved1", 12668 "PHYS_POLL", 12669 "PHYS_DISABLED", 12670 "PHYS_TRAINING", 12671 "PHYS_LINKUP", 12672 "PHYS_LINK_ERR_RECOVER", 12673 "PHYS_PHY_TEST", 12674 "reserved8", 12675 "PHYS_OFFLINE", 12676 "PHYS_GANGED", 12677 "PHYS_TEST", 12678 }; 12679 if (pstate < ARRAY_SIZE(port_physical_names)) 12680 return port_physical_names[pstate]; 12681 return "unknown"; 12682 } 12683 12684 /** 12685 * update_statusp - Update userspace status flag 12686 * @ppd: Port data structure 12687 * @state: port state information 12688 * 12689 * Actual port status is determined by the host_link_state value 12690 * in the ppd. 12691 * 12692 * host_link_state MUST be updated before updating the user space 12693 * statusp. 12694 */ 12695 static void update_statusp(struct hfi1_pportdata *ppd, u32 state) 12696 { 12697 /* 12698 * Set port status flags in the page mapped into userspace 12699 * memory. Do it here to ensure a reliable state - this is 12700 * the only function called by all state handling code. 12701 * Always set the flags due to the fact that the cache value 12702 * might have been changed explicitly outside of this 12703 * function. 12704 */ 12705 if (ppd->statusp) { 12706 switch (state) { 12707 case IB_PORT_DOWN: 12708 case IB_PORT_INIT: 12709 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF | 12710 HFI1_STATUS_IB_READY); 12711 break; 12712 case IB_PORT_ARMED: 12713 *ppd->statusp |= HFI1_STATUS_IB_CONF; 12714 break; 12715 case IB_PORT_ACTIVE: 12716 *ppd->statusp |= HFI1_STATUS_IB_READY; 12717 break; 12718 } 12719 } 12720 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n", 12721 opa_lstate_name(state), state); 12722 } 12723 12724 /** 12725 * wait_logical_linkstate - wait for an IB link state change to occur 12726 * @ppd: port device 12727 * @state: the state to wait for 12728 * @msecs: the number of milliseconds to wait 12729 * 12730 * Wait up to msecs milliseconds for IB link state change to occur. 12731 * For now, take the easy polling route. 12732 * Returns 0 if state reached, otherwise -ETIMEDOUT. 12733 */ 12734 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 12735 int msecs) 12736 { 12737 unsigned long timeout; 12738 u32 new_state; 12739 12740 timeout = jiffies + msecs_to_jiffies(msecs); 12741 while (1) { 12742 new_state = chip_to_opa_lstate(ppd->dd, 12743 read_logical_state(ppd->dd)); 12744 if (new_state == state) 12745 break; 12746 if (time_after(jiffies, timeout)) { 12747 dd_dev_err(ppd->dd, 12748 "timeout waiting for link state 0x%x\n", 12749 state); 12750 return -ETIMEDOUT; 12751 } 12752 msleep(20); 12753 } 12754 12755 return 0; 12756 } 12757 12758 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state) 12759 { 12760 u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state); 12761 12762 dd_dev_info(ppd->dd, 12763 "physical state changed to %s (0x%x), phy 0x%x\n", 12764 opa_pstate_name(ib_pstate), ib_pstate, state); 12765 } 12766 12767 /* 12768 * Read the physical hardware link state and check if it matches host 12769 * drivers anticipated state. 12770 */ 12771 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state) 12772 { 12773 u32 read_state = read_physical_state(ppd->dd); 12774 12775 if (read_state == state) { 12776 log_state_transition(ppd, state); 12777 } else { 12778 dd_dev_err(ppd->dd, 12779 "anticipated phy link state 0x%x, read 0x%x\n", 12780 state, read_state); 12781 } 12782 } 12783 12784 /* 12785 * wait_physical_linkstate - wait for an physical link state change to occur 12786 * @ppd: port device 12787 * @state: the state to wait for 12788 * @msecs: the number of milliseconds to wait 12789 * 12790 * Wait up to msecs milliseconds for physical link state change to occur. 12791 * Returns 0 if state reached, otherwise -ETIMEDOUT. 12792 */ 12793 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, 12794 int msecs) 12795 { 12796 u32 read_state; 12797 unsigned long timeout; 12798 12799 timeout = jiffies + msecs_to_jiffies(msecs); 12800 while (1) { 12801 read_state = read_physical_state(ppd->dd); 12802 if (read_state == state) 12803 break; 12804 if (time_after(jiffies, timeout)) { 12805 dd_dev_err(ppd->dd, 12806 "timeout waiting for phy link state 0x%x\n", 12807 state); 12808 return -ETIMEDOUT; 12809 } 12810 usleep_range(1950, 2050); /* sleep 2ms-ish */ 12811 } 12812 12813 log_state_transition(ppd, state); 12814 return 0; 12815 } 12816 12817 /* 12818 * wait_phys_link_offline_quiet_substates - wait for any offline substate 12819 * @ppd: port device 12820 * @msecs: the number of milliseconds to wait 12821 * 12822 * Wait up to msecs milliseconds for any offline physical link 12823 * state change to occur. 12824 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. 12825 */ 12826 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, 12827 int msecs) 12828 { 12829 u32 read_state; 12830 unsigned long timeout; 12831 12832 timeout = jiffies + msecs_to_jiffies(msecs); 12833 while (1) { 12834 read_state = read_physical_state(ppd->dd); 12835 if ((read_state & 0xF0) == PLS_OFFLINE) 12836 break; 12837 if (time_after(jiffies, timeout)) { 12838 dd_dev_err(ppd->dd, 12839 "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n", 12840 read_state, msecs); 12841 return -ETIMEDOUT; 12842 } 12843 usleep_range(1950, 2050); /* sleep 2ms-ish */ 12844 } 12845 12846 log_state_transition(ppd, read_state); 12847 return read_state; 12848 } 12849 12850 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \ 12851 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 12852 12853 #define SET_STATIC_RATE_CONTROL_SMASK(r) \ 12854 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 12855 12856 void hfi1_init_ctxt(struct send_context *sc) 12857 { 12858 if (sc) { 12859 struct hfi1_devdata *dd = sc->dd; 12860 u64 reg; 12861 u8 set = (sc->type == SC_USER ? 12862 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) : 12863 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL)); 12864 reg = read_kctxt_csr(dd, sc->hw_context, 12865 SEND_CTXT_CHECK_ENABLE); 12866 if (set) 12867 CLEAR_STATIC_RATE_CONTROL_SMASK(reg); 12868 else 12869 SET_STATIC_RATE_CONTROL_SMASK(reg); 12870 write_kctxt_csr(dd, sc->hw_context, 12871 SEND_CTXT_CHECK_ENABLE, reg); 12872 } 12873 } 12874 12875 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp) 12876 { 12877 int ret = 0; 12878 u64 reg; 12879 12880 if (dd->icode != ICODE_RTL_SILICON) { 12881 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 12882 dd_dev_info(dd, "%s: tempsense not supported by HW\n", 12883 __func__); 12884 return -EINVAL; 12885 } 12886 reg = read_csr(dd, ASIC_STS_THERM); 12887 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) & 12888 ASIC_STS_THERM_CURR_TEMP_MASK); 12889 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) & 12890 ASIC_STS_THERM_LO_TEMP_MASK); 12891 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) & 12892 ASIC_STS_THERM_HI_TEMP_MASK); 12893 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) & 12894 ASIC_STS_THERM_CRIT_TEMP_MASK); 12895 /* triggers is a 3-bit value - 1 bit per trigger. */ 12896 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7); 12897 12898 return ret; 12899 } 12900 12901 /** 12902 * get_int_mask - get 64 bit int mask 12903 * @dd - the devdata 12904 * @i - the csr (relative to CCE_INT_MASK) 12905 * 12906 * Returns the mask with the urgent interrupt mask 12907 * bit clear for kernel receive contexts. 12908 */ 12909 static u64 get_int_mask(struct hfi1_devdata *dd, u32 i) 12910 { 12911 u64 mask = U64_MAX; /* default to no change */ 12912 12913 if (i >= (IS_RCVURGENT_START / 64) && i < (IS_RCVURGENT_END / 64)) { 12914 int j = (i - (IS_RCVURGENT_START / 64)) * 64; 12915 int k = !j ? IS_RCVURGENT_START % 64 : 0; 12916 12917 if (j) 12918 j -= IS_RCVURGENT_START % 64; 12919 /* j = 0..dd->first_dyn_alloc_ctxt - 1,k = 0..63 */ 12920 for (; j < dd->first_dyn_alloc_ctxt && k < 64; j++, k++) 12921 /* convert to bit in mask and clear */ 12922 mask &= ~BIT_ULL(k); 12923 } 12924 return mask; 12925 } 12926 12927 /* ========================================================================= */ 12928 12929 /* 12930 * Enable/disable chip from delivering interrupts. 12931 */ 12932 void set_intr_state(struct hfi1_devdata *dd, u32 enable) 12933 { 12934 int i; 12935 12936 /* 12937 * In HFI, the mask needs to be 1 to allow interrupts. 12938 */ 12939 if (enable) { 12940 /* enable all interrupts but urgent on kernel contexts */ 12941 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 12942 u64 mask = get_int_mask(dd, i); 12943 12944 write_csr(dd, CCE_INT_MASK + (8 * i), mask); 12945 } 12946 12947 init_qsfp_int(dd); 12948 } else { 12949 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 12950 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull); 12951 } 12952 } 12953 12954 /* 12955 * Clear all interrupt sources on the chip. 12956 */ 12957 static void clear_all_interrupts(struct hfi1_devdata *dd) 12958 { 12959 int i; 12960 12961 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 12962 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0); 12963 12964 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0); 12965 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0); 12966 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0); 12967 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0); 12968 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0); 12969 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0); 12970 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0); 12971 for (i = 0; i < dd->chip_send_contexts; i++) 12972 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0); 12973 for (i = 0; i < dd->chip_sdma_engines; i++) 12974 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0); 12975 12976 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0); 12977 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0); 12978 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0); 12979 } 12980 12981 /* Move to pcie.c? */ 12982 static void disable_intx(struct pci_dev *pdev) 12983 { 12984 pci_intx(pdev, 0); 12985 } 12986 12987 static void clean_up_interrupts(struct hfi1_devdata *dd) 12988 { 12989 int i; 12990 12991 /* remove irqs - must happen before disabling/turning off */ 12992 if (dd->num_msix_entries) { 12993 /* MSI-X */ 12994 struct hfi1_msix_entry *me = dd->msix_entries; 12995 12996 for (i = 0; i < dd->num_msix_entries; i++, me++) { 12997 if (!me->arg) /* => no irq, no affinity */ 12998 continue; 12999 hfi1_put_irq_affinity(dd, me); 13000 pci_free_irq(dd->pcidev, i, me->arg); 13001 } 13002 13003 /* clean structures */ 13004 kfree(dd->msix_entries); 13005 dd->msix_entries = NULL; 13006 dd->num_msix_entries = 0; 13007 } else { 13008 /* INTx */ 13009 if (dd->requested_intx_irq) { 13010 pci_free_irq(dd->pcidev, 0, dd); 13011 dd->requested_intx_irq = 0; 13012 } 13013 disable_intx(dd->pcidev); 13014 } 13015 13016 pci_free_irq_vectors(dd->pcidev); 13017 } 13018 13019 /* 13020 * Remap the interrupt source from the general handler to the given MSI-X 13021 * interrupt. 13022 */ 13023 static void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr) 13024 { 13025 u64 reg; 13026 int m, n; 13027 13028 /* clear from the handled mask of the general interrupt */ 13029 m = isrc / 64; 13030 n = isrc % 64; 13031 if (likely(m < CCE_NUM_INT_CSRS)) { 13032 dd->gi_mask[m] &= ~((u64)1 << n); 13033 } else { 13034 dd_dev_err(dd, "remap interrupt err\n"); 13035 return; 13036 } 13037 13038 /* direct the chip source to the given MSI-X interrupt */ 13039 m = isrc / 8; 13040 n = isrc % 8; 13041 reg = read_csr(dd, CCE_INT_MAP + (8 * m)); 13042 reg &= ~((u64)0xff << (8 * n)); 13043 reg |= ((u64)msix_intr & 0xff) << (8 * n); 13044 write_csr(dd, CCE_INT_MAP + (8 * m), reg); 13045 } 13046 13047 static void remap_sdma_interrupts(struct hfi1_devdata *dd, 13048 int engine, int msix_intr) 13049 { 13050 /* 13051 * SDMA engine interrupt sources grouped by type, rather than 13052 * engine. Per-engine interrupts are as follows: 13053 * SDMA 13054 * SDMAProgress 13055 * SDMAIdle 13056 */ 13057 remap_intr(dd, IS_SDMA_START + 0 * TXE_NUM_SDMA_ENGINES + engine, 13058 msix_intr); 13059 remap_intr(dd, IS_SDMA_START + 1 * TXE_NUM_SDMA_ENGINES + engine, 13060 msix_intr); 13061 remap_intr(dd, IS_SDMA_START + 2 * TXE_NUM_SDMA_ENGINES + engine, 13062 msix_intr); 13063 } 13064 13065 static int request_intx_irq(struct hfi1_devdata *dd) 13066 { 13067 int ret; 13068 13069 ret = pci_request_irq(dd->pcidev, 0, general_interrupt, NULL, dd, 13070 DRIVER_NAME "_%d", dd->unit); 13071 if (ret) 13072 dd_dev_err(dd, "unable to request INTx interrupt, err %d\n", 13073 ret); 13074 else 13075 dd->requested_intx_irq = 1; 13076 return ret; 13077 } 13078 13079 static int request_msix_irqs(struct hfi1_devdata *dd) 13080 { 13081 int first_general, last_general; 13082 int first_sdma, last_sdma; 13083 int first_rx, last_rx; 13084 int i, ret = 0; 13085 13086 /* calculate the ranges we are going to use */ 13087 first_general = 0; 13088 last_general = first_general + 1; 13089 first_sdma = last_general; 13090 last_sdma = first_sdma + dd->num_sdma; 13091 first_rx = last_sdma; 13092 last_rx = first_rx + dd->n_krcv_queues + dd->num_vnic_contexts; 13093 13094 /* VNIC MSIx interrupts get mapped when VNIC contexts are created */ 13095 dd->first_dyn_msix_idx = first_rx + dd->n_krcv_queues; 13096 13097 /* 13098 * Sanity check - the code expects all SDMA chip source 13099 * interrupts to be in the same CSR, starting at bit 0. Verify 13100 * that this is true by checking the bit location of the start. 13101 */ 13102 BUILD_BUG_ON(IS_SDMA_START % 64); 13103 13104 for (i = 0; i < dd->num_msix_entries; i++) { 13105 struct hfi1_msix_entry *me = &dd->msix_entries[i]; 13106 const char *err_info; 13107 irq_handler_t handler; 13108 irq_handler_t thread = NULL; 13109 void *arg = NULL; 13110 int idx; 13111 struct hfi1_ctxtdata *rcd = NULL; 13112 struct sdma_engine *sde = NULL; 13113 char name[MAX_NAME_SIZE]; 13114 13115 /* obtain the arguments to pci_request_irq */ 13116 if (first_general <= i && i < last_general) { 13117 idx = i - first_general; 13118 handler = general_interrupt; 13119 arg = dd; 13120 snprintf(name, sizeof(name), 13121 DRIVER_NAME "_%d", dd->unit); 13122 err_info = "general"; 13123 me->type = IRQ_GENERAL; 13124 } else if (first_sdma <= i && i < last_sdma) { 13125 idx = i - first_sdma; 13126 sde = &dd->per_sdma[idx]; 13127 handler = sdma_interrupt; 13128 arg = sde; 13129 snprintf(name, sizeof(name), 13130 DRIVER_NAME "_%d sdma%d", dd->unit, idx); 13131 err_info = "sdma"; 13132 remap_sdma_interrupts(dd, idx, i); 13133 me->type = IRQ_SDMA; 13134 } else if (first_rx <= i && i < last_rx) { 13135 idx = i - first_rx; 13136 rcd = hfi1_rcd_get_by_index_safe(dd, idx); 13137 if (rcd) { 13138 /* 13139 * Set the interrupt register and mask for this 13140 * context's interrupt. 13141 */ 13142 rcd->ireg = (IS_RCVAVAIL_START + idx) / 64; 13143 rcd->imask = ((u64)1) << 13144 ((IS_RCVAVAIL_START + idx) % 64); 13145 handler = receive_context_interrupt; 13146 thread = receive_context_thread; 13147 arg = rcd; 13148 snprintf(name, sizeof(name), 13149 DRIVER_NAME "_%d kctxt%d", 13150 dd->unit, idx); 13151 err_info = "receive context"; 13152 remap_intr(dd, IS_RCVAVAIL_START + idx, i); 13153 me->type = IRQ_RCVCTXT; 13154 rcd->msix_intr = i; 13155 hfi1_rcd_put(rcd); 13156 } 13157 } else { 13158 /* not in our expected range - complain, then 13159 * ignore it 13160 */ 13161 dd_dev_err(dd, 13162 "Unexpected extra MSI-X interrupt %d\n", i); 13163 continue; 13164 } 13165 /* no argument, no interrupt */ 13166 if (!arg) 13167 continue; 13168 /* make sure the name is terminated */ 13169 name[sizeof(name) - 1] = 0; 13170 me->irq = pci_irq_vector(dd->pcidev, i); 13171 ret = pci_request_irq(dd->pcidev, i, handler, thread, arg, 13172 name); 13173 if (ret) { 13174 dd_dev_err(dd, 13175 "unable to allocate %s interrupt, irq %d, index %d, err %d\n", 13176 err_info, me->irq, idx, ret); 13177 return ret; 13178 } 13179 /* 13180 * assign arg after pci_request_irq call, so it will be 13181 * cleaned up 13182 */ 13183 me->arg = arg; 13184 13185 ret = hfi1_get_irq_affinity(dd, me); 13186 if (ret) 13187 dd_dev_err(dd, "unable to pin IRQ %d\n", ret); 13188 } 13189 13190 return ret; 13191 } 13192 13193 void hfi1_vnic_synchronize_irq(struct hfi1_devdata *dd) 13194 { 13195 int i; 13196 13197 if (!dd->num_msix_entries) { 13198 synchronize_irq(pci_irq_vector(dd->pcidev, 0)); 13199 return; 13200 } 13201 13202 for (i = 0; i < dd->vnic.num_ctxt; i++) { 13203 struct hfi1_ctxtdata *rcd = dd->vnic.ctxt[i]; 13204 struct hfi1_msix_entry *me = &dd->msix_entries[rcd->msix_intr]; 13205 13206 synchronize_irq(me->irq); 13207 } 13208 } 13209 13210 void hfi1_reset_vnic_msix_info(struct hfi1_ctxtdata *rcd) 13211 { 13212 struct hfi1_devdata *dd = rcd->dd; 13213 struct hfi1_msix_entry *me = &dd->msix_entries[rcd->msix_intr]; 13214 13215 if (!me->arg) /* => no irq, no affinity */ 13216 return; 13217 13218 hfi1_put_irq_affinity(dd, me); 13219 pci_free_irq(dd->pcidev, rcd->msix_intr, me->arg); 13220 13221 me->arg = NULL; 13222 } 13223 13224 void hfi1_set_vnic_msix_info(struct hfi1_ctxtdata *rcd) 13225 { 13226 struct hfi1_devdata *dd = rcd->dd; 13227 struct hfi1_msix_entry *me; 13228 int idx = rcd->ctxt; 13229 void *arg = rcd; 13230 int ret; 13231 13232 rcd->msix_intr = dd->vnic.msix_idx++; 13233 me = &dd->msix_entries[rcd->msix_intr]; 13234 13235 /* 13236 * Set the interrupt register and mask for this 13237 * context's interrupt. 13238 */ 13239 rcd->ireg = (IS_RCVAVAIL_START + idx) / 64; 13240 rcd->imask = ((u64)1) << 13241 ((IS_RCVAVAIL_START + idx) % 64); 13242 me->type = IRQ_RCVCTXT; 13243 me->irq = pci_irq_vector(dd->pcidev, rcd->msix_intr); 13244 remap_intr(dd, IS_RCVAVAIL_START + idx, rcd->msix_intr); 13245 13246 ret = pci_request_irq(dd->pcidev, rcd->msix_intr, 13247 receive_context_interrupt, 13248 receive_context_thread, arg, 13249 DRIVER_NAME "_%d kctxt%d", dd->unit, idx); 13250 if (ret) { 13251 dd_dev_err(dd, "vnic irq request (irq %d, idx %d) fail %d\n", 13252 me->irq, idx, ret); 13253 return; 13254 } 13255 /* 13256 * assign arg after pci_request_irq call, so it will be 13257 * cleaned up 13258 */ 13259 me->arg = arg; 13260 13261 ret = hfi1_get_irq_affinity(dd, me); 13262 if (ret) { 13263 dd_dev_err(dd, 13264 "unable to pin IRQ %d\n", ret); 13265 pci_free_irq(dd->pcidev, rcd->msix_intr, me->arg); 13266 } 13267 } 13268 13269 /* 13270 * Set the general handler to accept all interrupts, remap all 13271 * chip interrupts back to MSI-X 0. 13272 */ 13273 static void reset_interrupts(struct hfi1_devdata *dd) 13274 { 13275 int i; 13276 13277 /* all interrupts handled by the general handler */ 13278 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 13279 dd->gi_mask[i] = ~(u64)0; 13280 13281 /* all chip interrupts map to MSI-X 0 */ 13282 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13283 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 13284 } 13285 13286 static int set_up_interrupts(struct hfi1_devdata *dd) 13287 { 13288 u32 total; 13289 int ret, request; 13290 int single_interrupt = 0; /* we expect to have all the interrupts */ 13291 13292 /* 13293 * Interrupt count: 13294 * 1 general, "slow path" interrupt (includes the SDMA engines 13295 * slow source, SDMACleanupDone) 13296 * N interrupts - one per used SDMA engine 13297 * M interrupt - one per kernel receive context 13298 * V interrupt - one for each VNIC context 13299 */ 13300 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_vnic_contexts; 13301 13302 /* ask for MSI-X interrupts */ 13303 request = request_msix(dd, total); 13304 if (request < 0) { 13305 ret = request; 13306 goto fail; 13307 } else if (request == 0) { 13308 /* using INTx */ 13309 /* dd->num_msix_entries already zero */ 13310 single_interrupt = 1; 13311 dd_dev_err(dd, "MSI-X failed, using INTx interrupts\n"); 13312 } else if (request < total) { 13313 /* using MSI-X, with reduced interrupts */ 13314 dd_dev_err(dd, "reduced interrupt found, wanted %u, got %u\n", 13315 total, request); 13316 ret = -EINVAL; 13317 goto fail; 13318 } else { 13319 dd->msix_entries = kcalloc(total, sizeof(*dd->msix_entries), 13320 GFP_KERNEL); 13321 if (!dd->msix_entries) { 13322 ret = -ENOMEM; 13323 goto fail; 13324 } 13325 /* using MSI-X */ 13326 dd->num_msix_entries = total; 13327 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total); 13328 } 13329 13330 /* mask all interrupts */ 13331 set_intr_state(dd, 0); 13332 /* clear all pending interrupts */ 13333 clear_all_interrupts(dd); 13334 13335 /* reset general handler mask, chip MSI-X mappings */ 13336 reset_interrupts(dd); 13337 13338 if (single_interrupt) 13339 ret = request_intx_irq(dd); 13340 else 13341 ret = request_msix_irqs(dd); 13342 if (ret) 13343 goto fail; 13344 13345 return 0; 13346 13347 fail: 13348 clean_up_interrupts(dd); 13349 return ret; 13350 } 13351 13352 /* 13353 * Set up context values in dd. Sets: 13354 * 13355 * num_rcv_contexts - number of contexts being used 13356 * n_krcv_queues - number of kernel contexts 13357 * first_dyn_alloc_ctxt - first dynamically allocated context 13358 * in array of contexts 13359 * freectxts - number of free user contexts 13360 * num_send_contexts - number of PIO send contexts being used 13361 * num_vnic_contexts - number of contexts reserved for VNIC 13362 */ 13363 static int set_up_context_variables(struct hfi1_devdata *dd) 13364 { 13365 unsigned long num_kernel_contexts; 13366 u16 num_vnic_contexts = HFI1_NUM_VNIC_CTXT; 13367 int total_contexts; 13368 int ret; 13369 unsigned ngroups; 13370 int qos_rmt_count; 13371 int user_rmt_reduced; 13372 u32 n_usr_ctxts; 13373 13374 /* 13375 * Kernel receive contexts: 13376 * - Context 0 - control context (VL15/multicast/error) 13377 * - Context 1 - first kernel context 13378 * - Context 2 - second kernel context 13379 * ... 13380 */ 13381 if (n_krcvqs) 13382 /* 13383 * n_krcvqs is the sum of module parameter kernel receive 13384 * contexts, krcvqs[]. It does not include the control 13385 * context, so add that. 13386 */ 13387 num_kernel_contexts = n_krcvqs + 1; 13388 else 13389 num_kernel_contexts = DEFAULT_KRCVQS + 1; 13390 /* 13391 * Every kernel receive context needs an ACK send context. 13392 * one send context is allocated for each VL{0-7} and VL15 13393 */ 13394 if (num_kernel_contexts > (dd->chip_send_contexts - num_vls - 1)) { 13395 dd_dev_err(dd, 13396 "Reducing # kernel rcv contexts to: %d, from %lu\n", 13397 (int)(dd->chip_send_contexts - num_vls - 1), 13398 num_kernel_contexts); 13399 num_kernel_contexts = dd->chip_send_contexts - num_vls - 1; 13400 } 13401 13402 /* Accommodate VNIC contexts if possible */ 13403 if ((num_kernel_contexts + num_vnic_contexts) > dd->chip_rcv_contexts) { 13404 dd_dev_err(dd, "No receive contexts available for VNIC\n"); 13405 num_vnic_contexts = 0; 13406 } 13407 total_contexts = num_kernel_contexts + num_vnic_contexts; 13408 13409 /* 13410 * User contexts: 13411 * - default to 1 user context per real (non-HT) CPU core if 13412 * num_user_contexts is negative 13413 */ 13414 if (num_user_contexts < 0) 13415 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask); 13416 else 13417 n_usr_ctxts = num_user_contexts; 13418 /* 13419 * Adjust the counts given a global max. 13420 */ 13421 if (total_contexts + n_usr_ctxts > dd->chip_rcv_contexts) { 13422 dd_dev_err(dd, 13423 "Reducing # user receive contexts to: %d, from %u\n", 13424 (int)(dd->chip_rcv_contexts - total_contexts), 13425 n_usr_ctxts); 13426 /* recalculate */ 13427 n_usr_ctxts = dd->chip_rcv_contexts - total_contexts; 13428 } 13429 13430 /* each user context requires an entry in the RMT */ 13431 qos_rmt_count = qos_rmt_entries(dd, NULL, NULL); 13432 if (qos_rmt_count + n_usr_ctxts > NUM_MAP_ENTRIES) { 13433 user_rmt_reduced = NUM_MAP_ENTRIES - qos_rmt_count; 13434 dd_dev_err(dd, 13435 "RMT size is reducing the number of user receive contexts from %u to %d\n", 13436 n_usr_ctxts, 13437 user_rmt_reduced); 13438 /* recalculate */ 13439 n_usr_ctxts = user_rmt_reduced; 13440 } 13441 13442 total_contexts += n_usr_ctxts; 13443 13444 /* the first N are kernel contexts, the rest are user/vnic contexts */ 13445 dd->num_rcv_contexts = total_contexts; 13446 dd->n_krcv_queues = num_kernel_contexts; 13447 dd->first_dyn_alloc_ctxt = num_kernel_contexts; 13448 dd->num_vnic_contexts = num_vnic_contexts; 13449 dd->num_user_contexts = n_usr_ctxts; 13450 dd->freectxts = n_usr_ctxts; 13451 dd_dev_info(dd, 13452 "rcv contexts: chip %d, used %d (kernel %d, vnic %u, user %u)\n", 13453 (int)dd->chip_rcv_contexts, 13454 (int)dd->num_rcv_contexts, 13455 (int)dd->n_krcv_queues, 13456 dd->num_vnic_contexts, 13457 dd->num_user_contexts); 13458 13459 /* 13460 * Receive array allocation: 13461 * All RcvArray entries are divided into groups of 8. This 13462 * is required by the hardware and will speed up writes to 13463 * consecutive entries by using write-combining of the entire 13464 * cacheline. 13465 * 13466 * The number of groups are evenly divided among all contexts. 13467 * any left over groups will be given to the first N user 13468 * contexts. 13469 */ 13470 dd->rcv_entries.group_size = RCV_INCREMENT; 13471 ngroups = dd->chip_rcv_array_count / dd->rcv_entries.group_size; 13472 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts; 13473 dd->rcv_entries.nctxt_extra = ngroups - 13474 (dd->num_rcv_contexts * dd->rcv_entries.ngroups); 13475 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n", 13476 dd->rcv_entries.ngroups, 13477 dd->rcv_entries.nctxt_extra); 13478 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size > 13479 MAX_EAGER_ENTRIES * 2) { 13480 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) / 13481 dd->rcv_entries.group_size; 13482 dd_dev_info(dd, 13483 "RcvArray group count too high, change to %u\n", 13484 dd->rcv_entries.ngroups); 13485 dd->rcv_entries.nctxt_extra = 0; 13486 } 13487 /* 13488 * PIO send contexts 13489 */ 13490 ret = init_sc_pools_and_sizes(dd); 13491 if (ret >= 0) { /* success */ 13492 dd->num_send_contexts = ret; 13493 dd_dev_info( 13494 dd, 13495 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n", 13496 dd->chip_send_contexts, 13497 dd->num_send_contexts, 13498 dd->sc_sizes[SC_KERNEL].count, 13499 dd->sc_sizes[SC_ACK].count, 13500 dd->sc_sizes[SC_USER].count, 13501 dd->sc_sizes[SC_VL15].count); 13502 ret = 0; /* success */ 13503 } 13504 13505 return ret; 13506 } 13507 13508 /* 13509 * Set the device/port partition key table. The MAD code 13510 * will ensure that, at least, the partial management 13511 * partition key is present in the table. 13512 */ 13513 static void set_partition_keys(struct hfi1_pportdata *ppd) 13514 { 13515 struct hfi1_devdata *dd = ppd->dd; 13516 u64 reg = 0; 13517 int i; 13518 13519 dd_dev_info(dd, "Setting partition keys\n"); 13520 for (i = 0; i < hfi1_get_npkeys(dd); i++) { 13521 reg |= (ppd->pkeys[i] & 13522 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) << 13523 ((i % 4) * 13524 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT); 13525 /* Each register holds 4 PKey values. */ 13526 if ((i % 4) == 3) { 13527 write_csr(dd, RCV_PARTITION_KEY + 13528 ((i - 3) * 2), reg); 13529 reg = 0; 13530 } 13531 } 13532 13533 /* Always enable HW pkeys check when pkeys table is set */ 13534 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK); 13535 } 13536 13537 /* 13538 * These CSRs and memories are uninitialized on reset and must be 13539 * written before reading to set the ECC/parity bits. 13540 * 13541 * NOTE: All user context CSRs that are not mmaped write-only 13542 * (e.g. the TID flows) must be initialized even if the driver never 13543 * reads them. 13544 */ 13545 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd) 13546 { 13547 int i, j; 13548 13549 /* CceIntMap */ 13550 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13551 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 13552 13553 /* SendCtxtCreditReturnAddr */ 13554 for (i = 0; i < dd->chip_send_contexts; i++) 13555 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13556 13557 /* PIO Send buffers */ 13558 /* SDMA Send buffers */ 13559 /* 13560 * These are not normally read, and (presently) have no method 13561 * to be read, so are not pre-initialized 13562 */ 13563 13564 /* RcvHdrAddr */ 13565 /* RcvHdrTailAddr */ 13566 /* RcvTidFlowTable */ 13567 for (i = 0; i < dd->chip_rcv_contexts; i++) { 13568 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13569 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13570 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) 13571 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0); 13572 } 13573 13574 /* RcvArray */ 13575 for (i = 0; i < dd->chip_rcv_array_count; i++) 13576 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0); 13577 13578 /* RcvQPMapTable */ 13579 for (i = 0; i < 32; i++) 13580 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13581 } 13582 13583 /* 13584 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus. 13585 */ 13586 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits, 13587 u64 ctrl_bits) 13588 { 13589 unsigned long timeout; 13590 u64 reg; 13591 13592 /* is the condition present? */ 13593 reg = read_csr(dd, CCE_STATUS); 13594 if ((reg & status_bits) == 0) 13595 return; 13596 13597 /* clear the condition */ 13598 write_csr(dd, CCE_CTRL, ctrl_bits); 13599 13600 /* wait for the condition to clear */ 13601 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT); 13602 while (1) { 13603 reg = read_csr(dd, CCE_STATUS); 13604 if ((reg & status_bits) == 0) 13605 return; 13606 if (time_after(jiffies, timeout)) { 13607 dd_dev_err(dd, 13608 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n", 13609 status_bits, reg & status_bits); 13610 return; 13611 } 13612 udelay(1); 13613 } 13614 } 13615 13616 /* set CCE CSRs to chip reset defaults */ 13617 static void reset_cce_csrs(struct hfi1_devdata *dd) 13618 { 13619 int i; 13620 13621 /* CCE_REVISION read-only */ 13622 /* CCE_REVISION2 read-only */ 13623 /* CCE_CTRL - bits clear automatically */ 13624 /* CCE_STATUS read-only, use CceCtrl to clear */ 13625 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK); 13626 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK); 13627 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK); 13628 for (i = 0; i < CCE_NUM_SCRATCH; i++) 13629 write_csr(dd, CCE_SCRATCH + (8 * i), 0); 13630 /* CCE_ERR_STATUS read-only */ 13631 write_csr(dd, CCE_ERR_MASK, 0); 13632 write_csr(dd, CCE_ERR_CLEAR, ~0ull); 13633 /* CCE_ERR_FORCE leave alone */ 13634 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++) 13635 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0); 13636 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR); 13637 /* CCE_PCIE_CTRL leave alone */ 13638 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) { 13639 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0); 13640 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i), 13641 CCE_MSIX_TABLE_UPPER_RESETCSR); 13642 } 13643 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) { 13644 /* CCE_MSIX_PBA read-only */ 13645 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull); 13646 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull); 13647 } 13648 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13649 write_csr(dd, CCE_INT_MAP, 0); 13650 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 13651 /* CCE_INT_STATUS read-only */ 13652 write_csr(dd, CCE_INT_MASK + (8 * i), 0); 13653 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull); 13654 /* CCE_INT_FORCE leave alone */ 13655 /* CCE_INT_BLOCKED read-only */ 13656 } 13657 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++) 13658 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0); 13659 } 13660 13661 /* set MISC CSRs to chip reset defaults */ 13662 static void reset_misc_csrs(struct hfi1_devdata *dd) 13663 { 13664 int i; 13665 13666 for (i = 0; i < 32; i++) { 13667 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0); 13668 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0); 13669 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0); 13670 } 13671 /* 13672 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can 13673 * only be written 128-byte chunks 13674 */ 13675 /* init RSA engine to clear lingering errors */ 13676 write_csr(dd, MISC_CFG_RSA_CMD, 1); 13677 write_csr(dd, MISC_CFG_RSA_MU, 0); 13678 write_csr(dd, MISC_CFG_FW_CTRL, 0); 13679 /* MISC_STS_8051_DIGEST read-only */ 13680 /* MISC_STS_SBM_DIGEST read-only */ 13681 /* MISC_STS_PCIE_DIGEST read-only */ 13682 /* MISC_STS_FAB_DIGEST read-only */ 13683 /* MISC_ERR_STATUS read-only */ 13684 write_csr(dd, MISC_ERR_MASK, 0); 13685 write_csr(dd, MISC_ERR_CLEAR, ~0ull); 13686 /* MISC_ERR_FORCE leave alone */ 13687 } 13688 13689 /* set TXE CSRs to chip reset defaults */ 13690 static void reset_txe_csrs(struct hfi1_devdata *dd) 13691 { 13692 int i; 13693 13694 /* 13695 * TXE Kernel CSRs 13696 */ 13697 write_csr(dd, SEND_CTRL, 0); 13698 __cm_reset(dd, 0); /* reset CM internal state */ 13699 /* SEND_CONTEXTS read-only */ 13700 /* SEND_DMA_ENGINES read-only */ 13701 /* SEND_PIO_MEM_SIZE read-only */ 13702 /* SEND_DMA_MEM_SIZE read-only */ 13703 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0); 13704 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */ 13705 /* SEND_PIO_ERR_STATUS read-only */ 13706 write_csr(dd, SEND_PIO_ERR_MASK, 0); 13707 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull); 13708 /* SEND_PIO_ERR_FORCE leave alone */ 13709 /* SEND_DMA_ERR_STATUS read-only */ 13710 write_csr(dd, SEND_DMA_ERR_MASK, 0); 13711 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull); 13712 /* SEND_DMA_ERR_FORCE leave alone */ 13713 /* SEND_EGRESS_ERR_STATUS read-only */ 13714 write_csr(dd, SEND_EGRESS_ERR_MASK, 0); 13715 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull); 13716 /* SEND_EGRESS_ERR_FORCE leave alone */ 13717 write_csr(dd, SEND_BTH_QP, 0); 13718 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0); 13719 write_csr(dd, SEND_SC2VLT0, 0); 13720 write_csr(dd, SEND_SC2VLT1, 0); 13721 write_csr(dd, SEND_SC2VLT2, 0); 13722 write_csr(dd, SEND_SC2VLT3, 0); 13723 write_csr(dd, SEND_LEN_CHECK0, 0); 13724 write_csr(dd, SEND_LEN_CHECK1, 0); 13725 /* SEND_ERR_STATUS read-only */ 13726 write_csr(dd, SEND_ERR_MASK, 0); 13727 write_csr(dd, SEND_ERR_CLEAR, ~0ull); 13728 /* SEND_ERR_FORCE read-only */ 13729 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++) 13730 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0); 13731 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++) 13732 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0); 13733 for (i = 0; i < dd->chip_send_contexts / NUM_CONTEXTS_PER_SET; i++) 13734 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0); 13735 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++) 13736 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0); 13737 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++) 13738 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0); 13739 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR); 13740 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR); 13741 /* SEND_CM_CREDIT_USED_STATUS read-only */ 13742 write_csr(dd, SEND_CM_TIMER_CTRL, 0); 13743 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0); 13744 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0); 13745 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0); 13746 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0); 13747 for (i = 0; i < TXE_NUM_DATA_VL; i++) 13748 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 13749 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 13750 /* SEND_CM_CREDIT_USED_VL read-only */ 13751 /* SEND_CM_CREDIT_USED_VL15 read-only */ 13752 /* SEND_EGRESS_CTXT_STATUS read-only */ 13753 /* SEND_EGRESS_SEND_DMA_STATUS read-only */ 13754 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull); 13755 /* SEND_EGRESS_ERR_INFO read-only */ 13756 /* SEND_EGRESS_ERR_SOURCE read-only */ 13757 13758 /* 13759 * TXE Per-Context CSRs 13760 */ 13761 for (i = 0; i < dd->chip_send_contexts; i++) { 13762 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 13763 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0); 13764 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13765 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0); 13766 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0); 13767 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull); 13768 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0); 13769 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0); 13770 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0); 13771 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0); 13772 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0); 13773 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0); 13774 } 13775 13776 /* 13777 * TXE Per-SDMA CSRs 13778 */ 13779 for (i = 0; i < dd->chip_sdma_engines; i++) { 13780 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 13781 /* SEND_DMA_STATUS read-only */ 13782 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0); 13783 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0); 13784 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0); 13785 /* SEND_DMA_HEAD read-only */ 13786 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0); 13787 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0); 13788 /* SEND_DMA_IDLE_CNT read-only */ 13789 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0); 13790 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0); 13791 /* SEND_DMA_DESC_FETCHED_CNT read-only */ 13792 /* SEND_DMA_ENG_ERR_STATUS read-only */ 13793 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0); 13794 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull); 13795 /* SEND_DMA_ENG_ERR_FORCE leave alone */ 13796 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0); 13797 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0); 13798 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0); 13799 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0); 13800 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0); 13801 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0); 13802 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0); 13803 } 13804 } 13805 13806 /* 13807 * Expect on entry: 13808 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0 13809 */ 13810 static void init_rbufs(struct hfi1_devdata *dd) 13811 { 13812 u64 reg; 13813 int count; 13814 13815 /* 13816 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are 13817 * clear. 13818 */ 13819 count = 0; 13820 while (1) { 13821 reg = read_csr(dd, RCV_STATUS); 13822 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK 13823 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0) 13824 break; 13825 /* 13826 * Give up after 1ms - maximum wait time. 13827 * 13828 * RBuf size is 136KiB. Slowest possible is PCIe Gen1 x1 at 13829 * 250MB/s bandwidth. Lower rate to 66% for overhead to get: 13830 * 136 KB / (66% * 250MB/s) = 844us 13831 */ 13832 if (count++ > 500) { 13833 dd_dev_err(dd, 13834 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n", 13835 __func__, reg); 13836 break; 13837 } 13838 udelay(2); /* do not busy-wait the CSR */ 13839 } 13840 13841 /* start the init - expect RcvCtrl to be 0 */ 13842 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK); 13843 13844 /* 13845 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief 13846 * period after the write before RcvStatus.RxRbufInitDone is valid. 13847 * The delay in the first run through the loop below is sufficient and 13848 * required before the first read of RcvStatus.RxRbufInintDone. 13849 */ 13850 read_csr(dd, RCV_CTRL); 13851 13852 /* wait for the init to finish */ 13853 count = 0; 13854 while (1) { 13855 /* delay is required first time through - see above */ 13856 udelay(2); /* do not busy-wait the CSR */ 13857 reg = read_csr(dd, RCV_STATUS); 13858 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK)) 13859 break; 13860 13861 /* give up after 100us - slowest possible at 33MHz is 73us */ 13862 if (count++ > 50) { 13863 dd_dev_err(dd, 13864 "%s: RcvStatus.RxRbufInit not set, continuing\n", 13865 __func__); 13866 break; 13867 } 13868 } 13869 } 13870 13871 /* set RXE CSRs to chip reset defaults */ 13872 static void reset_rxe_csrs(struct hfi1_devdata *dd) 13873 { 13874 int i, j; 13875 13876 /* 13877 * RXE Kernel CSRs 13878 */ 13879 write_csr(dd, RCV_CTRL, 0); 13880 init_rbufs(dd); 13881 /* RCV_STATUS read-only */ 13882 /* RCV_CONTEXTS read-only */ 13883 /* RCV_ARRAY_CNT read-only */ 13884 /* RCV_BUF_SIZE read-only */ 13885 write_csr(dd, RCV_BTH_QP, 0); 13886 write_csr(dd, RCV_MULTICAST, 0); 13887 write_csr(dd, RCV_BYPASS, 0); 13888 write_csr(dd, RCV_VL15, 0); 13889 /* this is a clear-down */ 13890 write_csr(dd, RCV_ERR_INFO, 13891 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK); 13892 /* RCV_ERR_STATUS read-only */ 13893 write_csr(dd, RCV_ERR_MASK, 0); 13894 write_csr(dd, RCV_ERR_CLEAR, ~0ull); 13895 /* RCV_ERR_FORCE leave alone */ 13896 for (i = 0; i < 32; i++) 13897 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13898 for (i = 0; i < 4; i++) 13899 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0); 13900 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++) 13901 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0); 13902 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++) 13903 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0); 13904 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) 13905 clear_rsm_rule(dd, i); 13906 for (i = 0; i < 32; i++) 13907 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0); 13908 13909 /* 13910 * RXE Kernel and User Per-Context CSRs 13911 */ 13912 for (i = 0; i < dd->chip_rcv_contexts; i++) { 13913 /* kernel */ 13914 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0); 13915 /* RCV_CTXT_STATUS read-only */ 13916 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0); 13917 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0); 13918 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0); 13919 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13920 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0); 13921 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0); 13922 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0); 13923 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13924 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0); 13925 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0); 13926 13927 /* user */ 13928 /* RCV_HDR_TAIL read-only */ 13929 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0); 13930 /* RCV_EGR_INDEX_TAIL read-only */ 13931 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0); 13932 /* RCV_EGR_OFFSET_TAIL read-only */ 13933 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) { 13934 write_uctxt_csr(dd, i, 13935 RCV_TID_FLOW_TABLE + (8 * j), 0); 13936 } 13937 } 13938 } 13939 13940 /* 13941 * Set sc2vl tables. 13942 * 13943 * They power on to zeros, so to avoid send context errors 13944 * they need to be set: 13945 * 13946 * SC 0-7 -> VL 0-7 (respectively) 13947 * SC 15 -> VL 15 13948 * otherwise 13949 * -> VL 0 13950 */ 13951 static void init_sc2vl_tables(struct hfi1_devdata *dd) 13952 { 13953 int i; 13954 /* init per architecture spec, constrained by hardware capability */ 13955 13956 /* HFI maps sent packets */ 13957 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL( 13958 0, 13959 0, 0, 1, 1, 13960 2, 2, 3, 3, 13961 4, 4, 5, 5, 13962 6, 6, 7, 7)); 13963 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL( 13964 1, 13965 8, 0, 9, 0, 13966 10, 0, 11, 0, 13967 12, 0, 13, 0, 13968 14, 0, 15, 15)); 13969 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL( 13970 2, 13971 16, 0, 17, 0, 13972 18, 0, 19, 0, 13973 20, 0, 21, 0, 13974 22, 0, 23, 0)); 13975 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL( 13976 3, 13977 24, 0, 25, 0, 13978 26, 0, 27, 0, 13979 28, 0, 29, 0, 13980 30, 0, 31, 0)); 13981 13982 /* DC maps received packets */ 13983 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL( 13984 15_0, 13985 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 13986 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15)); 13987 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL( 13988 31_16, 13989 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 13990 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0)); 13991 13992 /* initialize the cached sc2vl values consistently with h/w */ 13993 for (i = 0; i < 32; i++) { 13994 if (i < 8 || i == 15) 13995 *((u8 *)(dd->sc2vl) + i) = (u8)i; 13996 else 13997 *((u8 *)(dd->sc2vl) + i) = 0; 13998 } 13999 } 14000 14001 /* 14002 * Read chip sizes and then reset parts to sane, disabled, values. We cannot 14003 * depend on the chip going through a power-on reset - a driver may be loaded 14004 * and unloaded many times. 14005 * 14006 * Do not write any CSR values to the chip in this routine - there may be 14007 * a reset following the (possible) FLR in this routine. 14008 * 14009 */ 14010 static int init_chip(struct hfi1_devdata *dd) 14011 { 14012 int i; 14013 int ret = 0; 14014 14015 /* 14016 * Put the HFI CSRs in a known state. 14017 * Combine this with a DC reset. 14018 * 14019 * Stop the device from doing anything while we do a 14020 * reset. We know there are no other active users of 14021 * the device since we are now in charge. Turn off 14022 * off all outbound and inbound traffic and make sure 14023 * the device does not generate any interrupts. 14024 */ 14025 14026 /* disable send contexts and SDMA engines */ 14027 write_csr(dd, SEND_CTRL, 0); 14028 for (i = 0; i < dd->chip_send_contexts; i++) 14029 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 14030 for (i = 0; i < dd->chip_sdma_engines; i++) 14031 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 14032 /* disable port (turn off RXE inbound traffic) and contexts */ 14033 write_csr(dd, RCV_CTRL, 0); 14034 for (i = 0; i < dd->chip_rcv_contexts; i++) 14035 write_csr(dd, RCV_CTXT_CTRL, 0); 14036 /* mask all interrupt sources */ 14037 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 14038 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull); 14039 14040 /* 14041 * DC Reset: do a full DC reset before the register clear. 14042 * A recommended length of time to hold is one CSR read, 14043 * so reread the CceDcCtrl. Then, hold the DC in reset 14044 * across the clear. 14045 */ 14046 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK); 14047 (void)read_csr(dd, CCE_DC_CTRL); 14048 14049 if (use_flr) { 14050 /* 14051 * A FLR will reset the SPC core and part of the PCIe. 14052 * The parts that need to be restored have already been 14053 * saved. 14054 */ 14055 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 14056 14057 /* do the FLR, the DC reset will remain */ 14058 pcie_flr(dd->pcidev); 14059 14060 /* restore command and BARs */ 14061 ret = restore_pci_variables(dd); 14062 if (ret) { 14063 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 14064 __func__); 14065 return ret; 14066 } 14067 14068 if (is_ax(dd)) { 14069 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 14070 pcie_flr(dd->pcidev); 14071 ret = restore_pci_variables(dd); 14072 if (ret) { 14073 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 14074 __func__); 14075 return ret; 14076 } 14077 } 14078 } else { 14079 dd_dev_info(dd, "Resetting CSRs with writes\n"); 14080 reset_cce_csrs(dd); 14081 reset_txe_csrs(dd); 14082 reset_rxe_csrs(dd); 14083 reset_misc_csrs(dd); 14084 } 14085 /* clear the DC reset */ 14086 write_csr(dd, CCE_DC_CTRL, 0); 14087 14088 /* Set the LED off */ 14089 setextled(dd, 0); 14090 14091 /* 14092 * Clear the QSFP reset. 14093 * An FLR enforces a 0 on all out pins. The driver does not touch 14094 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and 14095 * anything plugged constantly in reset, if it pays attention 14096 * to RESET_N. 14097 * Prime examples of this are optical cables. Set all pins high. 14098 * I2CCLK and I2CDAT will change per direction, and INT_N and 14099 * MODPRS_N are input only and their value is ignored. 14100 */ 14101 write_csr(dd, ASIC_QSFP1_OUT, 0x1f); 14102 write_csr(dd, ASIC_QSFP2_OUT, 0x1f); 14103 init_chip_resources(dd); 14104 return ret; 14105 } 14106 14107 static void init_early_variables(struct hfi1_devdata *dd) 14108 { 14109 int i; 14110 14111 /* assign link credit variables */ 14112 dd->vau = CM_VAU; 14113 dd->link_credits = CM_GLOBAL_CREDITS; 14114 if (is_ax(dd)) 14115 dd->link_credits--; 14116 dd->vcu = cu_to_vcu(hfi1_cu); 14117 /* enough room for 8 MAD packets plus header - 17K */ 14118 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau); 14119 if (dd->vl15_init > dd->link_credits) 14120 dd->vl15_init = dd->link_credits; 14121 14122 write_uninitialized_csrs_and_memories(dd); 14123 14124 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 14125 for (i = 0; i < dd->num_pports; i++) { 14126 struct hfi1_pportdata *ppd = &dd->pport[i]; 14127 14128 set_partition_keys(ppd); 14129 } 14130 init_sc2vl_tables(dd); 14131 } 14132 14133 static void init_kdeth_qp(struct hfi1_devdata *dd) 14134 { 14135 /* user changed the KDETH_QP */ 14136 if (kdeth_qp != 0 && kdeth_qp >= 0xff) { 14137 /* out of range or illegal value */ 14138 dd_dev_err(dd, "Invalid KDETH queue pair prefix, ignoring"); 14139 kdeth_qp = 0; 14140 } 14141 if (kdeth_qp == 0) /* not set, or failed range check */ 14142 kdeth_qp = DEFAULT_KDETH_QP; 14143 14144 write_csr(dd, SEND_BTH_QP, 14145 (kdeth_qp & SEND_BTH_QP_KDETH_QP_MASK) << 14146 SEND_BTH_QP_KDETH_QP_SHIFT); 14147 14148 write_csr(dd, RCV_BTH_QP, 14149 (kdeth_qp & RCV_BTH_QP_KDETH_QP_MASK) << 14150 RCV_BTH_QP_KDETH_QP_SHIFT); 14151 } 14152 14153 /** 14154 * init_qpmap_table 14155 * @dd - device data 14156 * @first_ctxt - first context 14157 * @last_ctxt - first context 14158 * 14159 * This return sets the qpn mapping table that 14160 * is indexed by qpn[8:1]. 14161 * 14162 * The routine will round robin the 256 settings 14163 * from first_ctxt to last_ctxt. 14164 * 14165 * The first/last looks ahead to having specialized 14166 * receive contexts for mgmt and bypass. Normal 14167 * verbs traffic will assumed to be on a range 14168 * of receive contexts. 14169 */ 14170 static void init_qpmap_table(struct hfi1_devdata *dd, 14171 u32 first_ctxt, 14172 u32 last_ctxt) 14173 { 14174 u64 reg = 0; 14175 u64 regno = RCV_QP_MAP_TABLE; 14176 int i; 14177 u64 ctxt = first_ctxt; 14178 14179 for (i = 0; i < 256; i++) { 14180 reg |= ctxt << (8 * (i % 8)); 14181 ctxt++; 14182 if (ctxt > last_ctxt) 14183 ctxt = first_ctxt; 14184 if (i % 8 == 7) { 14185 write_csr(dd, regno, reg); 14186 reg = 0; 14187 regno += 8; 14188 } 14189 } 14190 14191 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK 14192 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK); 14193 } 14194 14195 struct rsm_map_table { 14196 u64 map[NUM_MAP_REGS]; 14197 unsigned int used; 14198 }; 14199 14200 struct rsm_rule_data { 14201 u8 offset; 14202 u8 pkt_type; 14203 u32 field1_off; 14204 u32 field2_off; 14205 u32 index1_off; 14206 u32 index1_width; 14207 u32 index2_off; 14208 u32 index2_width; 14209 u32 mask1; 14210 u32 value1; 14211 u32 mask2; 14212 u32 value2; 14213 }; 14214 14215 /* 14216 * Return an initialized RMT map table for users to fill in. OK if it 14217 * returns NULL, indicating no table. 14218 */ 14219 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd) 14220 { 14221 struct rsm_map_table *rmt; 14222 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */ 14223 14224 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL); 14225 if (rmt) { 14226 memset(rmt->map, rxcontext, sizeof(rmt->map)); 14227 rmt->used = 0; 14228 } 14229 14230 return rmt; 14231 } 14232 14233 /* 14234 * Write the final RMT map table to the chip and free the table. OK if 14235 * table is NULL. 14236 */ 14237 static void complete_rsm_map_table(struct hfi1_devdata *dd, 14238 struct rsm_map_table *rmt) 14239 { 14240 int i; 14241 14242 if (rmt) { 14243 /* write table to chip */ 14244 for (i = 0; i < NUM_MAP_REGS; i++) 14245 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]); 14246 14247 /* enable RSM */ 14248 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14249 } 14250 } 14251 14252 /* 14253 * Add a receive side mapping rule. 14254 */ 14255 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index, 14256 struct rsm_rule_data *rrd) 14257 { 14258 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 14259 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT | 14260 1ull << rule_index | /* enable bit */ 14261 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT); 14262 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 14263 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT | 14264 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT | 14265 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT | 14266 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT | 14267 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT | 14268 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT); 14269 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 14270 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT | 14271 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT | 14272 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT | 14273 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT); 14274 } 14275 14276 /* 14277 * Clear a receive side mapping rule. 14278 */ 14279 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) 14280 { 14281 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0); 14282 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0); 14283 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0); 14284 } 14285 14286 /* return the number of RSM map table entries that will be used for QOS */ 14287 static int qos_rmt_entries(struct hfi1_devdata *dd, unsigned int *mp, 14288 unsigned int *np) 14289 { 14290 int i; 14291 unsigned int m, n; 14292 u8 max_by_vl = 0; 14293 14294 /* is QOS active at all? */ 14295 if (dd->n_krcv_queues <= MIN_KERNEL_KCTXTS || 14296 num_vls == 1 || 14297 krcvqsset <= 1) 14298 goto no_qos; 14299 14300 /* determine bits for qpn */ 14301 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++) 14302 if (krcvqs[i] > max_by_vl) 14303 max_by_vl = krcvqs[i]; 14304 if (max_by_vl > 32) 14305 goto no_qos; 14306 m = ilog2(__roundup_pow_of_two(max_by_vl)); 14307 14308 /* determine bits for vl */ 14309 n = ilog2(__roundup_pow_of_two(num_vls)); 14310 14311 /* reject if too much is used */ 14312 if ((m + n) > 7) 14313 goto no_qos; 14314 14315 if (mp) 14316 *mp = m; 14317 if (np) 14318 *np = n; 14319 14320 return 1 << (m + n); 14321 14322 no_qos: 14323 if (mp) 14324 *mp = 0; 14325 if (np) 14326 *np = 0; 14327 return 0; 14328 } 14329 14330 /** 14331 * init_qos - init RX qos 14332 * @dd - device data 14333 * @rmt - RSM map table 14334 * 14335 * This routine initializes Rule 0 and the RSM map table to implement 14336 * quality of service (qos). 14337 * 14338 * If all of the limit tests succeed, qos is applied based on the array 14339 * interpretation of krcvqs where entry 0 is VL0. 14340 * 14341 * The number of vl bits (n) and the number of qpn bits (m) are computed to 14342 * feed both the RSM map table and the single rule. 14343 */ 14344 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt) 14345 { 14346 struct rsm_rule_data rrd; 14347 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m; 14348 unsigned int rmt_entries; 14349 u64 reg; 14350 14351 if (!rmt) 14352 goto bail; 14353 rmt_entries = qos_rmt_entries(dd, &m, &n); 14354 if (rmt_entries == 0) 14355 goto bail; 14356 qpns_per_vl = 1 << m; 14357 14358 /* enough room in the map table? */ 14359 rmt_entries = 1 << (m + n); 14360 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES) 14361 goto bail; 14362 14363 /* add qos entries to the the RSM map table */ 14364 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) { 14365 unsigned tctxt; 14366 14367 for (qpn = 0, tctxt = ctxt; 14368 krcvqs[i] && qpn < qpns_per_vl; qpn++) { 14369 unsigned idx, regoff, regidx; 14370 14371 /* generate the index the hardware will produce */ 14372 idx = rmt->used + ((qpn << n) ^ i); 14373 regoff = (idx % 8) * 8; 14374 regidx = idx / 8; 14375 /* replace default with context number */ 14376 reg = rmt->map[regidx]; 14377 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK 14378 << regoff); 14379 reg |= (u64)(tctxt++) << regoff; 14380 rmt->map[regidx] = reg; 14381 if (tctxt == ctxt + krcvqs[i]) 14382 tctxt = ctxt; 14383 } 14384 ctxt += krcvqs[i]; 14385 } 14386 14387 rrd.offset = rmt->used; 14388 rrd.pkt_type = 2; 14389 rrd.field1_off = LRH_BTH_MATCH_OFFSET; 14390 rrd.field2_off = LRH_SC_MATCH_OFFSET; 14391 rrd.index1_off = LRH_SC_SELECT_OFFSET; 14392 rrd.index1_width = n; 14393 rrd.index2_off = QPN_SELECT_OFFSET; 14394 rrd.index2_width = m + n; 14395 rrd.mask1 = LRH_BTH_MASK; 14396 rrd.value1 = LRH_BTH_VALUE; 14397 rrd.mask2 = LRH_SC_MASK; 14398 rrd.value2 = LRH_SC_VALUE; 14399 14400 /* add rule 0 */ 14401 add_rsm_rule(dd, RSM_INS_VERBS, &rrd); 14402 14403 /* mark RSM map entries as used */ 14404 rmt->used += rmt_entries; 14405 /* map everything else to the mcast/err/vl15 context */ 14406 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT); 14407 dd->qos_shift = n + 1; 14408 return; 14409 bail: 14410 dd->qos_shift = 1; 14411 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1); 14412 } 14413 14414 static void init_user_fecn_handling(struct hfi1_devdata *dd, 14415 struct rsm_map_table *rmt) 14416 { 14417 struct rsm_rule_data rrd; 14418 u64 reg; 14419 int i, idx, regoff, regidx; 14420 u8 offset; 14421 14422 /* there needs to be enough room in the map table */ 14423 if (rmt->used + dd->num_user_contexts >= NUM_MAP_ENTRIES) { 14424 dd_dev_err(dd, "User FECN handling disabled - too many user contexts allocated\n"); 14425 return; 14426 } 14427 14428 /* 14429 * RSM will extract the destination context as an index into the 14430 * map table. The destination contexts are a sequential block 14431 * in the range first_dyn_alloc_ctxt...num_rcv_contexts-1 (inclusive). 14432 * Map entries are accessed as offset + extracted value. Adjust 14433 * the added offset so this sequence can be placed anywhere in 14434 * the table - as long as the entries themselves do not wrap. 14435 * There are only enough bits in offset for the table size, so 14436 * start with that to allow for a "negative" offset. 14437 */ 14438 offset = (u8)(NUM_MAP_ENTRIES + (int)rmt->used - 14439 (int)dd->first_dyn_alloc_ctxt); 14440 14441 for (i = dd->first_dyn_alloc_ctxt, idx = rmt->used; 14442 i < dd->num_rcv_contexts; i++, idx++) { 14443 /* replace with identity mapping */ 14444 regoff = (idx % 8) * 8; 14445 regidx = idx / 8; 14446 reg = rmt->map[regidx]; 14447 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff); 14448 reg |= (u64)i << regoff; 14449 rmt->map[regidx] = reg; 14450 } 14451 14452 /* 14453 * For RSM intercept of Expected FECN packets: 14454 * o packet type 0 - expected 14455 * o match on F (bit 95), using select/match 1, and 14456 * o match on SH (bit 133), using select/match 2. 14457 * 14458 * Use index 1 to extract the 8-bit receive context from DestQP 14459 * (start at bit 64). Use that as the RSM map table index. 14460 */ 14461 rrd.offset = offset; 14462 rrd.pkt_type = 0; 14463 rrd.field1_off = 95; 14464 rrd.field2_off = 133; 14465 rrd.index1_off = 64; 14466 rrd.index1_width = 8; 14467 rrd.index2_off = 0; 14468 rrd.index2_width = 0; 14469 rrd.mask1 = 1; 14470 rrd.value1 = 1; 14471 rrd.mask2 = 1; 14472 rrd.value2 = 1; 14473 14474 /* add rule 1 */ 14475 add_rsm_rule(dd, RSM_INS_FECN, &rrd); 14476 14477 rmt->used += dd->num_user_contexts; 14478 } 14479 14480 /* Initialize RSM for VNIC */ 14481 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd) 14482 { 14483 u8 i, j; 14484 u8 ctx_id = 0; 14485 u64 reg; 14486 u32 regoff; 14487 struct rsm_rule_data rrd; 14488 14489 if (hfi1_vnic_is_rsm_full(dd, NUM_VNIC_MAP_ENTRIES)) { 14490 dd_dev_err(dd, "Vnic RSM disabled, rmt entries used = %d\n", 14491 dd->vnic.rmt_start); 14492 return; 14493 } 14494 14495 dev_dbg(&(dd)->pcidev->dev, "Vnic rsm start = %d, end %d\n", 14496 dd->vnic.rmt_start, 14497 dd->vnic.rmt_start + NUM_VNIC_MAP_ENTRIES); 14498 14499 /* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */ 14500 regoff = RCV_RSM_MAP_TABLE + (dd->vnic.rmt_start / 8) * 8; 14501 reg = read_csr(dd, regoff); 14502 for (i = 0; i < NUM_VNIC_MAP_ENTRIES; i++) { 14503 /* Update map register with vnic context */ 14504 j = (dd->vnic.rmt_start + i) % 8; 14505 reg &= ~(0xffllu << (j * 8)); 14506 reg |= (u64)dd->vnic.ctxt[ctx_id++]->ctxt << (j * 8); 14507 /* Wrap up vnic ctx index */ 14508 ctx_id %= dd->vnic.num_ctxt; 14509 /* Write back map register */ 14510 if (j == 7 || ((i + 1) == NUM_VNIC_MAP_ENTRIES)) { 14511 dev_dbg(&(dd)->pcidev->dev, 14512 "Vnic rsm map reg[%d] =0x%llx\n", 14513 regoff - RCV_RSM_MAP_TABLE, reg); 14514 14515 write_csr(dd, regoff, reg); 14516 regoff += 8; 14517 if (i < (NUM_VNIC_MAP_ENTRIES - 1)) 14518 reg = read_csr(dd, regoff); 14519 } 14520 } 14521 14522 /* Add rule for vnic */ 14523 rrd.offset = dd->vnic.rmt_start; 14524 rrd.pkt_type = 4; 14525 /* Match 16B packets */ 14526 rrd.field1_off = L2_TYPE_MATCH_OFFSET; 14527 rrd.mask1 = L2_TYPE_MASK; 14528 rrd.value1 = L2_16B_VALUE; 14529 /* Match ETH L4 packets */ 14530 rrd.field2_off = L4_TYPE_MATCH_OFFSET; 14531 rrd.mask2 = L4_16B_TYPE_MASK; 14532 rrd.value2 = L4_16B_ETH_VALUE; 14533 /* Calc context from veswid and entropy */ 14534 rrd.index1_off = L4_16B_HDR_VESWID_OFFSET; 14535 rrd.index1_width = ilog2(NUM_VNIC_MAP_ENTRIES); 14536 rrd.index2_off = L2_16B_ENTROPY_OFFSET; 14537 rrd.index2_width = ilog2(NUM_VNIC_MAP_ENTRIES); 14538 add_rsm_rule(dd, RSM_INS_VNIC, &rrd); 14539 14540 /* Enable RSM if not already enabled */ 14541 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14542 } 14543 14544 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd) 14545 { 14546 clear_rsm_rule(dd, RSM_INS_VNIC); 14547 14548 /* Disable RSM if used only by vnic */ 14549 if (dd->vnic.rmt_start == 0) 14550 clear_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14551 } 14552 14553 static void init_rxe(struct hfi1_devdata *dd) 14554 { 14555 struct rsm_map_table *rmt; 14556 u64 val; 14557 14558 /* enable all receive errors */ 14559 write_csr(dd, RCV_ERR_MASK, ~0ull); 14560 14561 rmt = alloc_rsm_map_table(dd); 14562 /* set up QOS, including the QPN map table */ 14563 init_qos(dd, rmt); 14564 init_user_fecn_handling(dd, rmt); 14565 complete_rsm_map_table(dd, rmt); 14566 /* record number of used rsm map entries for vnic */ 14567 dd->vnic.rmt_start = rmt->used; 14568 kfree(rmt); 14569 14570 /* 14571 * make sure RcvCtrl.RcvWcb <= PCIe Device Control 14572 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config 14573 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one 14574 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and 14575 * Max_PayLoad_Size set to its minimum of 128. 14576 * 14577 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0 14578 * (64 bytes). Max_Payload_Size is possibly modified upward in 14579 * tune_pcie_caps() which is called after this routine. 14580 */ 14581 14582 /* Have 16 bytes (4DW) of bypass header available in header queue */ 14583 val = read_csr(dd, RCV_BYPASS); 14584 val |= (4ull << 16); 14585 write_csr(dd, RCV_BYPASS, val); 14586 } 14587 14588 static void init_other(struct hfi1_devdata *dd) 14589 { 14590 /* enable all CCE errors */ 14591 write_csr(dd, CCE_ERR_MASK, ~0ull); 14592 /* enable *some* Misc errors */ 14593 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK); 14594 /* enable all DC errors, except LCB */ 14595 write_csr(dd, DCC_ERR_FLG_EN, ~0ull); 14596 write_csr(dd, DC_DC8051_ERR_EN, ~0ull); 14597 } 14598 14599 /* 14600 * Fill out the given AU table using the given CU. A CU is defined in terms 14601 * AUs. The table is a an encoding: given the index, how many AUs does that 14602 * represent? 14603 * 14604 * NOTE: Assumes that the register layout is the same for the 14605 * local and remote tables. 14606 */ 14607 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu, 14608 u32 csr0to3, u32 csr4to7) 14609 { 14610 write_csr(dd, csr0to3, 14611 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT | 14612 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT | 14613 2ull * cu << 14614 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT | 14615 4ull * cu << 14616 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT); 14617 write_csr(dd, csr4to7, 14618 8ull * cu << 14619 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT | 14620 16ull * cu << 14621 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT | 14622 32ull * cu << 14623 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT | 14624 64ull * cu << 14625 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT); 14626 } 14627 14628 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 14629 { 14630 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3, 14631 SEND_CM_LOCAL_AU_TABLE4_TO7); 14632 } 14633 14634 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 14635 { 14636 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3, 14637 SEND_CM_REMOTE_AU_TABLE4_TO7); 14638 } 14639 14640 static void init_txe(struct hfi1_devdata *dd) 14641 { 14642 int i; 14643 14644 /* enable all PIO, SDMA, general, and Egress errors */ 14645 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull); 14646 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull); 14647 write_csr(dd, SEND_ERR_MASK, ~0ull); 14648 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull); 14649 14650 /* enable all per-context and per-SDMA engine errors */ 14651 for (i = 0; i < dd->chip_send_contexts; i++) 14652 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull); 14653 for (i = 0; i < dd->chip_sdma_engines; i++) 14654 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull); 14655 14656 /* set the local CU to AU mapping */ 14657 assign_local_cm_au_table(dd, dd->vcu); 14658 14659 /* 14660 * Set reasonable default for Credit Return Timer 14661 * Don't set on Simulator - causes it to choke. 14662 */ 14663 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR) 14664 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE); 14665 } 14666 14667 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, 14668 u16 jkey) 14669 { 14670 u8 hw_ctxt; 14671 u64 reg; 14672 14673 if (!rcd || !rcd->sc) 14674 return -EINVAL; 14675 14676 hw_ctxt = rcd->sc->hw_context; 14677 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */ 14678 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) << 14679 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT); 14680 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */ 14681 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY)) 14682 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK; 14683 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg); 14684 /* 14685 * Enable send-side J_KEY integrity check, unless this is A0 h/w 14686 */ 14687 if (!is_ax(dd)) { 14688 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14689 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14690 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14691 } 14692 14693 /* Enable J_KEY check on receive context. */ 14694 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK | 14695 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) << 14696 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT); 14697 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg); 14698 14699 return 0; 14700 } 14701 14702 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) 14703 { 14704 u8 hw_ctxt; 14705 u64 reg; 14706 14707 if (!rcd || !rcd->sc) 14708 return -EINVAL; 14709 14710 hw_ctxt = rcd->sc->hw_context; 14711 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0); 14712 /* 14713 * Disable send-side J_KEY integrity check, unless this is A0 h/w. 14714 * This check would not have been enabled for A0 h/w, see 14715 * set_ctxt_jkey(). 14716 */ 14717 if (!is_ax(dd)) { 14718 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14719 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14720 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14721 } 14722 /* Turn off the J_KEY on the receive side */ 14723 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0); 14724 14725 return 0; 14726 } 14727 14728 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, 14729 u16 pkey) 14730 { 14731 u8 hw_ctxt; 14732 u64 reg; 14733 14734 if (!rcd || !rcd->sc) 14735 return -EINVAL; 14736 14737 hw_ctxt = rcd->sc->hw_context; 14738 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) << 14739 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT; 14740 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg); 14741 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14742 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14743 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK; 14744 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14745 14746 return 0; 14747 } 14748 14749 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt) 14750 { 14751 u8 hw_ctxt; 14752 u64 reg; 14753 14754 if (!ctxt || !ctxt->sc) 14755 return -EINVAL; 14756 14757 hw_ctxt = ctxt->sc->hw_context; 14758 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14759 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14760 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14761 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0); 14762 14763 return 0; 14764 } 14765 14766 /* 14767 * Start doing the clean up the the chip. Our clean up happens in multiple 14768 * stages and this is just the first. 14769 */ 14770 void hfi1_start_cleanup(struct hfi1_devdata *dd) 14771 { 14772 aspm_exit(dd); 14773 free_cntrs(dd); 14774 free_rcverr(dd); 14775 clean_up_interrupts(dd); 14776 finish_chip_resources(dd); 14777 } 14778 14779 #define HFI_BASE_GUID(dev) \ 14780 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT)) 14781 14782 /* 14783 * Information can be shared between the two HFIs on the same ASIC 14784 * in the same OS. This function finds the peer device and sets 14785 * up a shared structure. 14786 */ 14787 static int init_asic_data(struct hfi1_devdata *dd) 14788 { 14789 unsigned long flags; 14790 struct hfi1_devdata *tmp, *peer = NULL; 14791 struct hfi1_asic_data *asic_data; 14792 int ret = 0; 14793 14794 /* pre-allocate the asic structure in case we are the first device */ 14795 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL); 14796 if (!asic_data) 14797 return -ENOMEM; 14798 14799 spin_lock_irqsave(&hfi1_devs_lock, flags); 14800 /* Find our peer device */ 14801 list_for_each_entry(tmp, &hfi1_dev_list, list) { 14802 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(tmp)) && 14803 dd->unit != tmp->unit) { 14804 peer = tmp; 14805 break; 14806 } 14807 } 14808 14809 if (peer) { 14810 /* use already allocated structure */ 14811 dd->asic_data = peer->asic_data; 14812 kfree(asic_data); 14813 } else { 14814 dd->asic_data = asic_data; 14815 mutex_init(&dd->asic_data->asic_resource_mutex); 14816 } 14817 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */ 14818 spin_unlock_irqrestore(&hfi1_devs_lock, flags); 14819 14820 /* first one through - set up i2c devices */ 14821 if (!peer) 14822 ret = set_up_i2c(dd, dd->asic_data); 14823 14824 return ret; 14825 } 14826 14827 /* 14828 * Set dd->boardname. Use a generic name if a name is not returned from 14829 * EFI variable space. 14830 * 14831 * Return 0 on success, -ENOMEM if space could not be allocated. 14832 */ 14833 static int obtain_boardname(struct hfi1_devdata *dd) 14834 { 14835 /* generic board description */ 14836 const char generic[] = 14837 "Intel Omni-Path Host Fabric Interface Adapter 100 Series"; 14838 unsigned long size; 14839 int ret; 14840 14841 ret = read_hfi1_efi_var(dd, "description", &size, 14842 (void **)&dd->boardname); 14843 if (ret) { 14844 dd_dev_info(dd, "Board description not found\n"); 14845 /* use generic description */ 14846 dd->boardname = kstrdup(generic, GFP_KERNEL); 14847 if (!dd->boardname) 14848 return -ENOMEM; 14849 } 14850 return 0; 14851 } 14852 14853 /* 14854 * Check the interrupt registers to make sure that they are mapped correctly. 14855 * It is intended to help user identify any mismapping by VMM when the driver 14856 * is running in a VM. This function should only be called before interrupt 14857 * is set up properly. 14858 * 14859 * Return 0 on success, -EINVAL on failure. 14860 */ 14861 static int check_int_registers(struct hfi1_devdata *dd) 14862 { 14863 u64 reg; 14864 u64 all_bits = ~(u64)0; 14865 u64 mask; 14866 14867 /* Clear CceIntMask[0] to avoid raising any interrupts */ 14868 mask = read_csr(dd, CCE_INT_MASK); 14869 write_csr(dd, CCE_INT_MASK, 0ull); 14870 reg = read_csr(dd, CCE_INT_MASK); 14871 if (reg) 14872 goto err_exit; 14873 14874 /* Clear all interrupt status bits */ 14875 write_csr(dd, CCE_INT_CLEAR, all_bits); 14876 reg = read_csr(dd, CCE_INT_STATUS); 14877 if (reg) 14878 goto err_exit; 14879 14880 /* Set all interrupt status bits */ 14881 write_csr(dd, CCE_INT_FORCE, all_bits); 14882 reg = read_csr(dd, CCE_INT_STATUS); 14883 if (reg != all_bits) 14884 goto err_exit; 14885 14886 /* Restore the interrupt mask */ 14887 write_csr(dd, CCE_INT_CLEAR, all_bits); 14888 write_csr(dd, CCE_INT_MASK, mask); 14889 14890 return 0; 14891 err_exit: 14892 write_csr(dd, CCE_INT_MASK, mask); 14893 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n"); 14894 return -EINVAL; 14895 } 14896 14897 /** 14898 * Allocate and initialize the device structure for the hfi. 14899 * @dev: the pci_dev for hfi1_ib device 14900 * @ent: pci_device_id struct for this dev 14901 * 14902 * Also allocates, initializes, and returns the devdata struct for this 14903 * device instance 14904 * 14905 * This is global, and is called directly at init to set up the 14906 * chip-specific function pointers for later use. 14907 */ 14908 struct hfi1_devdata *hfi1_init_dd(struct pci_dev *pdev, 14909 const struct pci_device_id *ent) 14910 { 14911 struct hfi1_devdata *dd; 14912 struct hfi1_pportdata *ppd; 14913 u64 reg; 14914 int i, ret; 14915 static const char * const inames[] = { /* implementation names */ 14916 "RTL silicon", 14917 "RTL VCS simulation", 14918 "RTL FPGA emulation", 14919 "Functional simulator" 14920 }; 14921 struct pci_dev *parent = pdev->bus->self; 14922 14923 dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS * 14924 sizeof(struct hfi1_pportdata)); 14925 if (IS_ERR(dd)) 14926 goto bail; 14927 ppd = dd->pport; 14928 for (i = 0; i < dd->num_pports; i++, ppd++) { 14929 int vl; 14930 /* init common fields */ 14931 hfi1_init_pportdata(pdev, ppd, dd, 0, 1); 14932 /* DC supports 4 link widths */ 14933 ppd->link_width_supported = 14934 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X | 14935 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X; 14936 ppd->link_width_downgrade_supported = 14937 ppd->link_width_supported; 14938 /* start out enabling only 4X */ 14939 ppd->link_width_enabled = OPA_LINK_WIDTH_4X; 14940 ppd->link_width_downgrade_enabled = 14941 ppd->link_width_downgrade_supported; 14942 /* link width active is 0 when link is down */ 14943 /* link width downgrade active is 0 when link is down */ 14944 14945 if (num_vls < HFI1_MIN_VLS_SUPPORTED || 14946 num_vls > HFI1_MAX_VLS_SUPPORTED) { 14947 hfi1_early_err(&pdev->dev, 14948 "Invalid num_vls %u, using %u VLs\n", 14949 num_vls, HFI1_MAX_VLS_SUPPORTED); 14950 num_vls = HFI1_MAX_VLS_SUPPORTED; 14951 } 14952 ppd->vls_supported = num_vls; 14953 ppd->vls_operational = ppd->vls_supported; 14954 /* Set the default MTU. */ 14955 for (vl = 0; vl < num_vls; vl++) 14956 dd->vld[vl].mtu = hfi1_max_mtu; 14957 dd->vld[15].mtu = MAX_MAD_PACKET; 14958 /* 14959 * Set the initial values to reasonable default, will be set 14960 * for real when link is up. 14961 */ 14962 ppd->overrun_threshold = 0x4; 14963 ppd->phy_error_threshold = 0xf; 14964 ppd->port_crc_mode_enabled = link_crc_mask; 14965 /* initialize supported LTP CRC mode */ 14966 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 14967 /* initialize enabled LTP CRC mode */ 14968 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4; 14969 /* start in offline */ 14970 ppd->host_link_state = HLS_DN_OFFLINE; 14971 init_vl_arb_caches(ppd); 14972 } 14973 14974 /* 14975 * Do remaining PCIe setup and save PCIe values in dd. 14976 * Any error printing is already done by the init code. 14977 * On return, we have the chip mapped. 14978 */ 14979 ret = hfi1_pcie_ddinit(dd, pdev); 14980 if (ret < 0) 14981 goto bail_free; 14982 14983 /* Save PCI space registers to rewrite after device reset */ 14984 ret = save_pci_variables(dd); 14985 if (ret < 0) 14986 goto bail_cleanup; 14987 14988 /* verify that reads actually work, save revision for reset check */ 14989 dd->revision = read_csr(dd, CCE_REVISION); 14990 if (dd->revision == ~(u64)0) { 14991 dd_dev_err(dd, "cannot read chip CSRs\n"); 14992 ret = -EINVAL; 14993 goto bail_cleanup; 14994 } 14995 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT) 14996 & CCE_REVISION_CHIP_REV_MAJOR_MASK; 14997 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT) 14998 & CCE_REVISION_CHIP_REV_MINOR_MASK; 14999 15000 /* 15001 * Check interrupt registers mapping if the driver has no access to 15002 * the upstream component. In this case, it is likely that the driver 15003 * is running in a VM. 15004 */ 15005 if (!parent) { 15006 ret = check_int_registers(dd); 15007 if (ret) 15008 goto bail_cleanup; 15009 } 15010 15011 /* 15012 * obtain the hardware ID - NOT related to unit, which is a 15013 * software enumeration 15014 */ 15015 reg = read_csr(dd, CCE_REVISION2); 15016 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT) 15017 & CCE_REVISION2_HFI_ID_MASK; 15018 /* the variable size will remove unwanted bits */ 15019 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT; 15020 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT; 15021 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n", 15022 dd->icode < ARRAY_SIZE(inames) ? 15023 inames[dd->icode] : "unknown", (int)dd->irev); 15024 15025 /* speeds the hardware can support */ 15026 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G; 15027 /* speeds allowed to run at */ 15028 dd->pport->link_speed_enabled = dd->pport->link_speed_supported; 15029 /* give a reasonable active value, will be set on link up */ 15030 dd->pport->link_speed_active = OPA_LINK_SPEED_25G; 15031 15032 dd->chip_rcv_contexts = read_csr(dd, RCV_CONTEXTS); 15033 dd->chip_send_contexts = read_csr(dd, SEND_CONTEXTS); 15034 dd->chip_sdma_engines = read_csr(dd, SEND_DMA_ENGINES); 15035 dd->chip_pio_mem_size = read_csr(dd, SEND_PIO_MEM_SIZE); 15036 dd->chip_sdma_mem_size = read_csr(dd, SEND_DMA_MEM_SIZE); 15037 /* fix up link widths for emulation _p */ 15038 ppd = dd->pport; 15039 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) { 15040 ppd->link_width_supported = 15041 ppd->link_width_enabled = 15042 ppd->link_width_downgrade_supported = 15043 ppd->link_width_downgrade_enabled = 15044 OPA_LINK_WIDTH_1X; 15045 } 15046 /* insure num_vls isn't larger than number of sdma engines */ 15047 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > dd->chip_sdma_engines) { 15048 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n", 15049 num_vls, dd->chip_sdma_engines); 15050 num_vls = dd->chip_sdma_engines; 15051 ppd->vls_supported = dd->chip_sdma_engines; 15052 ppd->vls_operational = ppd->vls_supported; 15053 } 15054 15055 /* 15056 * Convert the ns parameter to the 64 * cclocks used in the CSR. 15057 * Limit the max if larger than the field holds. If timeout is 15058 * non-zero, then the calculated field will be at least 1. 15059 * 15060 * Must be after icode is set up - the cclock rate depends 15061 * on knowing the hardware being used. 15062 */ 15063 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64; 15064 if (dd->rcv_intr_timeout_csr > 15065 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK) 15066 dd->rcv_intr_timeout_csr = 15067 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK; 15068 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout) 15069 dd->rcv_intr_timeout_csr = 1; 15070 15071 /* needs to be done before we look for the peer device */ 15072 read_guid(dd); 15073 15074 /* set up shared ASIC data with peer device */ 15075 ret = init_asic_data(dd); 15076 if (ret) 15077 goto bail_cleanup; 15078 15079 /* obtain chip sizes, reset chip CSRs */ 15080 ret = init_chip(dd); 15081 if (ret) 15082 goto bail_cleanup; 15083 15084 /* read in the PCIe link speed information */ 15085 ret = pcie_speeds(dd); 15086 if (ret) 15087 goto bail_cleanup; 15088 15089 /* call before get_platform_config(), after init_chip_resources() */ 15090 ret = eprom_init(dd); 15091 if (ret) 15092 goto bail_free_rcverr; 15093 15094 /* Needs to be called before hfi1_firmware_init */ 15095 get_platform_config(dd); 15096 15097 /* read in firmware */ 15098 ret = hfi1_firmware_init(dd); 15099 if (ret) 15100 goto bail_cleanup; 15101 15102 /* 15103 * In general, the PCIe Gen3 transition must occur after the 15104 * chip has been idled (so it won't initiate any PCIe transactions 15105 * e.g. an interrupt) and before the driver changes any registers 15106 * (the transition will reset the registers). 15107 * 15108 * In particular, place this call after: 15109 * - init_chip() - the chip will not initiate any PCIe transactions 15110 * - pcie_speeds() - reads the current link speed 15111 * - hfi1_firmware_init() - the needed firmware is ready to be 15112 * downloaded 15113 */ 15114 ret = do_pcie_gen3_transition(dd); 15115 if (ret) 15116 goto bail_cleanup; 15117 15118 /* start setting dd values and adjusting CSRs */ 15119 init_early_variables(dd); 15120 15121 parse_platform_config(dd); 15122 15123 ret = obtain_boardname(dd); 15124 if (ret) 15125 goto bail_cleanup; 15126 15127 snprintf(dd->boardversion, BOARD_VERS_MAX, 15128 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n", 15129 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN, 15130 (u32)dd->majrev, 15131 (u32)dd->minrev, 15132 (dd->revision >> CCE_REVISION_SW_SHIFT) 15133 & CCE_REVISION_SW_MASK); 15134 15135 ret = set_up_context_variables(dd); 15136 if (ret) 15137 goto bail_cleanup; 15138 15139 /* set initial RXE CSRs */ 15140 init_rxe(dd); 15141 /* set initial TXE CSRs */ 15142 init_txe(dd); 15143 /* set initial non-RXE, non-TXE CSRs */ 15144 init_other(dd); 15145 /* set up KDETH QP prefix in both RX and TX CSRs */ 15146 init_kdeth_qp(dd); 15147 15148 ret = hfi1_dev_affinity_init(dd); 15149 if (ret) 15150 goto bail_cleanup; 15151 15152 /* send contexts must be set up before receive contexts */ 15153 ret = init_send_contexts(dd); 15154 if (ret) 15155 goto bail_cleanup; 15156 15157 ret = hfi1_create_kctxts(dd); 15158 if (ret) 15159 goto bail_cleanup; 15160 15161 /* 15162 * Initialize aspm, to be done after gen3 transition and setting up 15163 * contexts and before enabling interrupts 15164 */ 15165 aspm_init(dd); 15166 15167 dd->rcvhdrsize = DEFAULT_RCVHDRSIZE; 15168 /* 15169 * rcd[0] is guaranteed to be valid by this point. Also, all 15170 * context are using the same value, as per the module parameter. 15171 */ 15172 dd->rhf_offset = dd->rcd[0]->rcvhdrqentsize - sizeof(u64) / sizeof(u32); 15173 15174 ret = init_pervl_scs(dd); 15175 if (ret) 15176 goto bail_cleanup; 15177 15178 /* sdma init */ 15179 for (i = 0; i < dd->num_pports; ++i) { 15180 ret = sdma_init(dd, i); 15181 if (ret) 15182 goto bail_cleanup; 15183 } 15184 15185 /* use contexts created by hfi1_create_kctxts */ 15186 ret = set_up_interrupts(dd); 15187 if (ret) 15188 goto bail_cleanup; 15189 15190 /* set up LCB access - must be after set_up_interrupts() */ 15191 init_lcb_access(dd); 15192 15193 /* 15194 * Serial number is created from the base guid: 15195 * [27:24] = base guid [38:35] 15196 * [23: 0] = base guid [23: 0] 15197 */ 15198 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n", 15199 (dd->base_guid & 0xFFFFFF) | 15200 ((dd->base_guid >> 11) & 0xF000000)); 15201 15202 dd->oui1 = dd->base_guid >> 56 & 0xFF; 15203 dd->oui2 = dd->base_guid >> 48 & 0xFF; 15204 dd->oui3 = dd->base_guid >> 40 & 0xFF; 15205 15206 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */ 15207 if (ret) 15208 goto bail_clear_intr; 15209 15210 thermal_init(dd); 15211 15212 ret = init_cntrs(dd); 15213 if (ret) 15214 goto bail_clear_intr; 15215 15216 ret = init_rcverr(dd); 15217 if (ret) 15218 goto bail_free_cntrs; 15219 15220 init_completion(&dd->user_comp); 15221 15222 /* The user refcount starts with one to inidicate an active device */ 15223 atomic_set(&dd->user_refcount, 1); 15224 15225 goto bail; 15226 15227 bail_free_rcverr: 15228 free_rcverr(dd); 15229 bail_free_cntrs: 15230 free_cntrs(dd); 15231 bail_clear_intr: 15232 clean_up_interrupts(dd); 15233 bail_cleanup: 15234 hfi1_pcie_ddcleanup(dd); 15235 bail_free: 15236 hfi1_free_devdata(dd); 15237 dd = ERR_PTR(ret); 15238 bail: 15239 return dd; 15240 } 15241 15242 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate, 15243 u32 dw_len) 15244 { 15245 u32 delta_cycles; 15246 u32 current_egress_rate = ppd->current_egress_rate; 15247 /* rates here are in units of 10^6 bits/sec */ 15248 15249 if (desired_egress_rate == -1) 15250 return 0; /* shouldn't happen */ 15251 15252 if (desired_egress_rate >= current_egress_rate) 15253 return 0; /* we can't help go faster, only slower */ 15254 15255 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) - 15256 egress_cycles(dw_len * 4, current_egress_rate); 15257 15258 return (u16)delta_cycles; 15259 } 15260 15261 /** 15262 * create_pbc - build a pbc for transmission 15263 * @flags: special case flags or-ed in built pbc 15264 * @srate: static rate 15265 * @vl: vl 15266 * @dwlen: dword length (header words + data words + pbc words) 15267 * 15268 * Create a PBC with the given flags, rate, VL, and length. 15269 * 15270 * NOTE: The PBC created will not insert any HCRC - all callers but one are 15271 * for verbs, which does not use this PSM feature. The lone other caller 15272 * is for the diagnostic interface which calls this if the user does not 15273 * supply their own PBC. 15274 */ 15275 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, 15276 u32 dw_len) 15277 { 15278 u64 pbc, delay = 0; 15279 15280 if (unlikely(srate_mbs)) 15281 delay = delay_cycles(ppd, srate_mbs, dw_len); 15282 15283 pbc = flags 15284 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT) 15285 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT) 15286 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT 15287 | (dw_len & PBC_LENGTH_DWS_MASK) 15288 << PBC_LENGTH_DWS_SHIFT; 15289 15290 return pbc; 15291 } 15292 15293 #define SBUS_THERMAL 0x4f 15294 #define SBUS_THERM_MONITOR_MODE 0x1 15295 15296 #define THERM_FAILURE(dev, ret, reason) \ 15297 dd_dev_err((dd), \ 15298 "Thermal sensor initialization failed: %s (%d)\n", \ 15299 (reason), (ret)) 15300 15301 /* 15302 * Initialize the thermal sensor. 15303 * 15304 * After initialization, enable polling of thermal sensor through 15305 * SBus interface. In order for this to work, the SBus Master 15306 * firmware has to be loaded due to the fact that the HW polling 15307 * logic uses SBus interrupts, which are not supported with 15308 * default firmware. Otherwise, no data will be returned through 15309 * the ASIC_STS_THERM CSR. 15310 */ 15311 static int thermal_init(struct hfi1_devdata *dd) 15312 { 15313 int ret = 0; 15314 15315 if (dd->icode != ICODE_RTL_SILICON || 15316 check_chip_resource(dd, CR_THERM_INIT, NULL)) 15317 return ret; 15318 15319 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT); 15320 if (ret) { 15321 THERM_FAILURE(dd, ret, "Acquire SBus"); 15322 return ret; 15323 } 15324 15325 dd_dev_info(dd, "Initializing thermal sensor\n"); 15326 /* Disable polling of thermal readings */ 15327 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0); 15328 msleep(100); 15329 /* Thermal Sensor Initialization */ 15330 /* Step 1: Reset the Thermal SBus Receiver */ 15331 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15332 RESET_SBUS_RECEIVER, 0); 15333 if (ret) { 15334 THERM_FAILURE(dd, ret, "Bus Reset"); 15335 goto done; 15336 } 15337 /* Step 2: Set Reset bit in Thermal block */ 15338 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15339 WRITE_SBUS_RECEIVER, 0x1); 15340 if (ret) { 15341 THERM_FAILURE(dd, ret, "Therm Block Reset"); 15342 goto done; 15343 } 15344 /* Step 3: Write clock divider value (100MHz -> 2MHz) */ 15345 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1, 15346 WRITE_SBUS_RECEIVER, 0x32); 15347 if (ret) { 15348 THERM_FAILURE(dd, ret, "Write Clock Div"); 15349 goto done; 15350 } 15351 /* Step 4: Select temperature mode */ 15352 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3, 15353 WRITE_SBUS_RECEIVER, 15354 SBUS_THERM_MONITOR_MODE); 15355 if (ret) { 15356 THERM_FAILURE(dd, ret, "Write Mode Sel"); 15357 goto done; 15358 } 15359 /* Step 5: De-assert block reset and start conversion */ 15360 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15361 WRITE_SBUS_RECEIVER, 0x2); 15362 if (ret) { 15363 THERM_FAILURE(dd, ret, "Write Reset Deassert"); 15364 goto done; 15365 } 15366 /* Step 5.1: Wait for first conversion (21.5ms per spec) */ 15367 msleep(22); 15368 15369 /* Enable polling of thermal readings */ 15370 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1); 15371 15372 /* Set initialized flag */ 15373 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0); 15374 if (ret) 15375 THERM_FAILURE(dd, ret, "Unable to set thermal init flag"); 15376 15377 done: 15378 release_chip_resource(dd, CR_SBUS); 15379 return ret; 15380 } 15381 15382 static void handle_temp_err(struct hfi1_devdata *dd) 15383 { 15384 struct hfi1_pportdata *ppd = &dd->pport[0]; 15385 /* 15386 * Thermal Critical Interrupt 15387 * Put the device into forced freeze mode, take link down to 15388 * offline, and put DC into reset. 15389 */ 15390 dd_dev_emerg(dd, 15391 "Critical temperature reached! Forcing device into freeze mode!\n"); 15392 dd->flags |= HFI1_FORCED_FREEZE; 15393 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT); 15394 /* 15395 * Shut DC down as much and as quickly as possible. 15396 * 15397 * Step 1: Take the link down to OFFLINE. This will cause the 15398 * 8051 to put the Serdes in reset. However, we don't want to 15399 * go through the entire link state machine since we want to 15400 * shutdown ASAP. Furthermore, this is not a graceful shutdown 15401 * but rather an attempt to save the chip. 15402 * Code below is almost the same as quiet_serdes() but avoids 15403 * all the extra work and the sleeps. 15404 */ 15405 ppd->driver_link_ready = 0; 15406 ppd->link_enabled = 0; 15407 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) | 15408 PLS_OFFLINE); 15409 /* 15410 * Step 2: Shutdown LCB and 8051 15411 * After shutdown, do not restore DC_CFG_RESET value. 15412 */ 15413 dc_shutdown(dd); 15414 } 15415