1 /*
2  * Copyright (c) 2011 - 2017 Intel Corporation.  All rights reserved.
3  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
4  * All rights reserved.
5  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 /*
36  * This file contains all of the code that is specific to the
37  * QLogic_IB 7220 chip (except that specific to the SerDes)
38  */
39 
40 #include <linux/interrupt.h>
41 #include <linux/pci.h>
42 #include <linux/delay.h>
43 #include <linux/module.h>
44 #include <linux/io.h>
45 #include <rdma/ib_verbs.h>
46 
47 #include "qib.h"
48 #include "qib_7220.h"
49 
50 static void qib_setup_7220_setextled(struct qib_pportdata *, u32);
51 static void qib_7220_handle_hwerrors(struct qib_devdata *, char *, size_t);
52 static void sendctrl_7220_mod(struct qib_pportdata *ppd, u32 op);
53 static u32 qib_7220_iblink_state(u64);
54 static u8 qib_7220_phys_portstate(u64);
55 static void qib_sdma_update_7220_tail(struct qib_pportdata *, u16);
56 static void qib_set_ib_7220_lstate(struct qib_pportdata *, u16, u16);
57 
58 /*
59  * This file contains almost all the chip-specific register information and
60  * access functions for the QLogic QLogic_IB 7220 PCI-Express chip, with the
61  * exception of SerDes support, which in qib_sd7220.c.
62  */
63 
64 /* Below uses machine-generated qib_chipnum_regs.h file */
65 #define KREG_IDX(regname) (QIB_7220_##regname##_OFFS / sizeof(u64))
66 
67 /* Use defines to tie machine-generated names to lower-case names */
68 #define kr_control KREG_IDX(Control)
69 #define kr_counterregbase KREG_IDX(CntrRegBase)
70 #define kr_errclear KREG_IDX(ErrClear)
71 #define kr_errmask KREG_IDX(ErrMask)
72 #define kr_errstatus KREG_IDX(ErrStatus)
73 #define kr_extctrl KREG_IDX(EXTCtrl)
74 #define kr_extstatus KREG_IDX(EXTStatus)
75 #define kr_gpio_clear KREG_IDX(GPIOClear)
76 #define kr_gpio_mask KREG_IDX(GPIOMask)
77 #define kr_gpio_out KREG_IDX(GPIOOut)
78 #define kr_gpio_status KREG_IDX(GPIOStatus)
79 #define kr_hrtbt_guid KREG_IDX(HRTBT_GUID)
80 #define kr_hwdiagctrl KREG_IDX(HwDiagCtrl)
81 #define kr_hwerrclear KREG_IDX(HwErrClear)
82 #define kr_hwerrmask KREG_IDX(HwErrMask)
83 #define kr_hwerrstatus KREG_IDX(HwErrStatus)
84 #define kr_ibcctrl KREG_IDX(IBCCtrl)
85 #define kr_ibcddrctrl KREG_IDX(IBCDDRCtrl)
86 #define kr_ibcddrstatus KREG_IDX(IBCDDRStatus)
87 #define kr_ibcstatus KREG_IDX(IBCStatus)
88 #define kr_ibserdesctrl KREG_IDX(IBSerDesCtrl)
89 #define kr_intclear KREG_IDX(IntClear)
90 #define kr_intmask KREG_IDX(IntMask)
91 #define kr_intstatus KREG_IDX(IntStatus)
92 #define kr_ncmodectrl KREG_IDX(IBNCModeCtrl)
93 #define kr_palign KREG_IDX(PageAlign)
94 #define kr_partitionkey KREG_IDX(RcvPartitionKey)
95 #define kr_portcnt KREG_IDX(PortCnt)
96 #define kr_rcvbthqp KREG_IDX(RcvBTHQP)
97 #define kr_rcvctrl KREG_IDX(RcvCtrl)
98 #define kr_rcvegrbase KREG_IDX(RcvEgrBase)
99 #define kr_rcvegrcnt KREG_IDX(RcvEgrCnt)
100 #define kr_rcvhdrcnt KREG_IDX(RcvHdrCnt)
101 #define kr_rcvhdrentsize KREG_IDX(RcvHdrEntSize)
102 #define kr_rcvhdrsize KREG_IDX(RcvHdrSize)
103 #define kr_rcvpktledcnt KREG_IDX(RcvPktLEDCnt)
104 #define kr_rcvtidbase KREG_IDX(RcvTIDBase)
105 #define kr_rcvtidcnt KREG_IDX(RcvTIDCnt)
106 #define kr_revision KREG_IDX(Revision)
107 #define kr_scratch KREG_IDX(Scratch)
108 #define kr_sendbuffererror KREG_IDX(SendBufErr0)
109 #define kr_sendctrl KREG_IDX(SendCtrl)
110 #define kr_senddmabase KREG_IDX(SendDmaBase)
111 #define kr_senddmabufmask0 KREG_IDX(SendDmaBufMask0)
112 #define kr_senddmabufmask1 (KREG_IDX(SendDmaBufMask0) + 1)
113 #define kr_senddmabufmask2 (KREG_IDX(SendDmaBufMask0) + 2)
114 #define kr_senddmahead KREG_IDX(SendDmaHead)
115 #define kr_senddmaheadaddr KREG_IDX(SendDmaHeadAddr)
116 #define kr_senddmalengen KREG_IDX(SendDmaLenGen)
117 #define kr_senddmastatus KREG_IDX(SendDmaStatus)
118 #define kr_senddmatail KREG_IDX(SendDmaTail)
119 #define kr_sendpioavailaddr KREG_IDX(SendBufAvailAddr)
120 #define kr_sendpiobufbase KREG_IDX(SendBufBase)
121 #define kr_sendpiobufcnt KREG_IDX(SendBufCnt)
122 #define kr_sendpiosize KREG_IDX(SendBufSize)
123 #define kr_sendregbase KREG_IDX(SendRegBase)
124 #define kr_userregbase KREG_IDX(UserRegBase)
125 #define kr_xgxs_cfg KREG_IDX(XGXSCfg)
126 
127 /* These must only be written via qib_write_kreg_ctxt() */
128 #define kr_rcvhdraddr KREG_IDX(RcvHdrAddr0)
129 #define kr_rcvhdrtailaddr KREG_IDX(RcvHdrTailAddr0)
130 
131 
132 #define CREG_IDX(regname) ((QIB_7220_##regname##_OFFS - \
133 			QIB_7220_LBIntCnt_OFFS) / sizeof(u64))
134 
135 #define cr_badformat CREG_IDX(RxVersionErrCnt)
136 #define cr_erricrc CREG_IDX(RxICRCErrCnt)
137 #define cr_errlink CREG_IDX(RxLinkMalformCnt)
138 #define cr_errlpcrc CREG_IDX(RxLPCRCErrCnt)
139 #define cr_errpkey CREG_IDX(RxPKeyMismatchCnt)
140 #define cr_rcvflowctrl_err CREG_IDX(RxFlowCtrlViolCnt)
141 #define cr_err_rlen CREG_IDX(RxLenErrCnt)
142 #define cr_errslen CREG_IDX(TxLenErrCnt)
143 #define cr_errtidfull CREG_IDX(RxTIDFullErrCnt)
144 #define cr_errtidvalid CREG_IDX(RxTIDValidErrCnt)
145 #define cr_errvcrc CREG_IDX(RxVCRCErrCnt)
146 #define cr_ibstatuschange CREG_IDX(IBStatusChangeCnt)
147 #define cr_lbint CREG_IDX(LBIntCnt)
148 #define cr_invalidrlen CREG_IDX(RxMaxMinLenErrCnt)
149 #define cr_invalidslen CREG_IDX(TxMaxMinLenErrCnt)
150 #define cr_lbflowstall CREG_IDX(LBFlowStallCnt)
151 #define cr_pktrcv CREG_IDX(RxDataPktCnt)
152 #define cr_pktrcvflowctrl CREG_IDX(RxFlowPktCnt)
153 #define cr_pktsend CREG_IDX(TxDataPktCnt)
154 #define cr_pktsendflow CREG_IDX(TxFlowPktCnt)
155 #define cr_portovfl CREG_IDX(RxP0HdrEgrOvflCnt)
156 #define cr_rcvebp CREG_IDX(RxEBPCnt)
157 #define cr_rcvovfl CREG_IDX(RxBufOvflCnt)
158 #define cr_senddropped CREG_IDX(TxDroppedPktCnt)
159 #define cr_sendstall CREG_IDX(TxFlowStallCnt)
160 #define cr_sendunderrun CREG_IDX(TxUnderrunCnt)
161 #define cr_wordrcv CREG_IDX(RxDwordCnt)
162 #define cr_wordsend CREG_IDX(TxDwordCnt)
163 #define cr_txunsupvl CREG_IDX(TxUnsupVLErrCnt)
164 #define cr_rxdroppkt CREG_IDX(RxDroppedPktCnt)
165 #define cr_iblinkerrrecov CREG_IDX(IBLinkErrRecoveryCnt)
166 #define cr_iblinkdown CREG_IDX(IBLinkDownedCnt)
167 #define cr_ibsymbolerr CREG_IDX(IBSymbolErrCnt)
168 #define cr_vl15droppedpkt CREG_IDX(RxVL15DroppedPktCnt)
169 #define cr_rxotherlocalphyerr CREG_IDX(RxOtherLocalPhyErrCnt)
170 #define cr_excessbufferovfl CREG_IDX(ExcessBufferOvflCnt)
171 #define cr_locallinkintegrityerr CREG_IDX(LocalLinkIntegrityErrCnt)
172 #define cr_rxvlerr CREG_IDX(RxVlErrCnt)
173 #define cr_rxdlidfltr CREG_IDX(RxDlidFltrCnt)
174 #define cr_psstat CREG_IDX(PSStat)
175 #define cr_psstart CREG_IDX(PSStart)
176 #define cr_psinterval CREG_IDX(PSInterval)
177 #define cr_psrcvdatacount CREG_IDX(PSRcvDataCount)
178 #define cr_psrcvpktscount CREG_IDX(PSRcvPktsCount)
179 #define cr_psxmitdatacount CREG_IDX(PSXmitDataCount)
180 #define cr_psxmitpktscount CREG_IDX(PSXmitPktsCount)
181 #define cr_psxmitwaitcount CREG_IDX(PSXmitWaitCount)
182 #define cr_txsdmadesc CREG_IDX(TxSDmaDescCnt)
183 #define cr_pcieretrydiag CREG_IDX(PcieRetryBufDiagQwordCnt)
184 
185 #define SYM_RMASK(regname, fldname) ((u64)              \
186 	QIB_7220_##regname##_##fldname##_RMASK)
187 #define SYM_MASK(regname, fldname) ((u64)               \
188 	QIB_7220_##regname##_##fldname##_RMASK <<       \
189 	 QIB_7220_##regname##_##fldname##_LSB)
190 #define SYM_LSB(regname, fldname) (QIB_7220_##regname##_##fldname##_LSB)
191 #define SYM_FIELD(value, regname, fldname) ((u64) \
192 	(((value) >> SYM_LSB(regname, fldname)) & \
193 	 SYM_RMASK(regname, fldname)))
194 #define ERR_MASK(fldname) SYM_MASK(ErrMask, fldname##Mask)
195 #define HWE_MASK(fldname) SYM_MASK(HwErrMask, fldname##Mask)
196 
197 /* ibcctrl bits */
198 #define QLOGIC_IB_IBCC_LINKINITCMD_DISABLE 1
199 /* cycle through TS1/TS2 till OK */
200 #define QLOGIC_IB_IBCC_LINKINITCMD_POLL 2
201 /* wait for TS1, then go on */
202 #define QLOGIC_IB_IBCC_LINKINITCMD_SLEEP 3
203 #define QLOGIC_IB_IBCC_LINKINITCMD_SHIFT 16
204 
205 #define QLOGIC_IB_IBCC_LINKCMD_DOWN 1           /* move to 0x11 */
206 #define QLOGIC_IB_IBCC_LINKCMD_ARMED 2          /* move to 0x21 */
207 #define QLOGIC_IB_IBCC_LINKCMD_ACTIVE 3 /* move to 0x31 */
208 
209 #define BLOB_7220_IBCHG 0x81
210 
211 /*
212  * We could have a single register get/put routine, that takes a group type,
213  * but this is somewhat clearer and cleaner.  It also gives us some error
214  * checking.  64 bit register reads should always work, but are inefficient
215  * on opteron (the northbridge always generates 2 separate HT 32 bit reads),
216  * so we use kreg32 wherever possible.  User register and counter register
217  * reads are always 32 bit reads, so only one form of those routines.
218  */
219 
220 /**
221  * qib_read_ureg32 - read 32-bit virtualized per-context register
222  * @dd: device
223  * @regno: register number
224  * @ctxt: context number
225  *
226  * Return the contents of a register that is virtualized to be per context.
227  * Returns -1 on errors (not distinguishable from valid contents at
228  * runtime; we may add a separate error variable at some point).
229  */
230 static inline u32 qib_read_ureg32(const struct qib_devdata *dd,
231 				  enum qib_ureg regno, int ctxt)
232 {
233 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
234 		return 0;
235 
236 	if (dd->userbase)
237 		return readl(regno + (u64 __iomem *)
238 			     ((char __iomem *)dd->userbase +
239 			      dd->ureg_align * ctxt));
240 	else
241 		return readl(regno + (u64 __iomem *)
242 			     (dd->uregbase +
243 			      (char __iomem *)dd->kregbase +
244 			      dd->ureg_align * ctxt));
245 }
246 
247 /**
248  * qib_write_ureg - write 32-bit virtualized per-context register
249  * @dd: device
250  * @regno: register number
251  * @value: value
252  * @ctxt: context
253  *
254  * Write the contents of a register that is virtualized to be per context.
255  */
256 static inline void qib_write_ureg(const struct qib_devdata *dd,
257 				  enum qib_ureg regno, u64 value, int ctxt)
258 {
259 	u64 __iomem *ubase;
260 
261 	if (dd->userbase)
262 		ubase = (u64 __iomem *)
263 			((char __iomem *) dd->userbase +
264 			 dd->ureg_align * ctxt);
265 	else
266 		ubase = (u64 __iomem *)
267 			(dd->uregbase +
268 			 (char __iomem *) dd->kregbase +
269 			 dd->ureg_align * ctxt);
270 
271 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
272 		writeq(value, &ubase[regno]);
273 }
274 
275 /**
276  * qib_write_kreg_ctxt - write a device's per-ctxt 64-bit kernel register
277  * @dd: the qlogic_ib device
278  * @regno: the register number to write
279  * @ctxt: the context containing the register
280  * @value: the value to write
281  */
282 static inline void qib_write_kreg_ctxt(const struct qib_devdata *dd,
283 				       const u16 regno, unsigned ctxt,
284 				       u64 value)
285 {
286 	qib_write_kreg(dd, regno + ctxt, value);
287 }
288 
289 static inline void write_7220_creg(const struct qib_devdata *dd,
290 				   u16 regno, u64 value)
291 {
292 	if (dd->cspec->cregbase && (dd->flags & QIB_PRESENT))
293 		writeq(value, &dd->cspec->cregbase[regno]);
294 }
295 
296 static inline u64 read_7220_creg(const struct qib_devdata *dd, u16 regno)
297 {
298 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
299 		return 0;
300 	return readq(&dd->cspec->cregbase[regno]);
301 }
302 
303 static inline u32 read_7220_creg32(const struct qib_devdata *dd, u16 regno)
304 {
305 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
306 		return 0;
307 	return readl(&dd->cspec->cregbase[regno]);
308 }
309 
310 /* kr_revision bits */
311 #define QLOGIC_IB_R_EMULATORREV_MASK ((1ULL << 22) - 1)
312 #define QLOGIC_IB_R_EMULATORREV_SHIFT 40
313 
314 /* kr_control bits */
315 #define QLOGIC_IB_C_RESET (1U << 7)
316 
317 /* kr_intstatus, kr_intclear, kr_intmask bits */
318 #define QLOGIC_IB_I_RCVURG_MASK ((1ULL << 17) - 1)
319 #define QLOGIC_IB_I_RCVURG_SHIFT 32
320 #define QLOGIC_IB_I_RCVAVAIL_MASK ((1ULL << 17) - 1)
321 #define QLOGIC_IB_I_RCVAVAIL_SHIFT 0
322 #define QLOGIC_IB_I_SERDESTRIMDONE (1ULL << 27)
323 
324 #define QLOGIC_IB_C_FREEZEMODE 0x00000002
325 #define QLOGIC_IB_C_LINKENABLE 0x00000004
326 
327 #define QLOGIC_IB_I_SDMAINT             0x8000000000000000ULL
328 #define QLOGIC_IB_I_SDMADISABLED        0x4000000000000000ULL
329 #define QLOGIC_IB_I_ERROR               0x0000000080000000ULL
330 #define QLOGIC_IB_I_SPIOSENT            0x0000000040000000ULL
331 #define QLOGIC_IB_I_SPIOBUFAVAIL        0x0000000020000000ULL
332 #define QLOGIC_IB_I_GPIO                0x0000000010000000ULL
333 
334 /* variables for sanity checking interrupt and errors */
335 #define QLOGIC_IB_I_BITSEXTANT \
336 		(QLOGIC_IB_I_SDMAINT | QLOGIC_IB_I_SDMADISABLED | \
337 		(QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT) | \
338 		(QLOGIC_IB_I_RCVAVAIL_MASK << \
339 		 QLOGIC_IB_I_RCVAVAIL_SHIFT) | \
340 		QLOGIC_IB_I_ERROR | QLOGIC_IB_I_SPIOSENT | \
341 		QLOGIC_IB_I_SPIOBUFAVAIL | QLOGIC_IB_I_GPIO | \
342 		QLOGIC_IB_I_SERDESTRIMDONE)
343 
344 #define IB_HWE_BITSEXTANT \
345 	       (HWE_MASK(RXEMemParityErr) | \
346 		HWE_MASK(TXEMemParityErr) | \
347 		(QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<	 \
348 		 QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) | \
349 		QLOGIC_IB_HWE_PCIE1PLLFAILED | \
350 		QLOGIC_IB_HWE_PCIE0PLLFAILED | \
351 		QLOGIC_IB_HWE_PCIEPOISONEDTLP | \
352 		QLOGIC_IB_HWE_PCIECPLTIMEOUT | \
353 		QLOGIC_IB_HWE_PCIEBUSPARITYXTLH | \
354 		QLOGIC_IB_HWE_PCIEBUSPARITYXADM | \
355 		QLOGIC_IB_HWE_PCIEBUSPARITYRADM | \
356 		HWE_MASK(PowerOnBISTFailed) |	  \
357 		QLOGIC_IB_HWE_COREPLL_FBSLIP | \
358 		QLOGIC_IB_HWE_COREPLL_RFSLIP | \
359 		QLOGIC_IB_HWE_SERDESPLLFAILED | \
360 		HWE_MASK(IBCBusToSPCParityErr) | \
361 		HWE_MASK(IBCBusFromSPCParityErr) | \
362 		QLOGIC_IB_HWE_PCIECPLDATAQUEUEERR | \
363 		QLOGIC_IB_HWE_PCIECPLHDRQUEUEERR | \
364 		QLOGIC_IB_HWE_SDMAMEMREADERR | \
365 		QLOGIC_IB_HWE_CLK_UC_PLLNOTLOCKED | \
366 		QLOGIC_IB_HWE_PCIESERDESQ0PCLKNOTDETECT | \
367 		QLOGIC_IB_HWE_PCIESERDESQ1PCLKNOTDETECT | \
368 		QLOGIC_IB_HWE_PCIESERDESQ2PCLKNOTDETECT | \
369 		QLOGIC_IB_HWE_PCIESERDESQ3PCLKNOTDETECT | \
370 		QLOGIC_IB_HWE_DDSRXEQMEMORYPARITYERR | \
371 		QLOGIC_IB_HWE_IB_UC_MEMORYPARITYERR | \
372 		QLOGIC_IB_HWE_PCIE_UC_OCT0MEMORYPARITYERR | \
373 		QLOGIC_IB_HWE_PCIE_UC_OCT1MEMORYPARITYERR)
374 
375 #define IB_E_BITSEXTANT							\
376 	(ERR_MASK(RcvFormatErr) | ERR_MASK(RcvVCRCErr) |		\
377 	 ERR_MASK(RcvICRCErr) | ERR_MASK(RcvMinPktLenErr) |		\
378 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvLongPktLenErr) |	\
379 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvUnexpectedCharErr) | \
380 	 ERR_MASK(RcvUnsupportedVLErr) | ERR_MASK(RcvEBPErr) |		\
381 	 ERR_MASK(RcvIBFlowErr) | ERR_MASK(RcvBadVersionErr) |		\
382 	 ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) |		\
383 	 ERR_MASK(RcvBadTidErr) | ERR_MASK(RcvHdrLenErr) |		\
384 	 ERR_MASK(RcvHdrErr) | ERR_MASK(RcvIBLostLinkErr) |		\
385 	 ERR_MASK(SendSpecialTriggerErr) |				\
386 	 ERR_MASK(SDmaDisabledErr) | ERR_MASK(SendMinPktLenErr) |	\
387 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnderRunErr) |	\
388 	 ERR_MASK(SendPktLenErr) | ERR_MASK(SendDroppedSmpPktErr) |	\
389 	 ERR_MASK(SendDroppedDataPktErr) |				\
390 	 ERR_MASK(SendPioArmLaunchErr) |				\
391 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
392 	 ERR_MASK(SendUnsupportedVLErr) | ERR_MASK(SendBufMisuseErr) |	\
393 	 ERR_MASK(SDmaGenMismatchErr) | ERR_MASK(SDmaOutOfBoundErr) |	\
394 	 ERR_MASK(SDmaTailOutOfBoundErr) | ERR_MASK(SDmaBaseErr) |	\
395 	 ERR_MASK(SDma1stDescErr) | ERR_MASK(SDmaRpyTagErr) |		\
396 	 ERR_MASK(SDmaDwEnErr) | ERR_MASK(SDmaMissingDwErr) |		\
397 	 ERR_MASK(SDmaUnexpDataErr) |					\
398 	 ERR_MASK(IBStatusChanged) | ERR_MASK(InvalidAddrErr) |		\
399 	 ERR_MASK(ResetNegated) | ERR_MASK(HardwareErr) |		\
400 	 ERR_MASK(SDmaDescAddrMisalignErr) |				\
401 	 ERR_MASK(InvalidEEPCmd))
402 
403 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
404 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK  0x00000000000000ffULL
405 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT 0
406 #define QLOGIC_IB_HWE_PCIEPOISONEDTLP      0x0000000010000000ULL
407 #define QLOGIC_IB_HWE_PCIECPLTIMEOUT       0x0000000020000000ULL
408 #define QLOGIC_IB_HWE_PCIEBUSPARITYXTLH    0x0000000040000000ULL
409 #define QLOGIC_IB_HWE_PCIEBUSPARITYXADM    0x0000000080000000ULL
410 #define QLOGIC_IB_HWE_PCIEBUSPARITYRADM    0x0000000100000000ULL
411 #define QLOGIC_IB_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
412 #define QLOGIC_IB_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
413 #define QLOGIC_IB_HWE_PCIE1PLLFAILED       0x0400000000000000ULL
414 #define QLOGIC_IB_HWE_PCIE0PLLFAILED       0x0800000000000000ULL
415 #define QLOGIC_IB_HWE_SERDESPLLFAILED      0x1000000000000000ULL
416 /* specific to this chip */
417 #define QLOGIC_IB_HWE_PCIECPLDATAQUEUEERR         0x0000000000000040ULL
418 #define QLOGIC_IB_HWE_PCIECPLHDRQUEUEERR          0x0000000000000080ULL
419 #define QLOGIC_IB_HWE_SDMAMEMREADERR              0x0000000010000000ULL
420 #define QLOGIC_IB_HWE_CLK_UC_PLLNOTLOCKED          0x2000000000000000ULL
421 #define QLOGIC_IB_HWE_PCIESERDESQ0PCLKNOTDETECT   0x0100000000000000ULL
422 #define QLOGIC_IB_HWE_PCIESERDESQ1PCLKNOTDETECT   0x0200000000000000ULL
423 #define QLOGIC_IB_HWE_PCIESERDESQ2PCLKNOTDETECT   0x0400000000000000ULL
424 #define QLOGIC_IB_HWE_PCIESERDESQ3PCLKNOTDETECT   0x0800000000000000ULL
425 #define QLOGIC_IB_HWE_DDSRXEQMEMORYPARITYERR       0x0000008000000000ULL
426 #define QLOGIC_IB_HWE_IB_UC_MEMORYPARITYERR        0x0000004000000000ULL
427 #define QLOGIC_IB_HWE_PCIE_UC_OCT0MEMORYPARITYERR 0x0000001000000000ULL
428 #define QLOGIC_IB_HWE_PCIE_UC_OCT1MEMORYPARITYERR 0x0000002000000000ULL
429 
430 #define IBA7220_IBCC_LINKCMD_SHIFT 19
431 
432 /* kr_ibcddrctrl bits */
433 #define IBA7220_IBC_DLIDLMC_MASK        0xFFFFFFFFUL
434 #define IBA7220_IBC_DLIDLMC_SHIFT       32
435 
436 #define IBA7220_IBC_HRTBT_MASK  (SYM_RMASK(IBCDDRCtrl, HRTBT_AUTO) | \
437 				 SYM_RMASK(IBCDDRCtrl, HRTBT_ENB))
438 #define IBA7220_IBC_HRTBT_SHIFT SYM_LSB(IBCDDRCtrl, HRTBT_ENB)
439 
440 #define IBA7220_IBC_LANE_REV_SUPPORTED (1<<8)
441 #define IBA7220_IBC_LREV_MASK   1
442 #define IBA7220_IBC_LREV_SHIFT  8
443 #define IBA7220_IBC_RXPOL_MASK  1
444 #define IBA7220_IBC_RXPOL_SHIFT 7
445 #define IBA7220_IBC_WIDTH_SHIFT 5
446 #define IBA7220_IBC_WIDTH_MASK  0x3
447 #define IBA7220_IBC_WIDTH_1X_ONLY       (0 << IBA7220_IBC_WIDTH_SHIFT)
448 #define IBA7220_IBC_WIDTH_4X_ONLY       (1 << IBA7220_IBC_WIDTH_SHIFT)
449 #define IBA7220_IBC_WIDTH_AUTONEG       (2 << IBA7220_IBC_WIDTH_SHIFT)
450 #define IBA7220_IBC_SPEED_AUTONEG       (1 << 1)
451 #define IBA7220_IBC_SPEED_SDR           (1 << 2)
452 #define IBA7220_IBC_SPEED_DDR           (1 << 3)
453 #define IBA7220_IBC_SPEED_AUTONEG_MASK  (0x7 << 1)
454 #define IBA7220_IBC_IBTA_1_2_MASK       (1)
455 
456 /* kr_ibcddrstatus */
457 /* link latency shift is 0, don't bother defining */
458 #define IBA7220_DDRSTAT_LINKLAT_MASK    0x3ffffff
459 
460 /* kr_extstatus bits */
461 #define QLOGIC_IB_EXTS_FREQSEL 0x2
462 #define QLOGIC_IB_EXTS_SERDESSEL 0x4
463 #define QLOGIC_IB_EXTS_MEMBIST_ENDTEST     0x0000000000004000
464 #define QLOGIC_IB_EXTS_MEMBIST_DISABLED    0x0000000000008000
465 
466 /* kr_xgxsconfig bits */
467 #define QLOGIC_IB_XGXS_RESET          0x5ULL
468 #define QLOGIC_IB_XGXS_FC_SAFE        (1ULL << 63)
469 
470 /* kr_rcvpktledcnt */
471 #define IBA7220_LEDBLINK_ON_SHIFT 32 /* 4ns period on after packet */
472 #define IBA7220_LEDBLINK_OFF_SHIFT 0 /* 4ns period off before next on */
473 
474 #define _QIB_GPIO_SDA_NUM 1
475 #define _QIB_GPIO_SCL_NUM 0
476 #define QIB_TWSI_EEPROM_DEV 0xA2 /* All Production 7220 cards. */
477 #define QIB_TWSI_TEMP_DEV 0x98
478 
479 /* HW counter clock is at 4nsec */
480 #define QIB_7220_PSXMITWAIT_CHECK_RATE 4000
481 
482 #define IBA7220_R_INTRAVAIL_SHIFT 17
483 #define IBA7220_R_PKEY_DIS_SHIFT 34
484 #define IBA7220_R_TAILUPD_SHIFT 35
485 #define IBA7220_R_CTXTCFG_SHIFT 36
486 
487 #define IBA7220_HDRHEAD_PKTINT_SHIFT 32 /* interrupt cnt in upper 32 bits */
488 
489 /*
490  * the size bits give us 2^N, in KB units.  0 marks as invalid,
491  * and 7 is reserved.  We currently use only 2KB and 4KB
492  */
493 #define IBA7220_TID_SZ_SHIFT 37 /* shift to 3bit size selector */
494 #define IBA7220_TID_SZ_2K (1UL << IBA7220_TID_SZ_SHIFT) /* 2KB */
495 #define IBA7220_TID_SZ_4K (2UL << IBA7220_TID_SZ_SHIFT) /* 4KB */
496 #define IBA7220_TID_PA_SHIFT 11U /* TID addr in chip stored w/o low bits */
497 #define PBC_7220_VL15_SEND (1ULL << 63) /* pbc; VL15, no credit check */
498 #define PBC_7220_VL15_SEND_CTRL (1ULL << 31) /* control version of same */
499 
500 #define AUTONEG_TRIES 5 /* sequential retries to negotiate DDR */
501 
502 /* packet rate matching delay multiplier */
503 static u8 rate_to_delay[2][2] = {
504 	/* 1x, 4x */
505 	{   8, 2 }, /* SDR */
506 	{   4, 1 }  /* DDR */
507 };
508 
509 static u8 ib_rate_to_delay[IB_RATE_120_GBPS + 1] = {
510 	[IB_RATE_2_5_GBPS] = 8,
511 	[IB_RATE_5_GBPS] = 4,
512 	[IB_RATE_10_GBPS] = 2,
513 	[IB_RATE_20_GBPS] = 1
514 };
515 
516 #define IBA7220_LINKSPEED_SHIFT SYM_LSB(IBCStatus, LinkSpeedActive)
517 #define IBA7220_LINKWIDTH_SHIFT SYM_LSB(IBCStatus, LinkWidthActive)
518 
519 /* link training states, from IBC */
520 #define IB_7220_LT_STATE_DISABLED        0x00
521 #define IB_7220_LT_STATE_LINKUP          0x01
522 #define IB_7220_LT_STATE_POLLACTIVE      0x02
523 #define IB_7220_LT_STATE_POLLQUIET       0x03
524 #define IB_7220_LT_STATE_SLEEPDELAY      0x04
525 #define IB_7220_LT_STATE_SLEEPQUIET      0x05
526 #define IB_7220_LT_STATE_CFGDEBOUNCE     0x08
527 #define IB_7220_LT_STATE_CFGRCVFCFG      0x09
528 #define IB_7220_LT_STATE_CFGWAITRMT      0x0a
529 #define IB_7220_LT_STATE_CFGIDLE 0x0b
530 #define IB_7220_LT_STATE_RECOVERRETRAIN  0x0c
531 #define IB_7220_LT_STATE_RECOVERWAITRMT  0x0e
532 #define IB_7220_LT_STATE_RECOVERIDLE     0x0f
533 
534 /* link state machine states from IBC */
535 #define IB_7220_L_STATE_DOWN             0x0
536 #define IB_7220_L_STATE_INIT             0x1
537 #define IB_7220_L_STATE_ARM              0x2
538 #define IB_7220_L_STATE_ACTIVE           0x3
539 #define IB_7220_L_STATE_ACT_DEFER        0x4
540 
541 static const u8 qib_7220_physportstate[0x20] = {
542 	[IB_7220_LT_STATE_DISABLED] = IB_PHYSPORTSTATE_DISABLED,
543 	[IB_7220_LT_STATE_LINKUP] = IB_PHYSPORTSTATE_LINKUP,
544 	[IB_7220_LT_STATE_POLLACTIVE] = IB_PHYSPORTSTATE_POLL,
545 	[IB_7220_LT_STATE_POLLQUIET] = IB_PHYSPORTSTATE_POLL,
546 	[IB_7220_LT_STATE_SLEEPDELAY] = IB_PHYSPORTSTATE_SLEEP,
547 	[IB_7220_LT_STATE_SLEEPQUIET] = IB_PHYSPORTSTATE_SLEEP,
548 	[IB_7220_LT_STATE_CFGDEBOUNCE] =
549 		IB_PHYSPORTSTATE_CFG_TRAIN,
550 	[IB_7220_LT_STATE_CFGRCVFCFG] =
551 		IB_PHYSPORTSTATE_CFG_TRAIN,
552 	[IB_7220_LT_STATE_CFGWAITRMT] =
553 		IB_PHYSPORTSTATE_CFG_TRAIN,
554 	[IB_7220_LT_STATE_CFGIDLE] = IB_PHYSPORTSTATE_CFG_TRAIN,
555 	[IB_7220_LT_STATE_RECOVERRETRAIN] =
556 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
557 	[IB_7220_LT_STATE_RECOVERWAITRMT] =
558 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
559 	[IB_7220_LT_STATE_RECOVERIDLE] =
560 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
561 	[0x10] = IB_PHYSPORTSTATE_CFG_TRAIN,
562 	[0x11] = IB_PHYSPORTSTATE_CFG_TRAIN,
563 	[0x12] = IB_PHYSPORTSTATE_CFG_TRAIN,
564 	[0x13] = IB_PHYSPORTSTATE_CFG_TRAIN,
565 	[0x14] = IB_PHYSPORTSTATE_CFG_TRAIN,
566 	[0x15] = IB_PHYSPORTSTATE_CFG_TRAIN,
567 	[0x16] = IB_PHYSPORTSTATE_CFG_TRAIN,
568 	[0x17] = IB_PHYSPORTSTATE_CFG_TRAIN
569 };
570 
571 int qib_special_trigger;
572 module_param_named(special_trigger, qib_special_trigger, int, S_IRUGO);
573 MODULE_PARM_DESC(special_trigger, "Enable SpecialTrigger arm/launch");
574 
575 #define IBCBUSFRSPCPARITYERR HWE_MASK(IBCBusFromSPCParityErr)
576 #define IBCBUSTOSPCPARITYERR HWE_MASK(IBCBusToSPCParityErr)
577 
578 #define SYM_MASK_BIT(regname, fldname, bit) ((u64) \
579 	(1ULL << (SYM_LSB(regname, fldname) + (bit))))
580 
581 #define TXEMEMPARITYERR_PIOBUF \
582 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 0)
583 #define TXEMEMPARITYERR_PIOPBC \
584 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 1)
585 #define TXEMEMPARITYERR_PIOLAUNCHFIFO \
586 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 2)
587 
588 #define RXEMEMPARITYERR_RCVBUF \
589 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 0)
590 #define RXEMEMPARITYERR_LOOKUPQ \
591 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 1)
592 #define RXEMEMPARITYERR_EXPTID \
593 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 2)
594 #define RXEMEMPARITYERR_EAGERTID \
595 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 3)
596 #define RXEMEMPARITYERR_FLAGBUF \
597 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 4)
598 #define RXEMEMPARITYERR_DATAINFO \
599 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 5)
600 #define RXEMEMPARITYERR_HDRINFO \
601 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 6)
602 
603 /* 7220 specific hardware errors... */
604 static const struct qib_hwerror_msgs qib_7220_hwerror_msgs[] = {
605 	/* generic hardware errors */
606 	QLOGIC_IB_HWE_MSG(IBCBUSFRSPCPARITYERR, "QIB2IB Parity"),
607 	QLOGIC_IB_HWE_MSG(IBCBUSTOSPCPARITYERR, "IB2QIB Parity"),
608 
609 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOBUF,
610 			  "TXE PIOBUF Memory Parity"),
611 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOPBC,
612 			  "TXE PIOPBC Memory Parity"),
613 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOLAUNCHFIFO,
614 			  "TXE PIOLAUNCHFIFO Memory Parity"),
615 
616 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_RCVBUF,
617 			  "RXE RCVBUF Memory Parity"),
618 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_LOOKUPQ,
619 			  "RXE LOOKUPQ Memory Parity"),
620 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EAGERTID,
621 			  "RXE EAGERTID Memory Parity"),
622 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EXPTID,
623 			  "RXE EXPTID Memory Parity"),
624 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_FLAGBUF,
625 			  "RXE FLAGBUF Memory Parity"),
626 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_DATAINFO,
627 			  "RXE DATAINFO Memory Parity"),
628 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_HDRINFO,
629 			  "RXE HDRINFO Memory Parity"),
630 
631 	/* chip-specific hardware errors */
632 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEPOISONEDTLP,
633 			  "PCIe Poisoned TLP"),
634 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLTIMEOUT,
635 			  "PCIe completion timeout"),
636 	/*
637 	 * In practice, it's unlikely that we'll see PCIe PLL, or bus
638 	 * parity or memory parity error failures, because most likely we
639 	 * won't be able to talk to the core of the chip.  Nonetheless, we
640 	 * might see them, if they are in parts of the PCIe core that aren't
641 	 * essential.
642 	 */
643 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE1PLLFAILED,
644 			  "PCIePLL1"),
645 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE0PLLFAILED,
646 			  "PCIePLL0"),
647 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXTLH,
648 			  "PCIe XTLH core parity"),
649 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXADM,
650 			  "PCIe ADM TX core parity"),
651 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYRADM,
652 			  "PCIe ADM RX core parity"),
653 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_SERDESPLLFAILED,
654 			  "SerDes PLL"),
655 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLDATAQUEUEERR,
656 			  "PCIe cpl header queue"),
657 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLHDRQUEUEERR,
658 			  "PCIe cpl data queue"),
659 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_SDMAMEMREADERR,
660 			  "Send DMA memory read"),
661 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_CLK_UC_PLLNOTLOCKED,
662 			  "uC PLL clock not locked"),
663 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIESERDESQ0PCLKNOTDETECT,
664 			  "PCIe serdes Q0 no clock"),
665 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIESERDESQ1PCLKNOTDETECT,
666 			  "PCIe serdes Q1 no clock"),
667 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIESERDESQ2PCLKNOTDETECT,
668 			  "PCIe serdes Q2 no clock"),
669 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIESERDESQ3PCLKNOTDETECT,
670 			  "PCIe serdes Q3 no clock"),
671 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_DDSRXEQMEMORYPARITYERR,
672 			  "DDS RXEQ memory parity"),
673 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_IB_UC_MEMORYPARITYERR,
674 			  "IB uC memory parity"),
675 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE_UC_OCT0MEMORYPARITYERR,
676 			  "PCIe uC oct0 memory parity"),
677 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE_UC_OCT1MEMORYPARITYERR,
678 			  "PCIe uC oct1 memory parity"),
679 };
680 
681 #define RXE_PARITY (RXEMEMPARITYERR_EAGERTID|RXEMEMPARITYERR_EXPTID)
682 
683 #define QLOGIC_IB_E_PKTERRS (\
684 		ERR_MASK(SendPktLenErr) |				\
685 		ERR_MASK(SendDroppedDataPktErr) |			\
686 		ERR_MASK(RcvVCRCErr) |					\
687 		ERR_MASK(RcvICRCErr) |					\
688 		ERR_MASK(RcvShortPktLenErr) |				\
689 		ERR_MASK(RcvEBPErr))
690 
691 /* Convenience for decoding Send DMA errors */
692 #define QLOGIC_IB_E_SDMAERRS ( \
693 		ERR_MASK(SDmaGenMismatchErr) |				\
694 		ERR_MASK(SDmaOutOfBoundErr) |				\
695 		ERR_MASK(SDmaTailOutOfBoundErr) | ERR_MASK(SDmaBaseErr) | \
696 		ERR_MASK(SDma1stDescErr) | ERR_MASK(SDmaRpyTagErr) |	\
697 		ERR_MASK(SDmaDwEnErr) | ERR_MASK(SDmaMissingDwErr) |	\
698 		ERR_MASK(SDmaUnexpDataErr) |				\
699 		ERR_MASK(SDmaDescAddrMisalignErr) |			\
700 		ERR_MASK(SDmaDisabledErr) |				\
701 		ERR_MASK(SendBufMisuseErr))
702 
703 /* These are all rcv-related errors which we want to count for stats */
704 #define E_SUM_PKTERRS \
705 	(ERR_MASK(RcvHdrLenErr) | ERR_MASK(RcvBadTidErr) |		\
706 	 ERR_MASK(RcvBadVersionErr) | ERR_MASK(RcvHdrErr) |		\
707 	 ERR_MASK(RcvLongPktLenErr) | ERR_MASK(RcvShortPktLenErr) |	\
708 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
709 	 ERR_MASK(RcvFormatErr) | ERR_MASK(RcvUnsupportedVLErr) |	\
710 	 ERR_MASK(RcvUnexpectedCharErr) | ERR_MASK(RcvEBPErr))
711 
712 /* These are all send-related errors which we want to count for stats */
713 #define E_SUM_ERRS \
714 	(ERR_MASK(SendPioArmLaunchErr) | ERR_MASK(SendUnexpectedPktNumErr) | \
715 	 ERR_MASK(SendDroppedDataPktErr) | ERR_MASK(SendDroppedSmpPktErr) | \
716 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnsupportedVLErr) |	\
717 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
718 	 ERR_MASK(InvalidAddrErr))
719 
720 /*
721  * this is similar to E_SUM_ERRS, but can't ignore armlaunch, don't ignore
722  * errors not related to freeze and cancelling buffers.  Can't ignore
723  * armlaunch because could get more while still cleaning up, and need
724  * to cancel those as they happen.
725  */
726 #define E_SPKT_ERRS_IGNORE \
727 	(ERR_MASK(SendDroppedDataPktErr) | ERR_MASK(SendDroppedSmpPktErr) | \
728 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendMinPktLenErr) |	\
729 	 ERR_MASK(SendPktLenErr))
730 
731 /*
732  * these are errors that can occur when the link changes state while
733  * a packet is being sent or received.  This doesn't cover things
734  * like EBP or VCRC that can be the result of a sending having the
735  * link change state, so we receive a "known bad" packet.
736  */
737 #define E_SUM_LINK_PKTERRS \
738 	(ERR_MASK(SendDroppedDataPktErr) | ERR_MASK(SendDroppedSmpPktErr) | \
739 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
740 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
741 	 ERR_MASK(RcvUnexpectedCharErr))
742 
743 static void autoneg_7220_work(struct work_struct *);
744 static u32 __iomem *qib_7220_getsendbuf(struct qib_pportdata *, u64, u32 *);
745 
746 /*
747  * Called when we might have an error that is specific to a particular
748  * PIO buffer, and may need to cancel that buffer, so it can be re-used.
749  * because we don't need to force the update of pioavail.
750  */
751 static void qib_disarm_7220_senderrbufs(struct qib_pportdata *ppd)
752 {
753 	unsigned long sbuf[3];
754 	struct qib_devdata *dd = ppd->dd;
755 
756 	/*
757 	 * It's possible that sendbuffererror could have bits set; might
758 	 * have already done this as a result of hardware error handling.
759 	 */
760 	/* read these before writing errorclear */
761 	sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
762 	sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
763 	sbuf[2] = qib_read_kreg64(dd, kr_sendbuffererror + 2);
764 
765 	if (sbuf[0] || sbuf[1] || sbuf[2])
766 		qib_disarm_piobufs_set(dd, sbuf,
767 				       dd->piobcnt2k + dd->piobcnt4k);
768 }
769 
770 static void qib_7220_txe_recover(struct qib_devdata *dd)
771 {
772 	qib_devinfo(dd->pcidev, "Recovering from TXE PIO parity error\n");
773 	qib_disarm_7220_senderrbufs(dd->pport);
774 }
775 
776 /*
777  * This is called with interrupts disabled and sdma_lock held.
778  */
779 static void qib_7220_sdma_sendctrl(struct qib_pportdata *ppd, unsigned op)
780 {
781 	struct qib_devdata *dd = ppd->dd;
782 	u64 set_sendctrl = 0;
783 	u64 clr_sendctrl = 0;
784 
785 	if (op & QIB_SDMA_SENDCTRL_OP_ENABLE)
786 		set_sendctrl |= SYM_MASK(SendCtrl, SDmaEnable);
787 	else
788 		clr_sendctrl |= SYM_MASK(SendCtrl, SDmaEnable);
789 
790 	if (op & QIB_SDMA_SENDCTRL_OP_INTENABLE)
791 		set_sendctrl |= SYM_MASK(SendCtrl, SDmaIntEnable);
792 	else
793 		clr_sendctrl |= SYM_MASK(SendCtrl, SDmaIntEnable);
794 
795 	if (op & QIB_SDMA_SENDCTRL_OP_HALT)
796 		set_sendctrl |= SYM_MASK(SendCtrl, SDmaHalt);
797 	else
798 		clr_sendctrl |= SYM_MASK(SendCtrl, SDmaHalt);
799 
800 	spin_lock(&dd->sendctrl_lock);
801 
802 	dd->sendctrl |= set_sendctrl;
803 	dd->sendctrl &= ~clr_sendctrl;
804 
805 	qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
806 	qib_write_kreg(dd, kr_scratch, 0);
807 
808 	spin_unlock(&dd->sendctrl_lock);
809 }
810 
811 static void qib_decode_7220_sdma_errs(struct qib_pportdata *ppd,
812 				      u64 err, char *buf, size_t blen)
813 {
814 	static const struct {
815 		u64 err;
816 		const char *msg;
817 	} errs[] = {
818 		{ ERR_MASK(SDmaGenMismatchErr),
819 		  "SDmaGenMismatch" },
820 		{ ERR_MASK(SDmaOutOfBoundErr),
821 		  "SDmaOutOfBound" },
822 		{ ERR_MASK(SDmaTailOutOfBoundErr),
823 		  "SDmaTailOutOfBound" },
824 		{ ERR_MASK(SDmaBaseErr),
825 		  "SDmaBase" },
826 		{ ERR_MASK(SDma1stDescErr),
827 		  "SDma1stDesc" },
828 		{ ERR_MASK(SDmaRpyTagErr),
829 		  "SDmaRpyTag" },
830 		{ ERR_MASK(SDmaDwEnErr),
831 		  "SDmaDwEn" },
832 		{ ERR_MASK(SDmaMissingDwErr),
833 		  "SDmaMissingDw" },
834 		{ ERR_MASK(SDmaUnexpDataErr),
835 		  "SDmaUnexpData" },
836 		{ ERR_MASK(SDmaDescAddrMisalignErr),
837 		  "SDmaDescAddrMisalign" },
838 		{ ERR_MASK(SendBufMisuseErr),
839 		  "SendBufMisuse" },
840 		{ ERR_MASK(SDmaDisabledErr),
841 		  "SDmaDisabled" },
842 	};
843 	int i;
844 	size_t bidx = 0;
845 
846 	for (i = 0; i < ARRAY_SIZE(errs); i++) {
847 		if (err & errs[i].err)
848 			bidx += scnprintf(buf + bidx, blen - bidx,
849 					 "%s ", errs[i].msg);
850 	}
851 }
852 
853 /*
854  * This is called as part of link down clean up so disarm and flush
855  * all send buffers so that SMP packets can be sent.
856  */
857 static void qib_7220_sdma_hw_clean_up(struct qib_pportdata *ppd)
858 {
859 	/* This will trigger the Abort interrupt */
860 	sendctrl_7220_mod(ppd, QIB_SENDCTRL_DISARM_ALL | QIB_SENDCTRL_FLUSH |
861 			  QIB_SENDCTRL_AVAIL_BLIP);
862 	ppd->dd->upd_pio_shadow  = 1; /* update our idea of what's busy */
863 }
864 
865 static void qib_sdma_7220_setlengen(struct qib_pportdata *ppd)
866 {
867 	/*
868 	 * Set SendDmaLenGen and clear and set
869 	 * the MSB of the generation count to enable generation checking
870 	 * and load the internal generation counter.
871 	 */
872 	qib_write_kreg(ppd->dd, kr_senddmalengen, ppd->sdma_descq_cnt);
873 	qib_write_kreg(ppd->dd, kr_senddmalengen,
874 		       ppd->sdma_descq_cnt |
875 		       (1ULL << QIB_7220_SendDmaLenGen_Generation_MSB));
876 }
877 
878 static void qib_7220_sdma_hw_start_up(struct qib_pportdata *ppd)
879 {
880 	qib_sdma_7220_setlengen(ppd);
881 	qib_sdma_update_7220_tail(ppd, 0); /* Set SendDmaTail */
882 	ppd->sdma_head_dma[0] = 0;
883 }
884 
885 #define DISABLES_SDMA (							\
886 		ERR_MASK(SDmaDisabledErr) |				\
887 		ERR_MASK(SDmaBaseErr) |					\
888 		ERR_MASK(SDmaTailOutOfBoundErr) |			\
889 		ERR_MASK(SDmaOutOfBoundErr) |				\
890 		ERR_MASK(SDma1stDescErr) |				\
891 		ERR_MASK(SDmaRpyTagErr) |				\
892 		ERR_MASK(SDmaGenMismatchErr) |				\
893 		ERR_MASK(SDmaDescAddrMisalignErr) |			\
894 		ERR_MASK(SDmaMissingDwErr) |				\
895 		ERR_MASK(SDmaDwEnErr))
896 
897 static void sdma_7220_errors(struct qib_pportdata *ppd, u64 errs)
898 {
899 	unsigned long flags;
900 	struct qib_devdata *dd = ppd->dd;
901 	char *msg;
902 
903 	errs &= QLOGIC_IB_E_SDMAERRS;
904 
905 	msg = dd->cspec->sdmamsgbuf;
906 	qib_decode_7220_sdma_errs(ppd, errs, msg,
907 		sizeof(dd->cspec->sdmamsgbuf));
908 	spin_lock_irqsave(&ppd->sdma_lock, flags);
909 
910 	if (errs & ERR_MASK(SendBufMisuseErr)) {
911 		unsigned long sbuf[3];
912 
913 		sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
914 		sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
915 		sbuf[2] = qib_read_kreg64(dd, kr_sendbuffererror + 2);
916 
917 		qib_dev_err(ppd->dd,
918 			    "IB%u:%u SendBufMisuse: %04lx %016lx %016lx\n",
919 			    ppd->dd->unit, ppd->port, sbuf[2], sbuf[1],
920 			    sbuf[0]);
921 	}
922 
923 	if (errs & ERR_MASK(SDmaUnexpDataErr))
924 		qib_dev_err(dd, "IB%u:%u SDmaUnexpData\n", ppd->dd->unit,
925 			    ppd->port);
926 
927 	switch (ppd->sdma_state.current_state) {
928 	case qib_sdma_state_s00_hw_down:
929 		/* not expecting any interrupts */
930 		break;
931 
932 	case qib_sdma_state_s10_hw_start_up_wait:
933 		/* handled in intr path */
934 		break;
935 
936 	case qib_sdma_state_s20_idle:
937 		/* not expecting any interrupts */
938 		break;
939 
940 	case qib_sdma_state_s30_sw_clean_up_wait:
941 		/* not expecting any interrupts */
942 		break;
943 
944 	case qib_sdma_state_s40_hw_clean_up_wait:
945 		if (errs & ERR_MASK(SDmaDisabledErr))
946 			__qib_sdma_process_event(ppd,
947 				qib_sdma_event_e50_hw_cleaned);
948 		break;
949 
950 	case qib_sdma_state_s50_hw_halt_wait:
951 		/* handled in intr path */
952 		break;
953 
954 	case qib_sdma_state_s99_running:
955 		if (errs & DISABLES_SDMA)
956 			__qib_sdma_process_event(ppd,
957 				qib_sdma_event_e7220_err_halted);
958 		break;
959 	}
960 
961 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
962 }
963 
964 /*
965  * Decode the error status into strings, deciding whether to always
966  * print * it or not depending on "normal packet errors" vs everything
967  * else.   Return 1 if "real" errors, otherwise 0 if only packet
968  * errors, so caller can decide what to print with the string.
969  */
970 static int qib_decode_7220_err(struct qib_devdata *dd, char *buf, size_t blen,
971 			       u64 err)
972 {
973 	int iserr = 1;
974 
975 	*buf = '\0';
976 	if (err & QLOGIC_IB_E_PKTERRS) {
977 		if (!(err & ~QLOGIC_IB_E_PKTERRS))
978 			iserr = 0;
979 		if ((err & ERR_MASK(RcvICRCErr)) &&
980 		    !(err & (ERR_MASK(RcvVCRCErr) | ERR_MASK(RcvEBPErr))))
981 			strlcat(buf, "CRC ", blen);
982 		if (!iserr)
983 			goto done;
984 	}
985 	if (err & ERR_MASK(RcvHdrLenErr))
986 		strlcat(buf, "rhdrlen ", blen);
987 	if (err & ERR_MASK(RcvBadTidErr))
988 		strlcat(buf, "rbadtid ", blen);
989 	if (err & ERR_MASK(RcvBadVersionErr))
990 		strlcat(buf, "rbadversion ", blen);
991 	if (err & ERR_MASK(RcvHdrErr))
992 		strlcat(buf, "rhdr ", blen);
993 	if (err & ERR_MASK(SendSpecialTriggerErr))
994 		strlcat(buf, "sendspecialtrigger ", blen);
995 	if (err & ERR_MASK(RcvLongPktLenErr))
996 		strlcat(buf, "rlongpktlen ", blen);
997 	if (err & ERR_MASK(RcvMaxPktLenErr))
998 		strlcat(buf, "rmaxpktlen ", blen);
999 	if (err & ERR_MASK(RcvMinPktLenErr))
1000 		strlcat(buf, "rminpktlen ", blen);
1001 	if (err & ERR_MASK(SendMinPktLenErr))
1002 		strlcat(buf, "sminpktlen ", blen);
1003 	if (err & ERR_MASK(RcvFormatErr))
1004 		strlcat(buf, "rformaterr ", blen);
1005 	if (err & ERR_MASK(RcvUnsupportedVLErr))
1006 		strlcat(buf, "runsupvl ", blen);
1007 	if (err & ERR_MASK(RcvUnexpectedCharErr))
1008 		strlcat(buf, "runexpchar ", blen);
1009 	if (err & ERR_MASK(RcvIBFlowErr))
1010 		strlcat(buf, "ribflow ", blen);
1011 	if (err & ERR_MASK(SendUnderRunErr))
1012 		strlcat(buf, "sunderrun ", blen);
1013 	if (err & ERR_MASK(SendPioArmLaunchErr))
1014 		strlcat(buf, "spioarmlaunch ", blen);
1015 	if (err & ERR_MASK(SendUnexpectedPktNumErr))
1016 		strlcat(buf, "sunexperrpktnum ", blen);
1017 	if (err & ERR_MASK(SendDroppedSmpPktErr))
1018 		strlcat(buf, "sdroppedsmppkt ", blen);
1019 	if (err & ERR_MASK(SendMaxPktLenErr))
1020 		strlcat(buf, "smaxpktlen ", blen);
1021 	if (err & ERR_MASK(SendUnsupportedVLErr))
1022 		strlcat(buf, "sunsupVL ", blen);
1023 	if (err & ERR_MASK(InvalidAddrErr))
1024 		strlcat(buf, "invalidaddr ", blen);
1025 	if (err & ERR_MASK(RcvEgrFullErr))
1026 		strlcat(buf, "rcvegrfull ", blen);
1027 	if (err & ERR_MASK(RcvHdrFullErr))
1028 		strlcat(buf, "rcvhdrfull ", blen);
1029 	if (err & ERR_MASK(IBStatusChanged))
1030 		strlcat(buf, "ibcstatuschg ", blen);
1031 	if (err & ERR_MASK(RcvIBLostLinkErr))
1032 		strlcat(buf, "riblostlink ", blen);
1033 	if (err & ERR_MASK(HardwareErr))
1034 		strlcat(buf, "hardware ", blen);
1035 	if (err & ERR_MASK(ResetNegated))
1036 		strlcat(buf, "reset ", blen);
1037 	if (err & QLOGIC_IB_E_SDMAERRS)
1038 		qib_decode_7220_sdma_errs(dd->pport, err, buf, blen);
1039 	if (err & ERR_MASK(InvalidEEPCmd))
1040 		strlcat(buf, "invalideepromcmd ", blen);
1041 done:
1042 	return iserr;
1043 }
1044 
1045 static void reenable_7220_chase(struct timer_list *t)
1046 {
1047 	struct qib_chippport_specific *cpspec = from_timer(cpspec, t,
1048 							 chase_timer);
1049 	struct qib_pportdata *ppd = &cpspec->pportdata;
1050 
1051 	ppd->cpspec->chase_timer.expires = 0;
1052 	qib_set_ib_7220_lstate(ppd, QLOGIC_IB_IBCC_LINKCMD_DOWN,
1053 		QLOGIC_IB_IBCC_LINKINITCMD_POLL);
1054 }
1055 
1056 static void handle_7220_chase(struct qib_pportdata *ppd, u64 ibcst)
1057 {
1058 	u8 ibclt;
1059 	unsigned long tnow;
1060 
1061 	ibclt = (u8)SYM_FIELD(ibcst, IBCStatus, LinkTrainingState);
1062 
1063 	/*
1064 	 * Detect and handle the state chase issue, where we can
1065 	 * get stuck if we are unlucky on timing on both sides of
1066 	 * the link.   If we are, we disable, set a timer, and
1067 	 * then re-enable.
1068 	 */
1069 	switch (ibclt) {
1070 	case IB_7220_LT_STATE_CFGRCVFCFG:
1071 	case IB_7220_LT_STATE_CFGWAITRMT:
1072 	case IB_7220_LT_STATE_TXREVLANES:
1073 	case IB_7220_LT_STATE_CFGENH:
1074 		tnow = jiffies;
1075 		if (ppd->cpspec->chase_end &&
1076 		    time_after(tnow, ppd->cpspec->chase_end)) {
1077 			ppd->cpspec->chase_end = 0;
1078 			qib_set_ib_7220_lstate(ppd,
1079 				QLOGIC_IB_IBCC_LINKCMD_DOWN,
1080 				QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1081 			ppd->cpspec->chase_timer.expires = jiffies +
1082 				QIB_CHASE_DIS_TIME;
1083 			add_timer(&ppd->cpspec->chase_timer);
1084 		} else if (!ppd->cpspec->chase_end)
1085 			ppd->cpspec->chase_end = tnow + QIB_CHASE_TIME;
1086 		break;
1087 
1088 	default:
1089 		ppd->cpspec->chase_end = 0;
1090 		break;
1091 	}
1092 }
1093 
1094 static void handle_7220_errors(struct qib_devdata *dd, u64 errs)
1095 {
1096 	char *msg;
1097 	u64 ignore_this_time = 0;
1098 	u64 iserr = 0;
1099 	struct qib_pportdata *ppd = dd->pport;
1100 	u64 mask;
1101 
1102 	/* don't report errors that are masked */
1103 	errs &= dd->cspec->errormask;
1104 	msg = dd->cspec->emsgbuf;
1105 
1106 	/* do these first, they are most important */
1107 	if (errs & ERR_MASK(HardwareErr))
1108 		qib_7220_handle_hwerrors(dd, msg, sizeof(dd->cspec->emsgbuf));
1109 
1110 	if (errs & QLOGIC_IB_E_SDMAERRS)
1111 		sdma_7220_errors(ppd, errs);
1112 
1113 	if (errs & ~IB_E_BITSEXTANT)
1114 		qib_dev_err(dd,
1115 			"error interrupt with unknown errors %llx set\n",
1116 			(unsigned long long) (errs & ~IB_E_BITSEXTANT));
1117 
1118 	if (errs & E_SUM_ERRS) {
1119 		qib_disarm_7220_senderrbufs(ppd);
1120 		if ((errs & E_SUM_LINK_PKTERRS) &&
1121 		    !(ppd->lflags & QIBL_LINKACTIVE)) {
1122 			/*
1123 			 * This can happen when trying to bring the link
1124 			 * up, but the IB link changes state at the "wrong"
1125 			 * time. The IB logic then complains that the packet
1126 			 * isn't valid.  We don't want to confuse people, so
1127 			 * we just don't print them, except at debug
1128 			 */
1129 			ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1130 		}
1131 	} else if ((errs & E_SUM_LINK_PKTERRS) &&
1132 		   !(ppd->lflags & QIBL_LINKACTIVE)) {
1133 		/*
1134 		 * This can happen when SMA is trying to bring the link
1135 		 * up, but the IB link changes state at the "wrong" time.
1136 		 * The IB logic then complains that the packet isn't
1137 		 * valid.  We don't want to confuse people, so we just
1138 		 * don't print them, except at debug
1139 		 */
1140 		ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1141 	}
1142 
1143 	qib_write_kreg(dd, kr_errclear, errs);
1144 
1145 	errs &= ~ignore_this_time;
1146 	if (!errs)
1147 		goto done;
1148 
1149 	/*
1150 	 * The ones we mask off are handled specially below
1151 	 * or above.  Also mask SDMADISABLED by default as it
1152 	 * is too chatty.
1153 	 */
1154 	mask = ERR_MASK(IBStatusChanged) |
1155 		ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) |
1156 		ERR_MASK(HardwareErr) | ERR_MASK(SDmaDisabledErr);
1157 
1158 	qib_decode_7220_err(dd, msg, sizeof(dd->cspec->emsgbuf), errs & ~mask);
1159 
1160 	if (errs & E_SUM_PKTERRS)
1161 		qib_stats.sps_rcverrs++;
1162 	if (errs & E_SUM_ERRS)
1163 		qib_stats.sps_txerrs++;
1164 	iserr = errs & ~(E_SUM_PKTERRS | QLOGIC_IB_E_PKTERRS |
1165 			 ERR_MASK(SDmaDisabledErr));
1166 
1167 	if (errs & ERR_MASK(IBStatusChanged)) {
1168 		u64 ibcs;
1169 
1170 		ibcs = qib_read_kreg64(dd, kr_ibcstatus);
1171 		if (!(ppd->lflags & QIBL_IB_AUTONEG_INPROG))
1172 			handle_7220_chase(ppd, ibcs);
1173 
1174 		/* Update our picture of width and speed from chip */
1175 		ppd->link_width_active =
1176 			((ibcs >> IBA7220_LINKWIDTH_SHIFT) & 1) ?
1177 			    IB_WIDTH_4X : IB_WIDTH_1X;
1178 		ppd->link_speed_active =
1179 			((ibcs >> IBA7220_LINKSPEED_SHIFT) & 1) ?
1180 			    QIB_IB_DDR : QIB_IB_SDR;
1181 
1182 		/*
1183 		 * Since going into a recovery state causes the link state
1184 		 * to go down and since recovery is transitory, it is better
1185 		 * if we "miss" ever seeing the link training state go into
1186 		 * recovery (i.e., ignore this transition for link state
1187 		 * special handling purposes) without updating lastibcstat.
1188 		 */
1189 		if (qib_7220_phys_portstate(ibcs) !=
1190 					    IB_PHYSPORTSTATE_LINK_ERR_RECOVER)
1191 			qib_handle_e_ibstatuschanged(ppd, ibcs);
1192 	}
1193 
1194 	if (errs & ERR_MASK(ResetNegated)) {
1195 		qib_dev_err(dd,
1196 			"Got reset, requires re-init (unload and reload driver)\n");
1197 		dd->flags &= ~QIB_INITTED;  /* needs re-init */
1198 		/* mark as having had error */
1199 		*dd->devstatusp |= QIB_STATUS_HWERROR;
1200 		*dd->pport->statusp &= ~QIB_STATUS_IB_CONF;
1201 	}
1202 
1203 	if (*msg && iserr)
1204 		qib_dev_porterr(dd, ppd->port, "%s error\n", msg);
1205 
1206 	if (ppd->state_wanted & ppd->lflags)
1207 		wake_up_interruptible(&ppd->state_wait);
1208 
1209 	/*
1210 	 * If there were hdrq or egrfull errors, wake up any processes
1211 	 * waiting in poll.  We used to try to check which contexts had
1212 	 * the overflow, but given the cost of that and the chip reads
1213 	 * to support it, it's better to just wake everybody up if we
1214 	 * get an overflow; waiters can poll again if it's not them.
1215 	 */
1216 	if (errs & (ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr))) {
1217 		qib_handle_urcv(dd, ~0U);
1218 		if (errs & ERR_MASK(RcvEgrFullErr))
1219 			qib_stats.sps_buffull++;
1220 		else
1221 			qib_stats.sps_hdrfull++;
1222 	}
1223 done:
1224 	return;
1225 }
1226 
1227 /* enable/disable chip from delivering interrupts */
1228 static void qib_7220_set_intr_state(struct qib_devdata *dd, u32 enable)
1229 {
1230 	if (enable) {
1231 		if (dd->flags & QIB_BADINTR)
1232 			return;
1233 		qib_write_kreg(dd, kr_intmask, ~0ULL);
1234 		/* force re-interrupt of any pending interrupts. */
1235 		qib_write_kreg(dd, kr_intclear, 0ULL);
1236 	} else
1237 		qib_write_kreg(dd, kr_intmask, 0ULL);
1238 }
1239 
1240 /*
1241  * Try to cleanup as much as possible for anything that might have gone
1242  * wrong while in freeze mode, such as pio buffers being written by user
1243  * processes (causing armlaunch), send errors due to going into freeze mode,
1244  * etc., and try to avoid causing extra interrupts while doing so.
1245  * Forcibly update the in-memory pioavail register copies after cleanup
1246  * because the chip won't do it while in freeze mode (the register values
1247  * themselves are kept correct).
1248  * Make sure that we don't lose any important interrupts by using the chip
1249  * feature that says that writing 0 to a bit in *clear that is set in
1250  * *status will cause an interrupt to be generated again (if allowed by
1251  * the *mask value).
1252  * This is in chip-specific code because of all of the register accesses,
1253  * even though the details are similar on most chips.
1254  */
1255 static void qib_7220_clear_freeze(struct qib_devdata *dd)
1256 {
1257 	/* disable error interrupts, to avoid confusion */
1258 	qib_write_kreg(dd, kr_errmask, 0ULL);
1259 
1260 	/* also disable interrupts; errormask is sometimes overwritten */
1261 	qib_7220_set_intr_state(dd, 0);
1262 
1263 	qib_cancel_sends(dd->pport);
1264 
1265 	/* clear the freeze, and be sure chip saw it */
1266 	qib_write_kreg(dd, kr_control, dd->control);
1267 	qib_read_kreg32(dd, kr_scratch);
1268 
1269 	/* force in-memory update now we are out of freeze */
1270 	qib_force_pio_avail_update(dd);
1271 
1272 	/*
1273 	 * force new interrupt if any hwerr, error or interrupt bits are
1274 	 * still set, and clear "safe" send packet errors related to freeze
1275 	 * and cancelling sends.  Re-enable error interrupts before possible
1276 	 * force of re-interrupt on pending interrupts.
1277 	 */
1278 	qib_write_kreg(dd, kr_hwerrclear, 0ULL);
1279 	qib_write_kreg(dd, kr_errclear, E_SPKT_ERRS_IGNORE);
1280 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1281 	qib_7220_set_intr_state(dd, 1);
1282 }
1283 
1284 /**
1285  * qib_7220_handle_hwerrors - display hardware errors.
1286  * @dd: the qlogic_ib device
1287  * @msg: the output buffer
1288  * @msgl: the size of the output buffer
1289  *
1290  * Use same msg buffer as regular errors to avoid excessive stack
1291  * use.  Most hardware errors are catastrophic, but for right now,
1292  * we'll print them and continue.  We reuse the same message buffer as
1293  * handle_7220_errors() to avoid excessive stack usage.
1294  */
1295 static void qib_7220_handle_hwerrors(struct qib_devdata *dd, char *msg,
1296 				     size_t msgl)
1297 {
1298 	u64 hwerrs;
1299 	u32 bits, ctrl;
1300 	int isfatal = 0;
1301 	char *bitsmsg;
1302 
1303 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
1304 	if (!hwerrs)
1305 		goto bail;
1306 	if (hwerrs == ~0ULL) {
1307 		qib_dev_err(dd,
1308 			"Read of hardware error status failed (all bits set); ignoring\n");
1309 		goto bail;
1310 	}
1311 	qib_stats.sps_hwerrs++;
1312 
1313 	/*
1314 	 * Always clear the error status register, except MEMBISTFAIL,
1315 	 * regardless of whether we continue or stop using the chip.
1316 	 * We want that set so we know it failed, even across driver reload.
1317 	 * We'll still ignore it in the hwerrmask.  We do this partly for
1318 	 * diagnostics, but also for support.
1319 	 */
1320 	qib_write_kreg(dd, kr_hwerrclear,
1321 		       hwerrs & ~HWE_MASK(PowerOnBISTFailed));
1322 
1323 	hwerrs &= dd->cspec->hwerrmask;
1324 
1325 	if (hwerrs & ~(TXEMEMPARITYERR_PIOBUF | TXEMEMPARITYERR_PIOPBC |
1326 		       RXE_PARITY))
1327 		qib_devinfo(dd->pcidev,
1328 			"Hardware error: hwerr=0x%llx (cleared)\n",
1329 			(unsigned long long) hwerrs);
1330 
1331 	if (hwerrs & ~IB_HWE_BITSEXTANT)
1332 		qib_dev_err(dd,
1333 			"hwerror interrupt with unknown errors %llx set\n",
1334 			(unsigned long long) (hwerrs & ~IB_HWE_BITSEXTANT));
1335 
1336 	if (hwerrs & QLOGIC_IB_HWE_IB_UC_MEMORYPARITYERR)
1337 		qib_sd7220_clr_ibpar(dd);
1338 
1339 	ctrl = qib_read_kreg32(dd, kr_control);
1340 	if ((ctrl & QLOGIC_IB_C_FREEZEMODE) && !dd->diag_client) {
1341 		/*
1342 		 * Parity errors in send memory are recoverable by h/w
1343 		 * just do housekeeping, exit freeze mode and continue.
1344 		 */
1345 		if (hwerrs & (TXEMEMPARITYERR_PIOBUF |
1346 			      TXEMEMPARITYERR_PIOPBC)) {
1347 			qib_7220_txe_recover(dd);
1348 			hwerrs &= ~(TXEMEMPARITYERR_PIOBUF |
1349 				    TXEMEMPARITYERR_PIOPBC);
1350 		}
1351 		if (hwerrs)
1352 			isfatal = 1;
1353 		else
1354 			qib_7220_clear_freeze(dd);
1355 	}
1356 
1357 	*msg = '\0';
1358 
1359 	if (hwerrs & HWE_MASK(PowerOnBISTFailed)) {
1360 		isfatal = 1;
1361 		strlcat(msg,
1362 			"[Memory BIST test failed, InfiniPath hardware unusable]",
1363 			msgl);
1364 		/* ignore from now on, so disable until driver reloaded */
1365 		dd->cspec->hwerrmask &= ~HWE_MASK(PowerOnBISTFailed);
1366 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1367 	}
1368 
1369 	qib_format_hwerrors(hwerrs, qib_7220_hwerror_msgs,
1370 			    ARRAY_SIZE(qib_7220_hwerror_msgs), msg, msgl);
1371 
1372 	bitsmsg = dd->cspec->bitsmsgbuf;
1373 	if (hwerrs & (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<
1374 		      QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT)) {
1375 		bits = (u32) ((hwerrs >>
1376 			       QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) &
1377 			      QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK);
1378 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
1379 			 "[PCIe Mem Parity Errs %x] ", bits);
1380 		strlcat(msg, bitsmsg, msgl);
1381 	}
1382 
1383 #define _QIB_PLL_FAIL (QLOGIC_IB_HWE_COREPLL_FBSLIP |   \
1384 			 QLOGIC_IB_HWE_COREPLL_RFSLIP)
1385 
1386 	if (hwerrs & _QIB_PLL_FAIL) {
1387 		isfatal = 1;
1388 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
1389 			 "[PLL failed (%llx), InfiniPath hardware unusable]",
1390 			 (unsigned long long) hwerrs & _QIB_PLL_FAIL);
1391 		strlcat(msg, bitsmsg, msgl);
1392 		/* ignore from now on, so disable until driver reloaded */
1393 		dd->cspec->hwerrmask &= ~(hwerrs & _QIB_PLL_FAIL);
1394 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1395 	}
1396 
1397 	if (hwerrs & QLOGIC_IB_HWE_SERDESPLLFAILED) {
1398 		/*
1399 		 * If it occurs, it is left masked since the eternal
1400 		 * interface is unused.
1401 		 */
1402 		dd->cspec->hwerrmask &= ~QLOGIC_IB_HWE_SERDESPLLFAILED;
1403 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1404 	}
1405 
1406 	qib_dev_err(dd, "%s hardware error\n", msg);
1407 
1408 	if (isfatal && !dd->diag_client) {
1409 		qib_dev_err(dd,
1410 			"Fatal Hardware Error, no longer usable, SN %.16s\n",
1411 			dd->serial);
1412 		/*
1413 		 * For /sys status file and user programs to print; if no
1414 		 * trailing brace is copied, we'll know it was truncated.
1415 		 */
1416 		if (dd->freezemsg)
1417 			snprintf(dd->freezemsg, dd->freezelen,
1418 				 "{%s}", msg);
1419 		qib_disable_after_error(dd);
1420 	}
1421 bail:;
1422 }
1423 
1424 /**
1425  * qib_7220_init_hwerrors - enable hardware errors
1426  * @dd: the qlogic_ib device
1427  *
1428  * now that we have finished initializing everything that might reasonably
1429  * cause a hardware error, and cleared those errors bits as they occur,
1430  * we can enable hardware errors in the mask (potentially enabling
1431  * freeze mode), and enable hardware errors as errors (along with
1432  * everything else) in errormask
1433  */
1434 static void qib_7220_init_hwerrors(struct qib_devdata *dd)
1435 {
1436 	u64 val;
1437 	u64 extsval;
1438 
1439 	extsval = qib_read_kreg64(dd, kr_extstatus);
1440 
1441 	if (!(extsval & (QLOGIC_IB_EXTS_MEMBIST_ENDTEST |
1442 			 QLOGIC_IB_EXTS_MEMBIST_DISABLED)))
1443 		qib_dev_err(dd, "MemBIST did not complete!\n");
1444 	if (extsval & QLOGIC_IB_EXTS_MEMBIST_DISABLED)
1445 		qib_devinfo(dd->pcidev, "MemBIST is disabled.\n");
1446 
1447 	val = ~0ULL;    /* default to all hwerrors become interrupts, */
1448 
1449 	val &= ~QLOGIC_IB_HWE_IB_UC_MEMORYPARITYERR;
1450 	dd->cspec->hwerrmask = val;
1451 
1452 	qib_write_kreg(dd, kr_hwerrclear, ~HWE_MASK(PowerOnBISTFailed));
1453 	qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1454 
1455 	/* clear all */
1456 	qib_write_kreg(dd, kr_errclear, ~0ULL);
1457 	/* enable errors that are masked, at least this first time. */
1458 	qib_write_kreg(dd, kr_errmask, ~0ULL);
1459 	dd->cspec->errormask = qib_read_kreg64(dd, kr_errmask);
1460 	/* clear any interrupts up to this point (ints still not enabled) */
1461 	qib_write_kreg(dd, kr_intclear, ~0ULL);
1462 }
1463 
1464 /*
1465  * Disable and enable the armlaunch error.  Used for PIO bandwidth testing
1466  * on chips that are count-based, rather than trigger-based.  There is no
1467  * reference counting, but that's also fine, given the intended use.
1468  * Only chip-specific because it's all register accesses
1469  */
1470 static void qib_set_7220_armlaunch(struct qib_devdata *dd, u32 enable)
1471 {
1472 	if (enable) {
1473 		qib_write_kreg(dd, kr_errclear, ERR_MASK(SendPioArmLaunchErr));
1474 		dd->cspec->errormask |= ERR_MASK(SendPioArmLaunchErr);
1475 	} else
1476 		dd->cspec->errormask &= ~ERR_MASK(SendPioArmLaunchErr);
1477 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1478 }
1479 
1480 /*
1481  * Formerly took parameter <which> in pre-shifted,
1482  * pre-merged form with LinkCmd and LinkInitCmd
1483  * together, and assuming the zero was NOP.
1484  */
1485 static void qib_set_ib_7220_lstate(struct qib_pportdata *ppd, u16 linkcmd,
1486 				   u16 linitcmd)
1487 {
1488 	u64 mod_wd;
1489 	struct qib_devdata *dd = ppd->dd;
1490 	unsigned long flags;
1491 
1492 	if (linitcmd == QLOGIC_IB_IBCC_LINKINITCMD_DISABLE) {
1493 		/*
1494 		 * If we are told to disable, note that so link-recovery
1495 		 * code does not attempt to bring us back up.
1496 		 */
1497 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1498 		ppd->lflags |= QIBL_IB_LINK_DISABLED;
1499 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1500 	} else if (linitcmd || linkcmd == QLOGIC_IB_IBCC_LINKCMD_DOWN) {
1501 		/*
1502 		 * Any other linkinitcmd will lead to LINKDOWN and then
1503 		 * to INIT (if all is well), so clear flag to let
1504 		 * link-recovery code attempt to bring us back up.
1505 		 */
1506 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1507 		ppd->lflags &= ~QIBL_IB_LINK_DISABLED;
1508 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1509 	}
1510 
1511 	mod_wd = (linkcmd << IBA7220_IBCC_LINKCMD_SHIFT) |
1512 		(linitcmd << QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1513 
1514 	qib_write_kreg(dd, kr_ibcctrl, ppd->cpspec->ibcctrl | mod_wd);
1515 	/* write to chip to prevent back-to-back writes of ibc reg */
1516 	qib_write_kreg(dd, kr_scratch, 0);
1517 }
1518 
1519 /*
1520  * All detailed interaction with the SerDes has been moved to qib_sd7220.c
1521  *
1522  * The portion of IBA7220-specific bringup_serdes() that actually deals with
1523  * registers and memory within the SerDes itself is qib_sd7220_init().
1524  */
1525 
1526 /**
1527  * qib_7220_bringup_serdes - bring up the serdes
1528  * @ppd: physical port on the qlogic_ib device
1529  */
1530 static int qib_7220_bringup_serdes(struct qib_pportdata *ppd)
1531 {
1532 	struct qib_devdata *dd = ppd->dd;
1533 	u64 val, prev_val, guid, ibc;
1534 	int ret = 0;
1535 
1536 	/* Put IBC in reset, sends disabled */
1537 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1538 	qib_write_kreg(dd, kr_control, 0ULL);
1539 
1540 	if (qib_compat_ddr_negotiate) {
1541 		ppd->cpspec->ibdeltainprog = 1;
1542 		ppd->cpspec->ibsymsnap = read_7220_creg32(dd, cr_ibsymbolerr);
1543 		ppd->cpspec->iblnkerrsnap =
1544 			read_7220_creg32(dd, cr_iblinkerrrecov);
1545 	}
1546 
1547 	/* flowcontrolwatermark is in units of KBytes */
1548 	ibc = 0x5ULL << SYM_LSB(IBCCtrl, FlowCtrlWaterMark);
1549 	/*
1550 	 * How often flowctrl sent.  More or less in usecs; balance against
1551 	 * watermark value, so that in theory senders always get a flow
1552 	 * control update in time to not let the IB link go idle.
1553 	 */
1554 	ibc |= 0x3ULL << SYM_LSB(IBCCtrl, FlowCtrlPeriod);
1555 	/* max error tolerance */
1556 	ibc |= 0xfULL << SYM_LSB(IBCCtrl, PhyerrThreshold);
1557 	/* use "real" buffer space for */
1558 	ibc |= 4ULL << SYM_LSB(IBCCtrl, CreditScale);
1559 	/* IB credit flow control. */
1560 	ibc |= 0xfULL << SYM_LSB(IBCCtrl, OverrunThreshold);
1561 	/*
1562 	 * set initial max size pkt IBC will send, including ICRC; it's the
1563 	 * PIO buffer size in dwords, less 1; also see qib_set_mtu()
1564 	 */
1565 	ibc |= ((u64)(ppd->ibmaxlen >> 2) + 1) << SYM_LSB(IBCCtrl, MaxPktLen);
1566 	ppd->cpspec->ibcctrl = ibc; /* without linkcmd or linkinitcmd! */
1567 
1568 	/* initially come up waiting for TS1, without sending anything. */
1569 	val = ppd->cpspec->ibcctrl | (QLOGIC_IB_IBCC_LINKINITCMD_DISABLE <<
1570 		QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1571 	qib_write_kreg(dd, kr_ibcctrl, val);
1572 
1573 	if (!ppd->cpspec->ibcddrctrl) {
1574 		/* not on re-init after reset */
1575 		ppd->cpspec->ibcddrctrl = qib_read_kreg64(dd, kr_ibcddrctrl);
1576 
1577 		if (ppd->link_speed_enabled == (QIB_IB_SDR | QIB_IB_DDR))
1578 			ppd->cpspec->ibcddrctrl |=
1579 				IBA7220_IBC_SPEED_AUTONEG_MASK |
1580 				IBA7220_IBC_IBTA_1_2_MASK;
1581 		else
1582 			ppd->cpspec->ibcddrctrl |=
1583 				ppd->link_speed_enabled == QIB_IB_DDR ?
1584 				IBA7220_IBC_SPEED_DDR : IBA7220_IBC_SPEED_SDR;
1585 		if ((ppd->link_width_enabled & (IB_WIDTH_1X | IB_WIDTH_4X)) ==
1586 		    (IB_WIDTH_1X | IB_WIDTH_4X))
1587 			ppd->cpspec->ibcddrctrl |= IBA7220_IBC_WIDTH_AUTONEG;
1588 		else
1589 			ppd->cpspec->ibcddrctrl |=
1590 				ppd->link_width_enabled == IB_WIDTH_4X ?
1591 				IBA7220_IBC_WIDTH_4X_ONLY :
1592 				IBA7220_IBC_WIDTH_1X_ONLY;
1593 
1594 		/* always enable these on driver reload, not sticky */
1595 		ppd->cpspec->ibcddrctrl |=
1596 			IBA7220_IBC_RXPOL_MASK << IBA7220_IBC_RXPOL_SHIFT;
1597 		ppd->cpspec->ibcddrctrl |=
1598 			IBA7220_IBC_HRTBT_MASK << IBA7220_IBC_HRTBT_SHIFT;
1599 
1600 		/* enable automatic lane reversal detection for receive */
1601 		ppd->cpspec->ibcddrctrl |= IBA7220_IBC_LANE_REV_SUPPORTED;
1602 	} else
1603 		/* write to chip to prevent back-to-back writes of ibc reg */
1604 		qib_write_kreg(dd, kr_scratch, 0);
1605 
1606 	qib_write_kreg(dd, kr_ibcddrctrl, ppd->cpspec->ibcddrctrl);
1607 	qib_write_kreg(dd, kr_scratch, 0);
1608 
1609 	qib_write_kreg(dd, kr_ncmodectrl, 0Ull);
1610 	qib_write_kreg(dd, kr_scratch, 0);
1611 
1612 	ret = qib_sd7220_init(dd);
1613 
1614 	val = qib_read_kreg64(dd, kr_xgxs_cfg);
1615 	prev_val = val;
1616 	val |= QLOGIC_IB_XGXS_FC_SAFE;
1617 	if (val != prev_val) {
1618 		qib_write_kreg(dd, kr_xgxs_cfg, val);
1619 		qib_read_kreg32(dd, kr_scratch);
1620 	}
1621 	if (val & QLOGIC_IB_XGXS_RESET)
1622 		val &= ~QLOGIC_IB_XGXS_RESET;
1623 	if (val != prev_val)
1624 		qib_write_kreg(dd, kr_xgxs_cfg, val);
1625 
1626 	/* first time through, set port guid */
1627 	if (!ppd->guid)
1628 		ppd->guid = dd->base_guid;
1629 	guid = be64_to_cpu(ppd->guid);
1630 
1631 	qib_write_kreg(dd, kr_hrtbt_guid, guid);
1632 	if (!ret) {
1633 		dd->control |= QLOGIC_IB_C_LINKENABLE;
1634 		qib_write_kreg(dd, kr_control, dd->control);
1635 	} else
1636 		/* write to chip to prevent back-to-back writes of ibc reg */
1637 		qib_write_kreg(dd, kr_scratch, 0);
1638 	return ret;
1639 }
1640 
1641 /**
1642  * qib_7220_quiet_serdes - set serdes to txidle
1643  * @ppd: physical port of the qlogic_ib device
1644  * Called when driver is being unloaded
1645  */
1646 static void qib_7220_quiet_serdes(struct qib_pportdata *ppd)
1647 {
1648 	u64 val;
1649 	struct qib_devdata *dd = ppd->dd;
1650 	unsigned long flags;
1651 
1652 	/* disable IBC */
1653 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1654 	qib_write_kreg(dd, kr_control,
1655 		       dd->control | QLOGIC_IB_C_FREEZEMODE);
1656 
1657 	ppd->cpspec->chase_end = 0;
1658 	if (ppd->cpspec->chase_timer.function) /* if initted */
1659 		del_timer_sync(&ppd->cpspec->chase_timer);
1660 
1661 	if (ppd->cpspec->ibsymdelta || ppd->cpspec->iblnkerrdelta ||
1662 	    ppd->cpspec->ibdeltainprog) {
1663 		u64 diagc;
1664 
1665 		/* enable counter writes */
1666 		diagc = qib_read_kreg64(dd, kr_hwdiagctrl);
1667 		qib_write_kreg(dd, kr_hwdiagctrl,
1668 			       diagc | SYM_MASK(HwDiagCtrl, CounterWrEnable));
1669 
1670 		if (ppd->cpspec->ibsymdelta || ppd->cpspec->ibdeltainprog) {
1671 			val = read_7220_creg32(dd, cr_ibsymbolerr);
1672 			if (ppd->cpspec->ibdeltainprog)
1673 				val -= val - ppd->cpspec->ibsymsnap;
1674 			val -= ppd->cpspec->ibsymdelta;
1675 			write_7220_creg(dd, cr_ibsymbolerr, val);
1676 		}
1677 		if (ppd->cpspec->iblnkerrdelta || ppd->cpspec->ibdeltainprog) {
1678 			val = read_7220_creg32(dd, cr_iblinkerrrecov);
1679 			if (ppd->cpspec->ibdeltainprog)
1680 				val -= val - ppd->cpspec->iblnkerrsnap;
1681 			val -= ppd->cpspec->iblnkerrdelta;
1682 			write_7220_creg(dd, cr_iblinkerrrecov, val);
1683 		}
1684 
1685 		/* and disable counter writes */
1686 		qib_write_kreg(dd, kr_hwdiagctrl, diagc);
1687 	}
1688 	qib_set_ib_7220_lstate(ppd, 0, QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1689 
1690 	spin_lock_irqsave(&ppd->lflags_lock, flags);
1691 	ppd->lflags &= ~QIBL_IB_AUTONEG_INPROG;
1692 	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1693 	wake_up(&ppd->cpspec->autoneg_wait);
1694 	cancel_delayed_work_sync(&ppd->cpspec->autoneg_work);
1695 
1696 	shutdown_7220_relock_poll(ppd->dd);
1697 	val = qib_read_kreg64(ppd->dd, kr_xgxs_cfg);
1698 	val |= QLOGIC_IB_XGXS_RESET;
1699 	qib_write_kreg(ppd->dd, kr_xgxs_cfg, val);
1700 }
1701 
1702 /**
1703  * qib_setup_7220_setextled - set the state of the two external LEDs
1704  * @ppd: the qlogic_ib device
1705  * @on: whether the link is up or not
1706  *
1707  * The exact combo of LEDs if on is true is determined by looking
1708  * at the ibcstatus.
1709  *
1710  * These LEDs indicate the physical and logical state of IB link.
1711  * For this chip (at least with recommended board pinouts), LED1
1712  * is Yellow (logical state) and LED2 is Green (physical state),
1713  *
1714  * Note:  We try to match the Mellanox HCA LED behavior as best
1715  * we can.  Green indicates physical link state is OK (something is
1716  * plugged in, and we can train).
1717  * Amber indicates the link is logically up (ACTIVE).
1718  * Mellanox further blinks the amber LED to indicate data packet
1719  * activity, but we have no hardware support for that, so it would
1720  * require waking up every 10-20 msecs and checking the counters
1721  * on the chip, and then turning the LED off if appropriate.  That's
1722  * visible overhead, so not something we will do.
1723  *
1724  */
1725 static void qib_setup_7220_setextled(struct qib_pportdata *ppd, u32 on)
1726 {
1727 	struct qib_devdata *dd = ppd->dd;
1728 	u64 extctl, ledblink = 0, val, lst, ltst;
1729 	unsigned long flags;
1730 
1731 	/*
1732 	 * The diags use the LED to indicate diag info, so we leave
1733 	 * the external LED alone when the diags are running.
1734 	 */
1735 	if (dd->diag_client)
1736 		return;
1737 
1738 	if (ppd->led_override) {
1739 		ltst = (ppd->led_override & QIB_LED_PHYS) ?
1740 			IB_PHYSPORTSTATE_LINKUP : IB_PHYSPORTSTATE_DISABLED,
1741 		lst = (ppd->led_override & QIB_LED_LOG) ?
1742 			IB_PORT_ACTIVE : IB_PORT_DOWN;
1743 	} else if (on) {
1744 		val = qib_read_kreg64(dd, kr_ibcstatus);
1745 		ltst = qib_7220_phys_portstate(val);
1746 		lst = qib_7220_iblink_state(val);
1747 	} else {
1748 		ltst = 0;
1749 		lst = 0;
1750 	}
1751 
1752 	spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
1753 	extctl = dd->cspec->extctrl & ~(SYM_MASK(EXTCtrl, LEDPriPortGreenOn) |
1754 				 SYM_MASK(EXTCtrl, LEDPriPortYellowOn));
1755 	if (ltst == IB_PHYSPORTSTATE_LINKUP) {
1756 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortGreenOn);
1757 		/*
1758 		 * counts are in chip clock (4ns) periods.
1759 		 * This is 1/16 sec (66.6ms) on,
1760 		 * 3/16 sec (187.5 ms) off, with packets rcvd
1761 		 */
1762 		ledblink = ((66600 * 1000UL / 4) << IBA7220_LEDBLINK_ON_SHIFT)
1763 			| ((187500 * 1000UL / 4) << IBA7220_LEDBLINK_OFF_SHIFT);
1764 	}
1765 	if (lst == IB_PORT_ACTIVE)
1766 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortYellowOn);
1767 	dd->cspec->extctrl = extctl;
1768 	qib_write_kreg(dd, kr_extctrl, extctl);
1769 	spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
1770 
1771 	if (ledblink) /* blink the LED on packet receive */
1772 		qib_write_kreg(dd, kr_rcvpktledcnt, ledblink);
1773 }
1774 
1775 /*
1776  * qib_setup_7220_cleanup - clean up any per-chip chip-specific stuff
1777  * @dd: the qlogic_ib device
1778  *
1779  * This is called during driver unload.
1780  *
1781  */
1782 static void qib_setup_7220_cleanup(struct qib_devdata *dd)
1783 {
1784 	qib_free_irq(dd);
1785 	kfree(dd->cspec->cntrs);
1786 	kfree(dd->cspec->portcntrs);
1787 }
1788 
1789 /*
1790  * This is only called for SDmaInt.
1791  * SDmaDisabled is handled on the error path.
1792  */
1793 static void sdma_7220_intr(struct qib_pportdata *ppd, u64 istat)
1794 {
1795 	unsigned long flags;
1796 
1797 	spin_lock_irqsave(&ppd->sdma_lock, flags);
1798 
1799 	switch (ppd->sdma_state.current_state) {
1800 	case qib_sdma_state_s00_hw_down:
1801 		break;
1802 
1803 	case qib_sdma_state_s10_hw_start_up_wait:
1804 		__qib_sdma_process_event(ppd, qib_sdma_event_e20_hw_started);
1805 		break;
1806 
1807 	case qib_sdma_state_s20_idle:
1808 		break;
1809 
1810 	case qib_sdma_state_s30_sw_clean_up_wait:
1811 		break;
1812 
1813 	case qib_sdma_state_s40_hw_clean_up_wait:
1814 		break;
1815 
1816 	case qib_sdma_state_s50_hw_halt_wait:
1817 		__qib_sdma_process_event(ppd, qib_sdma_event_e60_hw_halted);
1818 		break;
1819 
1820 	case qib_sdma_state_s99_running:
1821 		/* too chatty to print here */
1822 		__qib_sdma_intr(ppd);
1823 		break;
1824 	}
1825 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1826 }
1827 
1828 static void qib_wantpiobuf_7220_intr(struct qib_devdata *dd, u32 needint)
1829 {
1830 	unsigned long flags;
1831 
1832 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
1833 	if (needint) {
1834 		if (!(dd->sendctrl & SYM_MASK(SendCtrl, SendBufAvailUpd)))
1835 			goto done;
1836 		/*
1837 		 * blip the availupd off, next write will be on, so
1838 		 * we ensure an avail update, regardless of threshold or
1839 		 * buffers becoming free, whenever we want an interrupt
1840 		 */
1841 		qib_write_kreg(dd, kr_sendctrl, dd->sendctrl &
1842 			~SYM_MASK(SendCtrl, SendBufAvailUpd));
1843 		qib_write_kreg(dd, kr_scratch, 0ULL);
1844 		dd->sendctrl |= SYM_MASK(SendCtrl, SendIntBufAvail);
1845 	} else
1846 		dd->sendctrl &= ~SYM_MASK(SendCtrl, SendIntBufAvail);
1847 	qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
1848 	qib_write_kreg(dd, kr_scratch, 0ULL);
1849 done:
1850 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
1851 }
1852 
1853 /*
1854  * Handle errors and unusual events first, separate function
1855  * to improve cache hits for fast path interrupt handling.
1856  */
1857 static noinline void unlikely_7220_intr(struct qib_devdata *dd, u64 istat)
1858 {
1859 	if (unlikely(istat & ~QLOGIC_IB_I_BITSEXTANT))
1860 		qib_dev_err(dd,
1861 			    "interrupt with unknown interrupts %Lx set\n",
1862 			    istat & ~QLOGIC_IB_I_BITSEXTANT);
1863 
1864 	if (istat & QLOGIC_IB_I_GPIO) {
1865 		u32 gpiostatus;
1866 
1867 		/*
1868 		 * Boards for this chip currently don't use GPIO interrupts,
1869 		 * so clear by writing GPIOstatus to GPIOclear, and complain
1870 		 * to alert developer. To avoid endless repeats, clear
1871 		 * the bits in the mask, since there is some kind of
1872 		 * programming error or chip problem.
1873 		 */
1874 		gpiostatus = qib_read_kreg32(dd, kr_gpio_status);
1875 		/*
1876 		 * In theory, writing GPIOstatus to GPIOclear could
1877 		 * have a bad side-effect on some diagnostic that wanted
1878 		 * to poll for a status-change, but the various shadows
1879 		 * make that problematic at best. Diags will just suppress
1880 		 * all GPIO interrupts during such tests.
1881 		 */
1882 		qib_write_kreg(dd, kr_gpio_clear, gpiostatus);
1883 
1884 		if (gpiostatus) {
1885 			const u32 mask = qib_read_kreg32(dd, kr_gpio_mask);
1886 			u32 gpio_irq = mask & gpiostatus;
1887 
1888 			/*
1889 			 * A bit set in status and (chip) Mask register
1890 			 * would cause an interrupt. Since we are not
1891 			 * expecting any, report it. Also check that the
1892 			 * chip reflects our shadow, report issues,
1893 			 * and refresh from the shadow.
1894 			 */
1895 			/*
1896 			 * Clear any troublemakers, and update chip
1897 			 * from shadow
1898 			 */
1899 			dd->cspec->gpio_mask &= ~gpio_irq;
1900 			qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1901 		}
1902 	}
1903 
1904 	if (istat & QLOGIC_IB_I_ERROR) {
1905 		u64 estat;
1906 
1907 		qib_stats.sps_errints++;
1908 		estat = qib_read_kreg64(dd, kr_errstatus);
1909 		if (!estat)
1910 			qib_devinfo(dd->pcidev,
1911 				"error interrupt (%Lx), but no error bits set!\n",
1912 				istat);
1913 		else
1914 			handle_7220_errors(dd, estat);
1915 	}
1916 }
1917 
1918 static irqreturn_t qib_7220intr(int irq, void *data)
1919 {
1920 	struct qib_devdata *dd = data;
1921 	irqreturn_t ret;
1922 	u64 istat;
1923 	u64 ctxtrbits;
1924 	u64 rmask;
1925 	unsigned i;
1926 
1927 	if ((dd->flags & (QIB_PRESENT | QIB_BADINTR)) != QIB_PRESENT) {
1928 		/*
1929 		 * This return value is not great, but we do not want the
1930 		 * interrupt core code to remove our interrupt handler
1931 		 * because we don't appear to be handling an interrupt
1932 		 * during a chip reset.
1933 		 */
1934 		ret = IRQ_HANDLED;
1935 		goto bail;
1936 	}
1937 
1938 	istat = qib_read_kreg64(dd, kr_intstatus);
1939 
1940 	if (unlikely(!istat)) {
1941 		ret = IRQ_NONE; /* not our interrupt, or already handled */
1942 		goto bail;
1943 	}
1944 	if (unlikely(istat == -1)) {
1945 		qib_bad_intrstatus(dd);
1946 		/* don't know if it was our interrupt or not */
1947 		ret = IRQ_NONE;
1948 		goto bail;
1949 	}
1950 
1951 	this_cpu_inc(*dd->int_counter);
1952 	if (unlikely(istat & (~QLOGIC_IB_I_BITSEXTANT |
1953 			      QLOGIC_IB_I_GPIO | QLOGIC_IB_I_ERROR)))
1954 		unlikely_7220_intr(dd, istat);
1955 
1956 	/*
1957 	 * Clear the interrupt bits we found set, relatively early, so we
1958 	 * "know" know the chip will have seen this by the time we process
1959 	 * the queue, and will re-interrupt if necessary.  The processor
1960 	 * itself won't take the interrupt again until we return.
1961 	 */
1962 	qib_write_kreg(dd, kr_intclear, istat);
1963 
1964 	/*
1965 	 * Handle kernel receive queues before checking for pio buffers
1966 	 * available since receives can overflow; piobuf waiters can afford
1967 	 * a few extra cycles, since they were waiting anyway.
1968 	 */
1969 	ctxtrbits = istat &
1970 		((QLOGIC_IB_I_RCVAVAIL_MASK << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1971 		 (QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT));
1972 	if (ctxtrbits) {
1973 		rmask = (1ULL << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1974 			(1ULL << QLOGIC_IB_I_RCVURG_SHIFT);
1975 		for (i = 0; i < dd->first_user_ctxt; i++) {
1976 			if (ctxtrbits & rmask) {
1977 				ctxtrbits &= ~rmask;
1978 				qib_kreceive(dd->rcd[i], NULL, NULL);
1979 			}
1980 			rmask <<= 1;
1981 		}
1982 		if (ctxtrbits) {
1983 			ctxtrbits =
1984 				(ctxtrbits >> QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1985 				(ctxtrbits >> QLOGIC_IB_I_RCVURG_SHIFT);
1986 			qib_handle_urcv(dd, ctxtrbits);
1987 		}
1988 	}
1989 
1990 	/* only call for SDmaInt */
1991 	if (istat & QLOGIC_IB_I_SDMAINT)
1992 		sdma_7220_intr(dd->pport, istat);
1993 
1994 	if ((istat & QLOGIC_IB_I_SPIOBUFAVAIL) && (dd->flags & QIB_INITTED))
1995 		qib_ib_piobufavail(dd);
1996 
1997 	ret = IRQ_HANDLED;
1998 bail:
1999 	return ret;
2000 }
2001 
2002 /*
2003  * Set up our chip-specific interrupt handler.
2004  * The interrupt type has already been setup, so
2005  * we just need to do the registration and error checking.
2006  * If we are using MSI interrupts, we may fall back to
2007  * INTx later, if the interrupt handler doesn't get called
2008  * within 1/2 second (see verify_interrupt()).
2009  */
2010 static void qib_setup_7220_interrupt(struct qib_devdata *dd)
2011 {
2012 	int ret;
2013 
2014 	ret = pci_request_irq(dd->pcidev, 0, qib_7220intr, NULL, dd,
2015 			      QIB_DRV_NAME);
2016 	if (ret)
2017 		qib_dev_err(dd, "Couldn't setup %s interrupt (irq=%d): %d\n",
2018 			    dd->pcidev->msi_enabled ?  "MSI" : "INTx",
2019 			    pci_irq_vector(dd->pcidev, 0), ret);
2020 }
2021 
2022 /**
2023  * qib_7220_boardname - fill in the board name
2024  * @dd: the qlogic_ib device
2025  *
2026  * info is based on the board revision register
2027  */
2028 static void qib_7220_boardname(struct qib_devdata *dd)
2029 {
2030 	u32 boardid;
2031 
2032 	boardid = SYM_FIELD(dd->revision, Revision,
2033 			    BoardID);
2034 
2035 	switch (boardid) {
2036 	case 1:
2037 		dd->boardname = "InfiniPath_QLE7240";
2038 		break;
2039 	case 2:
2040 		dd->boardname = "InfiniPath_QLE7280";
2041 		break;
2042 	default:
2043 		qib_dev_err(dd, "Unknown 7220 board with ID %u\n", boardid);
2044 		dd->boardname = "Unknown_InfiniPath_7220";
2045 		break;
2046 	}
2047 
2048 	if (dd->majrev != 5 || !dd->minrev || dd->minrev > 2)
2049 		qib_dev_err(dd,
2050 			    "Unsupported InfiniPath hardware revision %u.%u!\n",
2051 			    dd->majrev, dd->minrev);
2052 
2053 	snprintf(dd->boardversion, sizeof(dd->boardversion),
2054 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
2055 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
2056 		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, Arch),
2057 		 dd->majrev, dd->minrev,
2058 		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, SW));
2059 }
2060 
2061 /*
2062  * This routine sleeps, so it can only be called from user context, not
2063  * from interrupt context.
2064  */
2065 static int qib_setup_7220_reset(struct qib_devdata *dd)
2066 {
2067 	u64 val;
2068 	int i;
2069 	int ret;
2070 	u16 cmdval;
2071 	u8 int_line, clinesz;
2072 	unsigned long flags;
2073 
2074 	qib_pcie_getcmd(dd, &cmdval, &int_line, &clinesz);
2075 
2076 	/* Use dev_err so it shows up in logs, etc. */
2077 	qib_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->unit);
2078 
2079 	/* no interrupts till re-initted */
2080 	qib_7220_set_intr_state(dd, 0);
2081 
2082 	dd->pport->cpspec->ibdeltainprog = 0;
2083 	dd->pport->cpspec->ibsymdelta = 0;
2084 	dd->pport->cpspec->iblnkerrdelta = 0;
2085 
2086 	/*
2087 	 * Keep chip from being accessed until we are ready.  Use
2088 	 * writeq() directly, to allow the write even though QIB_PRESENT
2089 	 * isn't set.
2090 	 */
2091 	dd->flags &= ~(QIB_INITTED | QIB_PRESENT);
2092 	/* so we check interrupts work again */
2093 	dd->z_int_counter = qib_int_counter(dd);
2094 	val = dd->control | QLOGIC_IB_C_RESET;
2095 	writeq(val, &dd->kregbase[kr_control]);
2096 	mb(); /* prevent compiler reordering around actual reset */
2097 
2098 	for (i = 1; i <= 5; i++) {
2099 		/*
2100 		 * Allow MBIST, etc. to complete; longer on each retry.
2101 		 * We sometimes get machine checks from bus timeout if no
2102 		 * response, so for now, make it *really* long.
2103 		 */
2104 		msleep(1000 + (1 + i) * 2000);
2105 
2106 		qib_pcie_reenable(dd, cmdval, int_line, clinesz);
2107 
2108 		/*
2109 		 * Use readq directly, so we don't need to mark it as PRESENT
2110 		 * until we get a successful indication that all is well.
2111 		 */
2112 		val = readq(&dd->kregbase[kr_revision]);
2113 		if (val == dd->revision) {
2114 			dd->flags |= QIB_PRESENT; /* it's back */
2115 			ret = qib_reinit_intr(dd);
2116 			goto bail;
2117 		}
2118 	}
2119 	ret = 0; /* failed */
2120 
2121 bail:
2122 	if (ret) {
2123 		if (qib_pcie_params(dd, dd->lbus_width, NULL))
2124 			qib_dev_err(dd,
2125 				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
2126 
2127 		/* hold IBC in reset, no sends, etc till later */
2128 		qib_write_kreg(dd, kr_control, 0ULL);
2129 
2130 		/* clear the reset error, init error/hwerror mask */
2131 		qib_7220_init_hwerrors(dd);
2132 
2133 		/* do setup similar to speed or link-width changes */
2134 		if (dd->pport->cpspec->ibcddrctrl & IBA7220_IBC_IBTA_1_2_MASK)
2135 			dd->cspec->presets_needed = 1;
2136 		spin_lock_irqsave(&dd->pport->lflags_lock, flags);
2137 		dd->pport->lflags |= QIBL_IB_FORCE_NOTIFY;
2138 		dd->pport->lflags &= ~QIBL_IB_AUTONEG_FAILED;
2139 		spin_unlock_irqrestore(&dd->pport->lflags_lock, flags);
2140 	}
2141 
2142 	return ret;
2143 }
2144 
2145 /**
2146  * qib_7220_put_tid - write a TID to the chip
2147  * @dd: the qlogic_ib device
2148  * @tidptr: pointer to the expected TID (in chip) to update
2149  * @type: 0 for eager, 1 for expected
2150  * @pa: physical address of in memory buffer; tidinvalid if freeing
2151  */
2152 static void qib_7220_put_tid(struct qib_devdata *dd, u64 __iomem *tidptr,
2153 			     u32 type, unsigned long pa)
2154 {
2155 	if (pa != dd->tidinvalid) {
2156 		u64 chippa = pa >> IBA7220_TID_PA_SHIFT;
2157 
2158 		/* paranoia checks */
2159 		if (pa != (chippa << IBA7220_TID_PA_SHIFT)) {
2160 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
2161 				    pa);
2162 			return;
2163 		}
2164 		if (chippa >= (1UL << IBA7220_TID_SZ_SHIFT)) {
2165 			qib_dev_err(dd,
2166 				"Physical page address 0x%lx larger than supported\n",
2167 				pa);
2168 			return;
2169 		}
2170 
2171 		if (type == RCVHQ_RCV_TYPE_EAGER)
2172 			chippa |= dd->tidtemplate;
2173 		else /* for now, always full 4KB page */
2174 			chippa |= IBA7220_TID_SZ_4K;
2175 		pa = chippa;
2176 	}
2177 	writeq(pa, tidptr);
2178 }
2179 
2180 /**
2181  * qib_7220_clear_tids - clear all TID entries for a ctxt, expected and eager
2182  * @dd: the qlogic_ib device
2183  * @rcd: the ctxt
2184  *
2185  * clear all TID entries for a ctxt, expected and eager.
2186  * Used from qib_close().  On this chip, TIDs are only 32 bits,
2187  * not 64, but they are still on 64 bit boundaries, so tidbase
2188  * is declared as u64 * for the pointer math, even though we write 32 bits
2189  */
2190 static void qib_7220_clear_tids(struct qib_devdata *dd,
2191 				struct qib_ctxtdata *rcd)
2192 {
2193 	u64 __iomem *tidbase;
2194 	unsigned long tidinv;
2195 	u32 ctxt;
2196 	int i;
2197 
2198 	if (!dd->kregbase || !rcd)
2199 		return;
2200 
2201 	ctxt = rcd->ctxt;
2202 
2203 	tidinv = dd->tidinvalid;
2204 	tidbase = (u64 __iomem *)
2205 		((char __iomem *)(dd->kregbase) +
2206 		 dd->rcvtidbase +
2207 		 ctxt * dd->rcvtidcnt * sizeof(*tidbase));
2208 
2209 	for (i = 0; i < dd->rcvtidcnt; i++)
2210 		qib_7220_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
2211 				 tidinv);
2212 
2213 	tidbase = (u64 __iomem *)
2214 		((char __iomem *)(dd->kregbase) +
2215 		 dd->rcvegrbase +
2216 		 rcd->rcvegr_tid_base * sizeof(*tidbase));
2217 
2218 	for (i = 0; i < rcd->rcvegrcnt; i++)
2219 		qib_7220_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EAGER,
2220 				 tidinv);
2221 }
2222 
2223 /**
2224  * qib_7220_tidtemplate - setup constants for TID updates
2225  * @dd: the qlogic_ib device
2226  *
2227  * We setup stuff that we use a lot, to avoid calculating each time
2228  */
2229 static void qib_7220_tidtemplate(struct qib_devdata *dd)
2230 {
2231 	if (dd->rcvegrbufsize == 2048)
2232 		dd->tidtemplate = IBA7220_TID_SZ_2K;
2233 	else if (dd->rcvegrbufsize == 4096)
2234 		dd->tidtemplate = IBA7220_TID_SZ_4K;
2235 	dd->tidinvalid = 0;
2236 }
2237 
2238 /**
2239  * qib_7220_get_base_info - set chip-specific flags for user code
2240  * @rcd: the qlogic_ib ctxt
2241  * @kinfo: qib_base_info pointer
2242  *
2243  * We set the PCIE flag because the lower bandwidth on PCIe vs
2244  * HyperTransport can affect some user packet algorithims.
2245  */
2246 static int qib_7220_get_base_info(struct qib_ctxtdata *rcd,
2247 				  struct qib_base_info *kinfo)
2248 {
2249 	kinfo->spi_runtime_flags |= QIB_RUNTIME_PCIE |
2250 		QIB_RUNTIME_NODMA_RTAIL | QIB_RUNTIME_SDMA;
2251 
2252 	if (rcd->dd->flags & QIB_USE_SPCL_TRIG)
2253 		kinfo->spi_runtime_flags |= QIB_RUNTIME_SPECIAL_TRIGGER;
2254 
2255 	return 0;
2256 }
2257 
2258 static struct qib_message_header *
2259 qib_7220_get_msgheader(struct qib_devdata *dd, __le32 *rhf_addr)
2260 {
2261 	u32 offset = qib_hdrget_offset(rhf_addr);
2262 
2263 	return (struct qib_message_header *)
2264 		(rhf_addr - dd->rhf_offset + offset);
2265 }
2266 
2267 static void qib_7220_config_ctxts(struct qib_devdata *dd)
2268 {
2269 	unsigned long flags;
2270 	u32 nchipctxts;
2271 
2272 	nchipctxts = qib_read_kreg32(dd, kr_portcnt);
2273 	dd->cspec->numctxts = nchipctxts;
2274 	if (qib_n_krcv_queues > 1) {
2275 		dd->qpn_mask = 0x3e;
2276 		dd->first_user_ctxt = qib_n_krcv_queues * dd->num_pports;
2277 		if (dd->first_user_ctxt > nchipctxts)
2278 			dd->first_user_ctxt = nchipctxts;
2279 	} else
2280 		dd->first_user_ctxt = dd->num_pports;
2281 	dd->n_krcv_queues = dd->first_user_ctxt;
2282 
2283 	if (!qib_cfgctxts) {
2284 		int nctxts = dd->first_user_ctxt + num_online_cpus();
2285 
2286 		if (nctxts <= 5)
2287 			dd->ctxtcnt = 5;
2288 		else if (nctxts <= 9)
2289 			dd->ctxtcnt = 9;
2290 		else if (nctxts <= nchipctxts)
2291 			dd->ctxtcnt = nchipctxts;
2292 	} else if (qib_cfgctxts <= nchipctxts)
2293 		dd->ctxtcnt = qib_cfgctxts;
2294 	if (!dd->ctxtcnt) /* none of the above, set to max */
2295 		dd->ctxtcnt = nchipctxts;
2296 
2297 	/*
2298 	 * Chip can be configured for 5, 9, or 17 ctxts, and choice
2299 	 * affects number of eager TIDs per ctxt (1K, 2K, 4K).
2300 	 * Lock to be paranoid about later motion, etc.
2301 	 */
2302 	spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2303 	if (dd->ctxtcnt > 9)
2304 		dd->rcvctrl |= 2ULL << IBA7220_R_CTXTCFG_SHIFT;
2305 	else if (dd->ctxtcnt > 5)
2306 		dd->rcvctrl |= 1ULL << IBA7220_R_CTXTCFG_SHIFT;
2307 	/* else configure for default 5 receive ctxts */
2308 	if (dd->qpn_mask)
2309 		dd->rcvctrl |= 1ULL << QIB_7220_RcvCtrl_RcvQPMapEnable_LSB;
2310 	qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2311 	spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2312 
2313 	/* kr_rcvegrcnt changes based on the number of contexts enabled */
2314 	dd->cspec->rcvegrcnt = qib_read_kreg32(dd, kr_rcvegrcnt);
2315 	dd->rcvhdrcnt = max(dd->cspec->rcvegrcnt, IBA7220_KRCVEGRCNT);
2316 }
2317 
2318 static int qib_7220_get_ib_cfg(struct qib_pportdata *ppd, int which)
2319 {
2320 	int lsb, ret = 0;
2321 	u64 maskr; /* right-justified mask */
2322 
2323 	switch (which) {
2324 	case QIB_IB_CFG_LWID_ENB: /* Get allowed Link-width */
2325 		ret = ppd->link_width_enabled;
2326 		goto done;
2327 
2328 	case QIB_IB_CFG_LWID: /* Get currently active Link-width */
2329 		ret = ppd->link_width_active;
2330 		goto done;
2331 
2332 	case QIB_IB_CFG_SPD_ENB: /* Get allowed Link speeds */
2333 		ret = ppd->link_speed_enabled;
2334 		goto done;
2335 
2336 	case QIB_IB_CFG_SPD: /* Get current Link spd */
2337 		ret = ppd->link_speed_active;
2338 		goto done;
2339 
2340 	case QIB_IB_CFG_RXPOL_ENB: /* Get Auto-RX-polarity enable */
2341 		lsb = IBA7220_IBC_RXPOL_SHIFT;
2342 		maskr = IBA7220_IBC_RXPOL_MASK;
2343 		break;
2344 
2345 	case QIB_IB_CFG_LREV_ENB: /* Get Auto-Lane-reversal enable */
2346 		lsb = IBA7220_IBC_LREV_SHIFT;
2347 		maskr = IBA7220_IBC_LREV_MASK;
2348 		break;
2349 
2350 	case QIB_IB_CFG_LINKLATENCY:
2351 		ret = qib_read_kreg64(ppd->dd, kr_ibcddrstatus)
2352 			& IBA7220_DDRSTAT_LINKLAT_MASK;
2353 		goto done;
2354 
2355 	case QIB_IB_CFG_OP_VLS:
2356 		ret = ppd->vls_operational;
2357 		goto done;
2358 
2359 	case QIB_IB_CFG_VL_HIGH_CAP:
2360 		ret = 0;
2361 		goto done;
2362 
2363 	case QIB_IB_CFG_VL_LOW_CAP:
2364 		ret = 0;
2365 		goto done;
2366 
2367 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2368 		ret = SYM_FIELD(ppd->cpspec->ibcctrl, IBCCtrl,
2369 				OverrunThreshold);
2370 		goto done;
2371 
2372 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2373 		ret = SYM_FIELD(ppd->cpspec->ibcctrl, IBCCtrl,
2374 				PhyerrThreshold);
2375 		goto done;
2376 
2377 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2378 		/* will only take effect when the link state changes */
2379 		ret = (ppd->cpspec->ibcctrl &
2380 		       SYM_MASK(IBCCtrl, LinkDownDefaultState)) ?
2381 			IB_LINKINITCMD_SLEEP : IB_LINKINITCMD_POLL;
2382 		goto done;
2383 
2384 	case QIB_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2385 		lsb = IBA7220_IBC_HRTBT_SHIFT;
2386 		maskr = IBA7220_IBC_HRTBT_MASK;
2387 		break;
2388 
2389 	case QIB_IB_CFG_PMA_TICKS:
2390 		/*
2391 		 * 0x00 = 10x link transfer rate or 4 nsec. for 2.5Gbs
2392 		 * Since the clock is always 250MHz, the value is 1 or 0.
2393 		 */
2394 		ret = (ppd->link_speed_active == QIB_IB_DDR);
2395 		goto done;
2396 
2397 	default:
2398 		ret = -EINVAL;
2399 		goto done;
2400 	}
2401 	ret = (int)((ppd->cpspec->ibcddrctrl >> lsb) & maskr);
2402 done:
2403 	return ret;
2404 }
2405 
2406 static int qib_7220_set_ib_cfg(struct qib_pportdata *ppd, int which, u32 val)
2407 {
2408 	struct qib_devdata *dd = ppd->dd;
2409 	u64 maskr; /* right-justified mask */
2410 	int lsb, ret = 0, setforce = 0;
2411 	u16 lcmd, licmd;
2412 	unsigned long flags;
2413 	u32 tmp = 0;
2414 
2415 	switch (which) {
2416 	case QIB_IB_CFG_LIDLMC:
2417 		/*
2418 		 * Set LID and LMC. Combined to avoid possible hazard
2419 		 * caller puts LMC in 16MSbits, DLID in 16LSbits of val
2420 		 */
2421 		lsb = IBA7220_IBC_DLIDLMC_SHIFT;
2422 		maskr = IBA7220_IBC_DLIDLMC_MASK;
2423 		break;
2424 
2425 	case QIB_IB_CFG_LWID_ENB: /* set allowed Link-width */
2426 		/*
2427 		 * As with speed, only write the actual register if
2428 		 * the link is currently down, otherwise takes effect
2429 		 * on next link change.
2430 		 */
2431 		ppd->link_width_enabled = val;
2432 		if (!(ppd->lflags & QIBL_LINKDOWN))
2433 			goto bail;
2434 		/*
2435 		 * We set the QIBL_IB_FORCE_NOTIFY bit so updown
2436 		 * will get called because we want update
2437 		 * link_width_active, and the change may not take
2438 		 * effect for some time (if we are in POLL), so this
2439 		 * flag will force the updown routine to be called
2440 		 * on the next ibstatuschange down interrupt, even
2441 		 * if it's not an down->up transition.
2442 		 */
2443 		val--; /* convert from IB to chip */
2444 		maskr = IBA7220_IBC_WIDTH_MASK;
2445 		lsb = IBA7220_IBC_WIDTH_SHIFT;
2446 		setforce = 1;
2447 		break;
2448 
2449 	case QIB_IB_CFG_SPD_ENB: /* set allowed Link speeds */
2450 		/*
2451 		 * If we turn off IB1.2, need to preset SerDes defaults,
2452 		 * but not right now. Set a flag for the next time
2453 		 * we command the link down.  As with width, only write the
2454 		 * actual register if the link is currently down, otherwise
2455 		 * takes effect on next link change.  Since setting is being
2456 		 * explicitly requested (via MAD or sysfs), clear autoneg
2457 		 * failure status if speed autoneg is enabled.
2458 		 */
2459 		ppd->link_speed_enabled = val;
2460 		if ((ppd->cpspec->ibcddrctrl & IBA7220_IBC_IBTA_1_2_MASK) &&
2461 		    !(val & (val - 1)))
2462 			dd->cspec->presets_needed = 1;
2463 		if (!(ppd->lflags & QIBL_LINKDOWN))
2464 			goto bail;
2465 		/*
2466 		 * We set the QIBL_IB_FORCE_NOTIFY bit so updown
2467 		 * will get called because we want update
2468 		 * link_speed_active, and the change may not take
2469 		 * effect for some time (if we are in POLL), so this
2470 		 * flag will force the updown routine to be called
2471 		 * on the next ibstatuschange down interrupt, even
2472 		 * if it's not an down->up transition.
2473 		 */
2474 		if (val == (QIB_IB_SDR | QIB_IB_DDR)) {
2475 			val = IBA7220_IBC_SPEED_AUTONEG_MASK |
2476 				IBA7220_IBC_IBTA_1_2_MASK;
2477 			spin_lock_irqsave(&ppd->lflags_lock, flags);
2478 			ppd->lflags &= ~QIBL_IB_AUTONEG_FAILED;
2479 			spin_unlock_irqrestore(&ppd->lflags_lock, flags);
2480 		} else
2481 			val = val == QIB_IB_DDR ?
2482 				IBA7220_IBC_SPEED_DDR : IBA7220_IBC_SPEED_SDR;
2483 		maskr = IBA7220_IBC_SPEED_AUTONEG_MASK |
2484 			IBA7220_IBC_IBTA_1_2_MASK;
2485 		/* IBTA 1.2 mode + speed bits are contiguous */
2486 		lsb = SYM_LSB(IBCDDRCtrl, IB_ENHANCED_MODE);
2487 		setforce = 1;
2488 		break;
2489 
2490 	case QIB_IB_CFG_RXPOL_ENB: /* set Auto-RX-polarity enable */
2491 		lsb = IBA7220_IBC_RXPOL_SHIFT;
2492 		maskr = IBA7220_IBC_RXPOL_MASK;
2493 		break;
2494 
2495 	case QIB_IB_CFG_LREV_ENB: /* set Auto-Lane-reversal enable */
2496 		lsb = IBA7220_IBC_LREV_SHIFT;
2497 		maskr = IBA7220_IBC_LREV_MASK;
2498 		break;
2499 
2500 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2501 		maskr = SYM_FIELD(ppd->cpspec->ibcctrl, IBCCtrl,
2502 				  OverrunThreshold);
2503 		if (maskr != val) {
2504 			ppd->cpspec->ibcctrl &=
2505 				~SYM_MASK(IBCCtrl, OverrunThreshold);
2506 			ppd->cpspec->ibcctrl |= (u64) val <<
2507 				SYM_LSB(IBCCtrl, OverrunThreshold);
2508 			qib_write_kreg(dd, kr_ibcctrl, ppd->cpspec->ibcctrl);
2509 			qib_write_kreg(dd, kr_scratch, 0);
2510 		}
2511 		goto bail;
2512 
2513 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2514 		maskr = SYM_FIELD(ppd->cpspec->ibcctrl, IBCCtrl,
2515 				  PhyerrThreshold);
2516 		if (maskr != val) {
2517 			ppd->cpspec->ibcctrl &=
2518 				~SYM_MASK(IBCCtrl, PhyerrThreshold);
2519 			ppd->cpspec->ibcctrl |= (u64) val <<
2520 				SYM_LSB(IBCCtrl, PhyerrThreshold);
2521 			qib_write_kreg(dd, kr_ibcctrl, ppd->cpspec->ibcctrl);
2522 			qib_write_kreg(dd, kr_scratch, 0);
2523 		}
2524 		goto bail;
2525 
2526 	case QIB_IB_CFG_PKEYS: /* update pkeys */
2527 		maskr = (u64) ppd->pkeys[0] | ((u64) ppd->pkeys[1] << 16) |
2528 			((u64) ppd->pkeys[2] << 32) |
2529 			((u64) ppd->pkeys[3] << 48);
2530 		qib_write_kreg(dd, kr_partitionkey, maskr);
2531 		goto bail;
2532 
2533 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2534 		/* will only take effect when the link state changes */
2535 		if (val == IB_LINKINITCMD_POLL)
2536 			ppd->cpspec->ibcctrl &=
2537 				~SYM_MASK(IBCCtrl, LinkDownDefaultState);
2538 		else /* SLEEP */
2539 			ppd->cpspec->ibcctrl |=
2540 				SYM_MASK(IBCCtrl, LinkDownDefaultState);
2541 		qib_write_kreg(dd, kr_ibcctrl, ppd->cpspec->ibcctrl);
2542 		qib_write_kreg(dd, kr_scratch, 0);
2543 		goto bail;
2544 
2545 	case QIB_IB_CFG_MTU: /* update the MTU in IBC */
2546 		/*
2547 		 * Update our housekeeping variables, and set IBC max
2548 		 * size, same as init code; max IBC is max we allow in
2549 		 * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2550 		 * Set even if it's unchanged, print debug message only
2551 		 * on changes.
2552 		 */
2553 		val = (ppd->ibmaxlen >> 2) + 1;
2554 		ppd->cpspec->ibcctrl &= ~SYM_MASK(IBCCtrl, MaxPktLen);
2555 		ppd->cpspec->ibcctrl |= (u64)val << SYM_LSB(IBCCtrl, MaxPktLen);
2556 		qib_write_kreg(dd, kr_ibcctrl, ppd->cpspec->ibcctrl);
2557 		qib_write_kreg(dd, kr_scratch, 0);
2558 		goto bail;
2559 
2560 	case QIB_IB_CFG_LSTATE: /* set the IB link state */
2561 		switch (val & 0xffff0000) {
2562 		case IB_LINKCMD_DOWN:
2563 			lcmd = QLOGIC_IB_IBCC_LINKCMD_DOWN;
2564 			if (!ppd->cpspec->ibdeltainprog &&
2565 			    qib_compat_ddr_negotiate) {
2566 				ppd->cpspec->ibdeltainprog = 1;
2567 				ppd->cpspec->ibsymsnap =
2568 					read_7220_creg32(dd, cr_ibsymbolerr);
2569 				ppd->cpspec->iblnkerrsnap =
2570 					read_7220_creg32(dd, cr_iblinkerrrecov);
2571 			}
2572 			break;
2573 
2574 		case IB_LINKCMD_ARMED:
2575 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ARMED;
2576 			break;
2577 
2578 		case IB_LINKCMD_ACTIVE:
2579 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ACTIVE;
2580 			break;
2581 
2582 		default:
2583 			ret = -EINVAL;
2584 			qib_dev_err(dd, "bad linkcmd req 0x%x\n", val >> 16);
2585 			goto bail;
2586 		}
2587 		switch (val & 0xffff) {
2588 		case IB_LINKINITCMD_NOP:
2589 			licmd = 0;
2590 			break;
2591 
2592 		case IB_LINKINITCMD_POLL:
2593 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_POLL;
2594 			break;
2595 
2596 		case IB_LINKINITCMD_SLEEP:
2597 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_SLEEP;
2598 			break;
2599 
2600 		case IB_LINKINITCMD_DISABLE:
2601 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_DISABLE;
2602 			ppd->cpspec->chase_end = 0;
2603 			/*
2604 			 * stop state chase counter and timer, if running.
2605 			 * wait forpending timer, but don't clear .data (ppd)!
2606 			 */
2607 			if (ppd->cpspec->chase_timer.expires) {
2608 				del_timer_sync(&ppd->cpspec->chase_timer);
2609 				ppd->cpspec->chase_timer.expires = 0;
2610 			}
2611 			break;
2612 
2613 		default:
2614 			ret = -EINVAL;
2615 			qib_dev_err(dd, "bad linkinitcmd req 0x%x\n",
2616 				    val & 0xffff);
2617 			goto bail;
2618 		}
2619 		qib_set_ib_7220_lstate(ppd, lcmd, licmd);
2620 
2621 		maskr = IBA7220_IBC_WIDTH_MASK;
2622 		lsb = IBA7220_IBC_WIDTH_SHIFT;
2623 		tmp = (ppd->cpspec->ibcddrctrl >> lsb) & maskr;
2624 		/* If the width active on the chip does not match the
2625 		 * width in the shadow register, write the new active
2626 		 * width to the chip.
2627 		 * We don't have to worry about speed as the speed is taken
2628 		 * care of by set_7220_ibspeed_fast called by ib_updown.
2629 		 */
2630 		if (ppd->link_width_enabled-1 != tmp) {
2631 			ppd->cpspec->ibcddrctrl &= ~(maskr << lsb);
2632 			ppd->cpspec->ibcddrctrl |=
2633 				(((u64)(ppd->link_width_enabled-1) & maskr) <<
2634 				 lsb);
2635 			qib_write_kreg(dd, kr_ibcddrctrl,
2636 				       ppd->cpspec->ibcddrctrl);
2637 			qib_write_kreg(dd, kr_scratch, 0);
2638 			spin_lock_irqsave(&ppd->lflags_lock, flags);
2639 			ppd->lflags |= QIBL_IB_FORCE_NOTIFY;
2640 			spin_unlock_irqrestore(&ppd->lflags_lock, flags);
2641 		}
2642 		goto bail;
2643 
2644 	case QIB_IB_CFG_HRTBT: /* set Heartbeat off/enable/auto */
2645 		if (val > IBA7220_IBC_HRTBT_MASK) {
2646 			ret = -EINVAL;
2647 			goto bail;
2648 		}
2649 		lsb = IBA7220_IBC_HRTBT_SHIFT;
2650 		maskr = IBA7220_IBC_HRTBT_MASK;
2651 		break;
2652 
2653 	default:
2654 		ret = -EINVAL;
2655 		goto bail;
2656 	}
2657 	ppd->cpspec->ibcddrctrl &= ~(maskr << lsb);
2658 	ppd->cpspec->ibcddrctrl |= (((u64) val & maskr) << lsb);
2659 	qib_write_kreg(dd, kr_ibcddrctrl, ppd->cpspec->ibcddrctrl);
2660 	qib_write_kreg(dd, kr_scratch, 0);
2661 	if (setforce) {
2662 		spin_lock_irqsave(&ppd->lflags_lock, flags);
2663 		ppd->lflags |= QIBL_IB_FORCE_NOTIFY;
2664 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
2665 	}
2666 bail:
2667 	return ret;
2668 }
2669 
2670 static int qib_7220_set_loopback(struct qib_pportdata *ppd, const char *what)
2671 {
2672 	int ret = 0;
2673 	u64 val, ddr;
2674 
2675 	if (!strncmp(what, "ibc", 3)) {
2676 		ppd->cpspec->ibcctrl |= SYM_MASK(IBCCtrl, Loopback);
2677 		val = 0; /* disable heart beat, so link will come up */
2678 		qib_devinfo(ppd->dd->pcidev, "Enabling IB%u:%u IBC loopback\n",
2679 			 ppd->dd->unit, ppd->port);
2680 	} else if (!strncmp(what, "off", 3)) {
2681 		ppd->cpspec->ibcctrl &= ~SYM_MASK(IBCCtrl, Loopback);
2682 		/* enable heart beat again */
2683 		val = IBA7220_IBC_HRTBT_MASK << IBA7220_IBC_HRTBT_SHIFT;
2684 		qib_devinfo(ppd->dd->pcidev,
2685 			"Disabling IB%u:%u IBC loopback (normal)\n",
2686 			ppd->dd->unit, ppd->port);
2687 	} else
2688 		ret = -EINVAL;
2689 	if (!ret) {
2690 		qib_write_kreg(ppd->dd, kr_ibcctrl, ppd->cpspec->ibcctrl);
2691 		ddr = ppd->cpspec->ibcddrctrl & ~(IBA7220_IBC_HRTBT_MASK
2692 					     << IBA7220_IBC_HRTBT_SHIFT);
2693 		ppd->cpspec->ibcddrctrl = ddr | val;
2694 		qib_write_kreg(ppd->dd, kr_ibcddrctrl,
2695 			       ppd->cpspec->ibcddrctrl);
2696 		qib_write_kreg(ppd->dd, kr_scratch, 0);
2697 	}
2698 	return ret;
2699 }
2700 
2701 static void qib_update_7220_usrhead(struct qib_ctxtdata *rcd, u64 hd,
2702 				    u32 updegr, u32 egrhd, u32 npkts)
2703 {
2704 	if (updegr)
2705 		qib_write_ureg(rcd->dd, ur_rcvegrindexhead, egrhd, rcd->ctxt);
2706 	qib_write_ureg(rcd->dd, ur_rcvhdrhead, hd, rcd->ctxt);
2707 }
2708 
2709 static u32 qib_7220_hdrqempty(struct qib_ctxtdata *rcd)
2710 {
2711 	u32 head, tail;
2712 
2713 	head = qib_read_ureg32(rcd->dd, ur_rcvhdrhead, rcd->ctxt);
2714 	if (rcd->rcvhdrtail_kvaddr)
2715 		tail = qib_get_rcvhdrtail(rcd);
2716 	else
2717 		tail = qib_read_ureg32(rcd->dd, ur_rcvhdrtail, rcd->ctxt);
2718 	return head == tail;
2719 }
2720 
2721 /*
2722  * Modify the RCVCTRL register in chip-specific way. This
2723  * is a function because bit positions and (future) register
2724  * location is chip-specifc, but the needed operations are
2725  * generic. <op> is a bit-mask because we often want to
2726  * do multiple modifications.
2727  */
2728 static void rcvctrl_7220_mod(struct qib_pportdata *ppd, unsigned int op,
2729 			     int ctxt)
2730 {
2731 	struct qib_devdata *dd = ppd->dd;
2732 	u64 mask, val;
2733 	unsigned long flags;
2734 
2735 	spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2736 	if (op & QIB_RCVCTRL_TAILUPD_ENB)
2737 		dd->rcvctrl |= (1ULL << IBA7220_R_TAILUPD_SHIFT);
2738 	if (op & QIB_RCVCTRL_TAILUPD_DIS)
2739 		dd->rcvctrl &= ~(1ULL << IBA7220_R_TAILUPD_SHIFT);
2740 	if (op & QIB_RCVCTRL_PKEY_ENB)
2741 		dd->rcvctrl &= ~(1ULL << IBA7220_R_PKEY_DIS_SHIFT);
2742 	if (op & QIB_RCVCTRL_PKEY_DIS)
2743 		dd->rcvctrl |= (1ULL << IBA7220_R_PKEY_DIS_SHIFT);
2744 	if (ctxt < 0)
2745 		mask = (1ULL << dd->ctxtcnt) - 1;
2746 	else
2747 		mask = (1ULL << ctxt);
2748 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2749 		/* always done for specific ctxt */
2750 		dd->rcvctrl |= (mask << SYM_LSB(RcvCtrl, PortEnable));
2751 		if (!(dd->flags & QIB_NODMA_RTAIL))
2752 			dd->rcvctrl |= 1ULL << IBA7220_R_TAILUPD_SHIFT;
2753 		/* Write these registers before the context is enabled. */
2754 		qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2755 			dd->rcd[ctxt]->rcvhdrqtailaddr_phys);
2756 		qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2757 			dd->rcd[ctxt]->rcvhdrq_phys);
2758 		dd->rcd[ctxt]->seq_cnt = 1;
2759 	}
2760 	if (op & QIB_RCVCTRL_CTXT_DIS)
2761 		dd->rcvctrl &= ~(mask << SYM_LSB(RcvCtrl, PortEnable));
2762 	if (op & QIB_RCVCTRL_INTRAVAIL_ENB)
2763 		dd->rcvctrl |= (mask << IBA7220_R_INTRAVAIL_SHIFT);
2764 	if (op & QIB_RCVCTRL_INTRAVAIL_DIS)
2765 		dd->rcvctrl &= ~(mask << IBA7220_R_INTRAVAIL_SHIFT);
2766 	qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2767 	if ((op & QIB_RCVCTRL_INTRAVAIL_ENB) && dd->rhdrhead_intr_off) {
2768 		/* arm rcv interrupt */
2769 		val = qib_read_ureg32(dd, ur_rcvhdrhead, ctxt) |
2770 			dd->rhdrhead_intr_off;
2771 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2772 	}
2773 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2774 		/*
2775 		 * Init the context registers also; if we were
2776 		 * disabled, tail and head should both be zero
2777 		 * already from the enable, but since we don't
2778 		 * know, we have to do it explicitly.
2779 		 */
2780 		val = qib_read_ureg32(dd, ur_rcvegrindextail, ctxt);
2781 		qib_write_ureg(dd, ur_rcvegrindexhead, val, ctxt);
2782 
2783 		val = qib_read_ureg32(dd, ur_rcvhdrtail, ctxt);
2784 		dd->rcd[ctxt]->head = val;
2785 		/* If kctxt, interrupt on next receive. */
2786 		if (ctxt < dd->first_user_ctxt)
2787 			val |= dd->rhdrhead_intr_off;
2788 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2789 	}
2790 	if (op & QIB_RCVCTRL_CTXT_DIS) {
2791 		if (ctxt >= 0) {
2792 			qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt, 0);
2793 			qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt, 0);
2794 		} else {
2795 			unsigned i;
2796 
2797 			for (i = 0; i < dd->cfgctxts; i++) {
2798 				qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr,
2799 						    i, 0);
2800 				qib_write_kreg_ctxt(dd, kr_rcvhdraddr, i, 0);
2801 			}
2802 		}
2803 	}
2804 	spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2805 }
2806 
2807 /*
2808  * Modify the SENDCTRL register in chip-specific way. This
2809  * is a function there may be multiple such registers with
2810  * slightly different layouts. To start, we assume the
2811  * "canonical" register layout of the first chips.
2812  * Chip requires no back-back sendctrl writes, so write
2813  * scratch register after writing sendctrl
2814  */
2815 static void sendctrl_7220_mod(struct qib_pportdata *ppd, u32 op)
2816 {
2817 	struct qib_devdata *dd = ppd->dd;
2818 	u64 tmp_dd_sendctrl;
2819 	unsigned long flags;
2820 
2821 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
2822 
2823 	/* First the ones that are "sticky", saved in shadow */
2824 	if (op & QIB_SENDCTRL_CLEAR)
2825 		dd->sendctrl = 0;
2826 	if (op & QIB_SENDCTRL_SEND_DIS)
2827 		dd->sendctrl &= ~SYM_MASK(SendCtrl, SPioEnable);
2828 	else if (op & QIB_SENDCTRL_SEND_ENB) {
2829 		dd->sendctrl |= SYM_MASK(SendCtrl, SPioEnable);
2830 		if (dd->flags & QIB_USE_SPCL_TRIG)
2831 			dd->sendctrl |= SYM_MASK(SendCtrl,
2832 						 SSpecialTriggerEn);
2833 	}
2834 	if (op & QIB_SENDCTRL_AVAIL_DIS)
2835 		dd->sendctrl &= ~SYM_MASK(SendCtrl, SendBufAvailUpd);
2836 	else if (op & QIB_SENDCTRL_AVAIL_ENB)
2837 		dd->sendctrl |= SYM_MASK(SendCtrl, SendBufAvailUpd);
2838 
2839 	if (op & QIB_SENDCTRL_DISARM_ALL) {
2840 		u32 i, last;
2841 
2842 		tmp_dd_sendctrl = dd->sendctrl;
2843 		/*
2844 		 * disarm any that are not yet launched, disabling sends
2845 		 * and updates until done.
2846 		 */
2847 		last = dd->piobcnt2k + dd->piobcnt4k;
2848 		tmp_dd_sendctrl &=
2849 			~(SYM_MASK(SendCtrl, SPioEnable) |
2850 			  SYM_MASK(SendCtrl, SendBufAvailUpd));
2851 		for (i = 0; i < last; i++) {
2852 			qib_write_kreg(dd, kr_sendctrl,
2853 				       tmp_dd_sendctrl |
2854 				       SYM_MASK(SendCtrl, Disarm) | i);
2855 			qib_write_kreg(dd, kr_scratch, 0);
2856 		}
2857 	}
2858 
2859 	tmp_dd_sendctrl = dd->sendctrl;
2860 
2861 	if (op & QIB_SENDCTRL_FLUSH)
2862 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Abort);
2863 	if (op & QIB_SENDCTRL_DISARM)
2864 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Disarm) |
2865 			((op & QIB_7220_SendCtrl_DisarmPIOBuf_RMASK) <<
2866 			 SYM_LSB(SendCtrl, DisarmPIOBuf));
2867 	if ((op & QIB_SENDCTRL_AVAIL_BLIP) &&
2868 	    (dd->sendctrl & SYM_MASK(SendCtrl, SendBufAvailUpd)))
2869 		tmp_dd_sendctrl &= ~SYM_MASK(SendCtrl, SendBufAvailUpd);
2870 
2871 	qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl);
2872 	qib_write_kreg(dd, kr_scratch, 0);
2873 
2874 	if (op & QIB_SENDCTRL_AVAIL_BLIP) {
2875 		qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
2876 		qib_write_kreg(dd, kr_scratch, 0);
2877 	}
2878 
2879 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
2880 
2881 	if (op & QIB_SENDCTRL_FLUSH) {
2882 		u32 v;
2883 		/*
2884 		 * ensure writes have hit chip, then do a few
2885 		 * more reads, to allow DMA of pioavail registers
2886 		 * to occur, so in-memory copy is in sync with
2887 		 * the chip.  Not always safe to sleep.
2888 		 */
2889 		v = qib_read_kreg32(dd, kr_scratch);
2890 		qib_write_kreg(dd, kr_scratch, v);
2891 		v = qib_read_kreg32(dd, kr_scratch);
2892 		qib_write_kreg(dd, kr_scratch, v);
2893 		qib_read_kreg32(dd, kr_scratch);
2894 	}
2895 }
2896 
2897 /**
2898  * qib_portcntr_7220 - read a per-port counter
2899  * @ppd: the qlogic_ib device
2900  * @reg: the counter to snapshot
2901  */
2902 static u64 qib_portcntr_7220(struct qib_pportdata *ppd, u32 reg)
2903 {
2904 	u64 ret = 0ULL;
2905 	struct qib_devdata *dd = ppd->dd;
2906 	u16 creg;
2907 	/* 0xffff for unimplemented or synthesized counters */
2908 	static const u16 xlator[] = {
2909 		[QIBPORTCNTR_PKTSEND] = cr_pktsend,
2910 		[QIBPORTCNTR_WORDSEND] = cr_wordsend,
2911 		[QIBPORTCNTR_PSXMITDATA] = cr_psxmitdatacount,
2912 		[QIBPORTCNTR_PSXMITPKTS] = cr_psxmitpktscount,
2913 		[QIBPORTCNTR_PSXMITWAIT] = cr_psxmitwaitcount,
2914 		[QIBPORTCNTR_SENDSTALL] = cr_sendstall,
2915 		[QIBPORTCNTR_PKTRCV] = cr_pktrcv,
2916 		[QIBPORTCNTR_PSRCVDATA] = cr_psrcvdatacount,
2917 		[QIBPORTCNTR_PSRCVPKTS] = cr_psrcvpktscount,
2918 		[QIBPORTCNTR_RCVEBP] = cr_rcvebp,
2919 		[QIBPORTCNTR_RCVOVFL] = cr_rcvovfl,
2920 		[QIBPORTCNTR_WORDRCV] = cr_wordrcv,
2921 		[QIBPORTCNTR_RXDROPPKT] = cr_rxdroppkt,
2922 		[QIBPORTCNTR_RXLOCALPHYERR] = cr_rxotherlocalphyerr,
2923 		[QIBPORTCNTR_RXVLERR] = cr_rxvlerr,
2924 		[QIBPORTCNTR_ERRICRC] = cr_erricrc,
2925 		[QIBPORTCNTR_ERRVCRC] = cr_errvcrc,
2926 		[QIBPORTCNTR_ERRLPCRC] = cr_errlpcrc,
2927 		[QIBPORTCNTR_BADFORMAT] = cr_badformat,
2928 		[QIBPORTCNTR_ERR_RLEN] = cr_err_rlen,
2929 		[QIBPORTCNTR_IBSYMBOLERR] = cr_ibsymbolerr,
2930 		[QIBPORTCNTR_INVALIDRLEN] = cr_invalidrlen,
2931 		[QIBPORTCNTR_UNSUPVL] = cr_txunsupvl,
2932 		[QIBPORTCNTR_EXCESSBUFOVFL] = cr_excessbufferovfl,
2933 		[QIBPORTCNTR_ERRLINK] = cr_errlink,
2934 		[QIBPORTCNTR_IBLINKDOWN] = cr_iblinkdown,
2935 		[QIBPORTCNTR_IBLINKERRRECOV] = cr_iblinkerrrecov,
2936 		[QIBPORTCNTR_LLI] = cr_locallinkintegrityerr,
2937 		[QIBPORTCNTR_PSINTERVAL] = cr_psinterval,
2938 		[QIBPORTCNTR_PSSTART] = cr_psstart,
2939 		[QIBPORTCNTR_PSSTAT] = cr_psstat,
2940 		[QIBPORTCNTR_VL15PKTDROP] = cr_vl15droppedpkt,
2941 		[QIBPORTCNTR_ERRPKEY] = cr_errpkey,
2942 		[QIBPORTCNTR_KHDROVFL] = 0xffff,
2943 	};
2944 
2945 	if (reg >= ARRAY_SIZE(xlator)) {
2946 		qib_devinfo(ppd->dd->pcidev,
2947 			 "Unimplemented portcounter %u\n", reg);
2948 		goto done;
2949 	}
2950 	creg = xlator[reg];
2951 
2952 	if (reg == QIBPORTCNTR_KHDROVFL) {
2953 		int i;
2954 
2955 		/* sum over all kernel contexts */
2956 		for (i = 0; i < dd->first_user_ctxt; i++)
2957 			ret += read_7220_creg32(dd, cr_portovfl + i);
2958 	}
2959 	if (creg == 0xffff)
2960 		goto done;
2961 
2962 	/*
2963 	 * only fast incrementing counters are 64bit; use 32 bit reads to
2964 	 * avoid two independent reads when on opteron
2965 	 */
2966 	if ((creg == cr_wordsend || creg == cr_wordrcv ||
2967 	     creg == cr_pktsend || creg == cr_pktrcv))
2968 		ret = read_7220_creg(dd, creg);
2969 	else
2970 		ret = read_7220_creg32(dd, creg);
2971 	if (creg == cr_ibsymbolerr) {
2972 		if (dd->pport->cpspec->ibdeltainprog)
2973 			ret -= ret - ppd->cpspec->ibsymsnap;
2974 		ret -= dd->pport->cpspec->ibsymdelta;
2975 	} else if (creg == cr_iblinkerrrecov) {
2976 		if (dd->pport->cpspec->ibdeltainprog)
2977 			ret -= ret - ppd->cpspec->iblnkerrsnap;
2978 		ret -= dd->pport->cpspec->iblnkerrdelta;
2979 	}
2980 done:
2981 	return ret;
2982 }
2983 
2984 /*
2985  * Device counter names (not port-specific), one line per stat,
2986  * single string.  Used by utilities like ipathstats to print the stats
2987  * in a way which works for different versions of drivers, without changing
2988  * the utility.  Names need to be 12 chars or less (w/o newline), for proper
2989  * display by utility.
2990  * Non-error counters are first.
2991  * Start of "error" counters is indicated by a leading "E " on the first
2992  * "error" counter, and doesn't count in label length.
2993  * The EgrOvfl list needs to be last so we truncate them at the configured
2994  * context count for the device.
2995  * cntr7220indices contains the corresponding register indices.
2996  */
2997 static const char cntr7220names[] =
2998 	"Interrupts\n"
2999 	"HostBusStall\n"
3000 	"E RxTIDFull\n"
3001 	"RxTIDInvalid\n"
3002 	"Ctxt0EgrOvfl\n"
3003 	"Ctxt1EgrOvfl\n"
3004 	"Ctxt2EgrOvfl\n"
3005 	"Ctxt3EgrOvfl\n"
3006 	"Ctxt4EgrOvfl\n"
3007 	"Ctxt5EgrOvfl\n"
3008 	"Ctxt6EgrOvfl\n"
3009 	"Ctxt7EgrOvfl\n"
3010 	"Ctxt8EgrOvfl\n"
3011 	"Ctxt9EgrOvfl\n"
3012 	"Ctx10EgrOvfl\n"
3013 	"Ctx11EgrOvfl\n"
3014 	"Ctx12EgrOvfl\n"
3015 	"Ctx13EgrOvfl\n"
3016 	"Ctx14EgrOvfl\n"
3017 	"Ctx15EgrOvfl\n"
3018 	"Ctx16EgrOvfl\n";
3019 
3020 static const size_t cntr7220indices[] = {
3021 	cr_lbint,
3022 	cr_lbflowstall,
3023 	cr_errtidfull,
3024 	cr_errtidvalid,
3025 	cr_portovfl + 0,
3026 	cr_portovfl + 1,
3027 	cr_portovfl + 2,
3028 	cr_portovfl + 3,
3029 	cr_portovfl + 4,
3030 	cr_portovfl + 5,
3031 	cr_portovfl + 6,
3032 	cr_portovfl + 7,
3033 	cr_portovfl + 8,
3034 	cr_portovfl + 9,
3035 	cr_portovfl + 10,
3036 	cr_portovfl + 11,
3037 	cr_portovfl + 12,
3038 	cr_portovfl + 13,
3039 	cr_portovfl + 14,
3040 	cr_portovfl + 15,
3041 	cr_portovfl + 16,
3042 };
3043 
3044 /*
3045  * same as cntr7220names and cntr7220indices, but for port-specific counters.
3046  * portcntr7220indices is somewhat complicated by some registers needing
3047  * adjustments of various kinds, and those are ORed with _PORT_VIRT_FLAG
3048  */
3049 static const char portcntr7220names[] =
3050 	"TxPkt\n"
3051 	"TxFlowPkt\n"
3052 	"TxWords\n"
3053 	"RxPkt\n"
3054 	"RxFlowPkt\n"
3055 	"RxWords\n"
3056 	"TxFlowStall\n"
3057 	"TxDmaDesc\n"  /* 7220 and 7322-only */
3058 	"E RxDlidFltr\n"  /* 7220 and 7322-only */
3059 	"IBStatusChng\n"
3060 	"IBLinkDown\n"
3061 	"IBLnkRecov\n"
3062 	"IBRxLinkErr\n"
3063 	"IBSymbolErr\n"
3064 	"RxLLIErr\n"
3065 	"RxBadFormat\n"
3066 	"RxBadLen\n"
3067 	"RxBufOvrfl\n"
3068 	"RxEBP\n"
3069 	"RxFlowCtlErr\n"
3070 	"RxICRCerr\n"
3071 	"RxLPCRCerr\n"
3072 	"RxVCRCerr\n"
3073 	"RxInvalLen\n"
3074 	"RxInvalPKey\n"
3075 	"RxPktDropped\n"
3076 	"TxBadLength\n"
3077 	"TxDropped\n"
3078 	"TxInvalLen\n"
3079 	"TxUnderrun\n"
3080 	"TxUnsupVL\n"
3081 	"RxLclPhyErr\n" /* 7220 and 7322-only */
3082 	"RxVL15Drop\n" /* 7220 and 7322-only */
3083 	"RxVlErr\n" /* 7220 and 7322-only */
3084 	"XcessBufOvfl\n" /* 7220 and 7322-only */
3085 	;
3086 
3087 #define _PORT_VIRT_FLAG 0x8000 /* "virtual", need adjustments */
3088 static const size_t portcntr7220indices[] = {
3089 	QIBPORTCNTR_PKTSEND | _PORT_VIRT_FLAG,
3090 	cr_pktsendflow,
3091 	QIBPORTCNTR_WORDSEND | _PORT_VIRT_FLAG,
3092 	QIBPORTCNTR_PKTRCV | _PORT_VIRT_FLAG,
3093 	cr_pktrcvflowctrl,
3094 	QIBPORTCNTR_WORDRCV | _PORT_VIRT_FLAG,
3095 	QIBPORTCNTR_SENDSTALL | _PORT_VIRT_FLAG,
3096 	cr_txsdmadesc,
3097 	cr_rxdlidfltr,
3098 	cr_ibstatuschange,
3099 	QIBPORTCNTR_IBLINKDOWN | _PORT_VIRT_FLAG,
3100 	QIBPORTCNTR_IBLINKERRRECOV | _PORT_VIRT_FLAG,
3101 	QIBPORTCNTR_ERRLINK | _PORT_VIRT_FLAG,
3102 	QIBPORTCNTR_IBSYMBOLERR | _PORT_VIRT_FLAG,
3103 	QIBPORTCNTR_LLI | _PORT_VIRT_FLAG,
3104 	QIBPORTCNTR_BADFORMAT | _PORT_VIRT_FLAG,
3105 	QIBPORTCNTR_ERR_RLEN | _PORT_VIRT_FLAG,
3106 	QIBPORTCNTR_RCVOVFL | _PORT_VIRT_FLAG,
3107 	QIBPORTCNTR_RCVEBP | _PORT_VIRT_FLAG,
3108 	cr_rcvflowctrl_err,
3109 	QIBPORTCNTR_ERRICRC | _PORT_VIRT_FLAG,
3110 	QIBPORTCNTR_ERRLPCRC | _PORT_VIRT_FLAG,
3111 	QIBPORTCNTR_ERRVCRC | _PORT_VIRT_FLAG,
3112 	QIBPORTCNTR_INVALIDRLEN | _PORT_VIRT_FLAG,
3113 	QIBPORTCNTR_ERRPKEY | _PORT_VIRT_FLAG,
3114 	QIBPORTCNTR_RXDROPPKT | _PORT_VIRT_FLAG,
3115 	cr_invalidslen,
3116 	cr_senddropped,
3117 	cr_errslen,
3118 	cr_sendunderrun,
3119 	cr_txunsupvl,
3120 	QIBPORTCNTR_RXLOCALPHYERR | _PORT_VIRT_FLAG,
3121 	QIBPORTCNTR_VL15PKTDROP | _PORT_VIRT_FLAG,
3122 	QIBPORTCNTR_RXVLERR | _PORT_VIRT_FLAG,
3123 	QIBPORTCNTR_EXCESSBUFOVFL | _PORT_VIRT_FLAG,
3124 };
3125 
3126 /* do all the setup to make the counter reads efficient later */
3127 static void init_7220_cntrnames(struct qib_devdata *dd)
3128 {
3129 	int i, j = 0;
3130 	char *s;
3131 
3132 	for (i = 0, s = (char *)cntr7220names; s && j <= dd->cfgctxts;
3133 	     i++) {
3134 		/* we always have at least one counter before the egrovfl */
3135 		if (!j && !strncmp("Ctxt0EgrOvfl", s + 1, 12))
3136 			j = 1;
3137 		s = strchr(s + 1, '\n');
3138 		if (s && j)
3139 			j++;
3140 	}
3141 	dd->cspec->ncntrs = i;
3142 	if (!s)
3143 		/* full list; size is without terminating null */
3144 		dd->cspec->cntrnamelen = sizeof(cntr7220names) - 1;
3145 	else
3146 		dd->cspec->cntrnamelen = 1 + s - cntr7220names;
3147 	dd->cspec->cntrs = kmalloc_array(dd->cspec->ncntrs, sizeof(u64),
3148 					 GFP_KERNEL);
3149 
3150 	for (i = 0, s = (char *)portcntr7220names; s; i++)
3151 		s = strchr(s + 1, '\n');
3152 	dd->cspec->nportcntrs = i - 1;
3153 	dd->cspec->portcntrnamelen = sizeof(portcntr7220names) - 1;
3154 	dd->cspec->portcntrs = kmalloc_array(dd->cspec->nportcntrs,
3155 					     sizeof(u64),
3156 					     GFP_KERNEL);
3157 }
3158 
3159 static u32 qib_read_7220cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
3160 			      u64 **cntrp)
3161 {
3162 	u32 ret;
3163 
3164 	if (!dd->cspec->cntrs) {
3165 		ret = 0;
3166 		goto done;
3167 	}
3168 
3169 	if (namep) {
3170 		*namep = (char *)cntr7220names;
3171 		ret = dd->cspec->cntrnamelen;
3172 		if (pos >= ret)
3173 			ret = 0; /* final read after getting everything */
3174 	} else {
3175 		u64 *cntr = dd->cspec->cntrs;
3176 		int i;
3177 
3178 		ret = dd->cspec->ncntrs * sizeof(u64);
3179 		if (!cntr || pos >= ret) {
3180 			/* everything read, or couldn't get memory */
3181 			ret = 0;
3182 			goto done;
3183 		}
3184 
3185 		*cntrp = cntr;
3186 		for (i = 0; i < dd->cspec->ncntrs; i++)
3187 			*cntr++ = read_7220_creg32(dd, cntr7220indices[i]);
3188 	}
3189 done:
3190 	return ret;
3191 }
3192 
3193 static u32 qib_read_7220portcntrs(struct qib_devdata *dd, loff_t pos, u32 port,
3194 				  char **namep, u64 **cntrp)
3195 {
3196 	u32 ret;
3197 
3198 	if (!dd->cspec->portcntrs) {
3199 		ret = 0;
3200 		goto done;
3201 	}
3202 	if (namep) {
3203 		*namep = (char *)portcntr7220names;
3204 		ret = dd->cspec->portcntrnamelen;
3205 		if (pos >= ret)
3206 			ret = 0; /* final read after getting everything */
3207 	} else {
3208 		u64 *cntr = dd->cspec->portcntrs;
3209 		struct qib_pportdata *ppd = &dd->pport[port];
3210 		int i;
3211 
3212 		ret = dd->cspec->nportcntrs * sizeof(u64);
3213 		if (!cntr || pos >= ret) {
3214 			/* everything read, or couldn't get memory */
3215 			ret = 0;
3216 			goto done;
3217 		}
3218 		*cntrp = cntr;
3219 		for (i = 0; i < dd->cspec->nportcntrs; i++) {
3220 			if (portcntr7220indices[i] & _PORT_VIRT_FLAG)
3221 				*cntr++ = qib_portcntr_7220(ppd,
3222 					portcntr7220indices[i] &
3223 					~_PORT_VIRT_FLAG);
3224 			else
3225 				*cntr++ = read_7220_creg32(dd,
3226 					   portcntr7220indices[i]);
3227 		}
3228 	}
3229 done:
3230 	return ret;
3231 }
3232 
3233 /**
3234  * qib_get_7220_faststats - get word counters from chip before they overflow
3235  * @t: contains a pointer to the qlogic_ib device qib_devdata
3236  *
3237  * This needs more work; in particular, decision on whether we really
3238  * need traffic_wds done the way it is
3239  * called from add_timer
3240  */
3241 static void qib_get_7220_faststats(struct timer_list *t)
3242 {
3243 	struct qib_devdata *dd = from_timer(dd, t, stats_timer);
3244 	struct qib_pportdata *ppd = dd->pport;
3245 	unsigned long flags;
3246 	u64 traffic_wds;
3247 
3248 	/*
3249 	 * don't access the chip while running diags, or memory diags can
3250 	 * fail
3251 	 */
3252 	if (!(dd->flags & QIB_INITTED) || dd->diag_client)
3253 		/* but re-arm the timer, for diags case; won't hurt other */
3254 		goto done;
3255 
3256 	/*
3257 	 * We now try to maintain an activity timer, based on traffic
3258 	 * exceeding a threshold, so we need to check the word-counts
3259 	 * even if they are 64-bit.
3260 	 */
3261 	traffic_wds = qib_portcntr_7220(ppd, cr_wordsend) +
3262 		qib_portcntr_7220(ppd, cr_wordrcv);
3263 	spin_lock_irqsave(&dd->eep_st_lock, flags);
3264 	traffic_wds -= dd->traffic_wds;
3265 	dd->traffic_wds += traffic_wds;
3266 	spin_unlock_irqrestore(&dd->eep_st_lock, flags);
3267 done:
3268 	mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
3269 }
3270 
3271 /*
3272  * If we are using MSI, try to fallback to INTx.
3273  */
3274 static int qib_7220_intr_fallback(struct qib_devdata *dd)
3275 {
3276 	if (!dd->msi_lo)
3277 		return 0;
3278 
3279 	qib_devinfo(dd->pcidev,
3280 		    "MSI interrupt not detected, trying INTx interrupts\n");
3281 
3282 	qib_free_irq(dd);
3283 	dd->msi_lo = 0;
3284 	if (pci_alloc_irq_vectors(dd->pcidev, 1, 1, PCI_IRQ_LEGACY) < 0)
3285 		qib_dev_err(dd, "Failed to enable INTx\n");
3286 	qib_setup_7220_interrupt(dd);
3287 	return 1;
3288 }
3289 
3290 /*
3291  * Reset the XGXS (between serdes and IBC).  Slightly less intrusive
3292  * than resetting the IBC or external link state, and useful in some
3293  * cases to cause some retraining.  To do this right, we reset IBC
3294  * as well.
3295  */
3296 static void qib_7220_xgxs_reset(struct qib_pportdata *ppd)
3297 {
3298 	u64 val, prev_val;
3299 	struct qib_devdata *dd = ppd->dd;
3300 
3301 	prev_val = qib_read_kreg64(dd, kr_xgxs_cfg);
3302 	val = prev_val | QLOGIC_IB_XGXS_RESET;
3303 	prev_val &= ~QLOGIC_IB_XGXS_RESET; /* be sure */
3304 	qib_write_kreg(dd, kr_control,
3305 		       dd->control & ~QLOGIC_IB_C_LINKENABLE);
3306 	qib_write_kreg(dd, kr_xgxs_cfg, val);
3307 	qib_read_kreg32(dd, kr_scratch);
3308 	qib_write_kreg(dd, kr_xgxs_cfg, prev_val);
3309 	qib_write_kreg(dd, kr_control, dd->control);
3310 }
3311 
3312 /*
3313  * For this chip, we want to use the same buffer every time
3314  * when we are trying to bring the link up (they are always VL15
3315  * packets).  At that link state the packet should always go out immediately
3316  * (or at least be discarded at the tx interface if the link is down).
3317  * If it doesn't, and the buffer isn't available, that means some other
3318  * sender has gotten ahead of us, and is preventing our packet from going
3319  * out.  In that case, we flush all packets, and try again.  If that still
3320  * fails, we fail the request, and hope things work the next time around.
3321  *
3322  * We don't need very complicated heuristics on whether the packet had
3323  * time to go out or not, since even at SDR 1X, it goes out in very short
3324  * time periods, covered by the chip reads done here and as part of the
3325  * flush.
3326  */
3327 static u32 __iomem *get_7220_link_buf(struct qib_pportdata *ppd, u32 *bnum)
3328 {
3329 	u32 __iomem *buf;
3330 	u32 lbuf = ppd->dd->cspec->lastbuf_for_pio;
3331 	int do_cleanup;
3332 	unsigned long flags;
3333 
3334 	/*
3335 	 * always blip to get avail list updated, since it's almost
3336 	 * always needed, and is fairly cheap.
3337 	 */
3338 	sendctrl_7220_mod(ppd->dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
3339 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3340 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3341 	if (buf)
3342 		goto done;
3343 
3344 	spin_lock_irqsave(&ppd->sdma_lock, flags);
3345 	if (ppd->sdma_state.current_state == qib_sdma_state_s20_idle &&
3346 	    ppd->sdma_state.current_state != qib_sdma_state_s00_hw_down) {
3347 		__qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down);
3348 		do_cleanup = 0;
3349 	} else {
3350 		do_cleanup = 1;
3351 		qib_7220_sdma_hw_clean_up(ppd);
3352 	}
3353 	spin_unlock_irqrestore(&ppd->sdma_lock, flags);
3354 
3355 	if (do_cleanup) {
3356 		qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3357 		buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3358 	}
3359 done:
3360 	return buf;
3361 }
3362 
3363 /*
3364  * This code for non-IBTA-compliant IB speed negotiation is only known to
3365  * work for the SDR to DDR transition, and only between an HCA and a switch
3366  * with recent firmware.  It is based on observed heuristics, rather than
3367  * actual knowledge of the non-compliant speed negotiation.
3368  * It has a number of hard-coded fields, since the hope is to rewrite this
3369  * when a spec is available on how the negoation is intended to work.
3370  */
3371 static void autoneg_7220_sendpkt(struct qib_pportdata *ppd, u32 *hdr,
3372 				 u32 dcnt, u32 *data)
3373 {
3374 	int i;
3375 	u64 pbc;
3376 	u32 __iomem *piobuf;
3377 	u32 pnum;
3378 	struct qib_devdata *dd = ppd->dd;
3379 
3380 	i = 0;
3381 	pbc = 7 + dcnt + 1; /* 7 dword header, dword data, icrc */
3382 	pbc |= PBC_7220_VL15_SEND;
3383 	while (!(piobuf = get_7220_link_buf(ppd, &pnum))) {
3384 		if (i++ > 5)
3385 			return;
3386 		udelay(2);
3387 	}
3388 	sendctrl_7220_mod(dd->pport, QIB_SENDCTRL_DISARM_BUF(pnum));
3389 	writeq(pbc, piobuf);
3390 	qib_flush_wc();
3391 	qib_pio_copy(piobuf + 2, hdr, 7);
3392 	qib_pio_copy(piobuf + 9, data, dcnt);
3393 	if (dd->flags & QIB_USE_SPCL_TRIG) {
3394 		u32 spcl_off = (pnum >= dd->piobcnt2k) ? 2047 : 1023;
3395 
3396 		qib_flush_wc();
3397 		__raw_writel(0xaebecede, piobuf + spcl_off);
3398 	}
3399 	qib_flush_wc();
3400 	qib_sendbuf_done(dd, pnum);
3401 }
3402 
3403 /*
3404  * _start packet gets sent twice at start, _done gets sent twice at end
3405  */
3406 static void autoneg_7220_send(struct qib_pportdata *ppd, int which)
3407 {
3408 	struct qib_devdata *dd = ppd->dd;
3409 	static u32 swapped;
3410 	u32 dw, i, hcnt, dcnt, *data;
3411 	static u32 hdr[7] = { 0xf002ffff, 0x48ffff, 0x6400abba };
3412 	static u32 madpayload_start[0x40] = {
3413 		0x1810103, 0x1, 0x0, 0x0, 0x2c90000, 0x2c9, 0x0, 0x0,
3414 		0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
3415 		0x1, 0x1388, 0x15e, 0x1, /* rest 0's */
3416 		};
3417 	static u32 madpayload_done[0x40] = {
3418 		0x1810103, 0x1, 0x0, 0x0, 0x2c90000, 0x2c9, 0x0, 0x0,
3419 		0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
3420 		0x40000001, 0x1388, 0x15e, /* rest 0's */
3421 		};
3422 
3423 	dcnt = ARRAY_SIZE(madpayload_start);
3424 	hcnt = ARRAY_SIZE(hdr);
3425 	if (!swapped) {
3426 		/* for maintainability, do it at runtime */
3427 		for (i = 0; i < hcnt; i++) {
3428 			dw = (__force u32) cpu_to_be32(hdr[i]);
3429 			hdr[i] = dw;
3430 		}
3431 		for (i = 0; i < dcnt; i++) {
3432 			dw = (__force u32) cpu_to_be32(madpayload_start[i]);
3433 			madpayload_start[i] = dw;
3434 			dw = (__force u32) cpu_to_be32(madpayload_done[i]);
3435 			madpayload_done[i] = dw;
3436 		}
3437 		swapped = 1;
3438 	}
3439 
3440 	data = which ? madpayload_done : madpayload_start;
3441 
3442 	autoneg_7220_sendpkt(ppd, hdr, dcnt, data);
3443 	qib_read_kreg64(dd, kr_scratch);
3444 	udelay(2);
3445 	autoneg_7220_sendpkt(ppd, hdr, dcnt, data);
3446 	qib_read_kreg64(dd, kr_scratch);
3447 	udelay(2);
3448 }
3449 
3450 /*
3451  * Do the absolute minimum to cause an IB speed change, and make it
3452  * ready, but don't actually trigger the change.   The caller will
3453  * do that when ready (if link is in Polling training state, it will
3454  * happen immediately, otherwise when link next goes down)
3455  *
3456  * This routine should only be used as part of the DDR autonegotation
3457  * code for devices that are not compliant with IB 1.2 (or code that
3458  * fixes things up for same).
3459  *
3460  * When link has gone down, and autoneg enabled, or autoneg has
3461  * failed and we give up until next time we set both speeds, and
3462  * then we want IBTA enabled as well as "use max enabled speed.
3463  */
3464 static void set_7220_ibspeed_fast(struct qib_pportdata *ppd, u32 speed)
3465 {
3466 	ppd->cpspec->ibcddrctrl &= ~(IBA7220_IBC_SPEED_AUTONEG_MASK |
3467 		IBA7220_IBC_IBTA_1_2_MASK);
3468 
3469 	if (speed == (QIB_IB_SDR | QIB_IB_DDR))
3470 		ppd->cpspec->ibcddrctrl |= IBA7220_IBC_SPEED_AUTONEG_MASK |
3471 			IBA7220_IBC_IBTA_1_2_MASK;
3472 	else
3473 		ppd->cpspec->ibcddrctrl |= speed == QIB_IB_DDR ?
3474 			IBA7220_IBC_SPEED_DDR : IBA7220_IBC_SPEED_SDR;
3475 
3476 	qib_write_kreg(ppd->dd, kr_ibcddrctrl, ppd->cpspec->ibcddrctrl);
3477 	qib_write_kreg(ppd->dd, kr_scratch, 0);
3478 }
3479 
3480 /*
3481  * This routine is only used when we are not talking to another
3482  * IB 1.2-compliant device that we think can do DDR.
3483  * (This includes all existing switch chips as of Oct 2007.)
3484  * 1.2-compliant devices go directly to DDR prior to reaching INIT
3485  */
3486 static void try_7220_autoneg(struct qib_pportdata *ppd)
3487 {
3488 	unsigned long flags;
3489 
3490 	/*
3491 	 * Required for older non-IB1.2 DDR switches.  Newer
3492 	 * non-IB-compliant switches don't need it, but so far,
3493 	 * aren't bothered by it either.  "Magic constant"
3494 	 */
3495 	qib_write_kreg(ppd->dd, kr_ncmodectrl, 0x3b9dc07);
3496 
3497 	spin_lock_irqsave(&ppd->lflags_lock, flags);
3498 	ppd->lflags |= QIBL_IB_AUTONEG_INPROG;
3499 	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3500 	autoneg_7220_send(ppd, 0);
3501 	set_7220_ibspeed_fast(ppd, QIB_IB_DDR);
3502 
3503 	toggle_7220_rclkrls(ppd->dd);
3504 	/* 2 msec is minimum length of a poll cycle */
3505 	queue_delayed_work(ib_wq, &ppd->cpspec->autoneg_work,
3506 			   msecs_to_jiffies(2));
3507 }
3508 
3509 /*
3510  * Handle the empirically determined mechanism for auto-negotiation
3511  * of DDR speed with switches.
3512  */
3513 static void autoneg_7220_work(struct work_struct *work)
3514 {
3515 	struct qib_pportdata *ppd;
3516 	struct qib_devdata *dd;
3517 	u32 i;
3518 	unsigned long flags;
3519 
3520 	ppd = &container_of(work, struct qib_chippport_specific,
3521 			    autoneg_work.work)->pportdata;
3522 	dd = ppd->dd;
3523 
3524 	/*
3525 	 * Busy wait for this first part, it should be at most a
3526 	 * few hundred usec, since we scheduled ourselves for 2msec.
3527 	 */
3528 	for (i = 0; i < 25; i++) {
3529 		if (SYM_FIELD(ppd->lastibcstat, IBCStatus, LinkTrainingState)
3530 		     == IB_7220_LT_STATE_POLLQUIET) {
3531 			qib_set_linkstate(ppd, QIB_IB_LINKDOWN_DISABLE);
3532 			break;
3533 		}
3534 		udelay(100);
3535 	}
3536 
3537 	if (!(ppd->lflags & QIBL_IB_AUTONEG_INPROG))
3538 		goto done; /* we got there early or told to stop */
3539 
3540 	/* we expect this to timeout */
3541 	if (wait_event_timeout(ppd->cpspec->autoneg_wait,
3542 			       !(ppd->lflags & QIBL_IB_AUTONEG_INPROG),
3543 			       msecs_to_jiffies(90)))
3544 		goto done;
3545 
3546 	toggle_7220_rclkrls(dd);
3547 
3548 	/* we expect this to timeout */
3549 	if (wait_event_timeout(ppd->cpspec->autoneg_wait,
3550 			       !(ppd->lflags & QIBL_IB_AUTONEG_INPROG),
3551 			       msecs_to_jiffies(1700)))
3552 		goto done;
3553 
3554 	set_7220_ibspeed_fast(ppd, QIB_IB_SDR);
3555 	toggle_7220_rclkrls(dd);
3556 
3557 	/*
3558 	 * Wait up to 250 msec for link to train and get to INIT at DDR;
3559 	 * this should terminate early.
3560 	 */
3561 	wait_event_timeout(ppd->cpspec->autoneg_wait,
3562 		!(ppd->lflags & QIBL_IB_AUTONEG_INPROG),
3563 		msecs_to_jiffies(250));
3564 done:
3565 	if (ppd->lflags & QIBL_IB_AUTONEG_INPROG) {
3566 		spin_lock_irqsave(&ppd->lflags_lock, flags);
3567 		ppd->lflags &= ~QIBL_IB_AUTONEG_INPROG;
3568 		if (dd->cspec->autoneg_tries == AUTONEG_TRIES) {
3569 			ppd->lflags |= QIBL_IB_AUTONEG_FAILED;
3570 			dd->cspec->autoneg_tries = 0;
3571 		}
3572 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3573 		set_7220_ibspeed_fast(ppd, ppd->link_speed_enabled);
3574 	}
3575 }
3576 
3577 static u32 qib_7220_iblink_state(u64 ibcs)
3578 {
3579 	u32 state = (u32)SYM_FIELD(ibcs, IBCStatus, LinkState);
3580 
3581 	switch (state) {
3582 	case IB_7220_L_STATE_INIT:
3583 		state = IB_PORT_INIT;
3584 		break;
3585 	case IB_7220_L_STATE_ARM:
3586 		state = IB_PORT_ARMED;
3587 		break;
3588 	case IB_7220_L_STATE_ACTIVE:
3589 	case IB_7220_L_STATE_ACT_DEFER:
3590 		state = IB_PORT_ACTIVE;
3591 		break;
3592 	default:
3593 		fallthrough;
3594 	case IB_7220_L_STATE_DOWN:
3595 		state = IB_PORT_DOWN;
3596 		break;
3597 	}
3598 	return state;
3599 }
3600 
3601 /* returns the IBTA port state, rather than the IBC link training state */
3602 static u8 qib_7220_phys_portstate(u64 ibcs)
3603 {
3604 	u8 state = (u8)SYM_FIELD(ibcs, IBCStatus, LinkTrainingState);
3605 	return qib_7220_physportstate[state];
3606 }
3607 
3608 static int qib_7220_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs)
3609 {
3610 	int ret = 0, symadj = 0;
3611 	struct qib_devdata *dd = ppd->dd;
3612 	unsigned long flags;
3613 
3614 	spin_lock_irqsave(&ppd->lflags_lock, flags);
3615 	ppd->lflags &= ~QIBL_IB_FORCE_NOTIFY;
3616 	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3617 
3618 	if (!ibup) {
3619 		/*
3620 		 * When the link goes down we don't want AEQ running, so it
3621 		 * won't interfere with IBC training, etc., and we need
3622 		 * to go back to the static SerDes preset values.
3623 		 */
3624 		if (!(ppd->lflags & (QIBL_IB_AUTONEG_FAILED |
3625 				     QIBL_IB_AUTONEG_INPROG)))
3626 			set_7220_ibspeed_fast(ppd, ppd->link_speed_enabled);
3627 		if (!(ppd->lflags & QIBL_IB_AUTONEG_INPROG)) {
3628 			qib_sd7220_presets(dd);
3629 			qib_cancel_sends(ppd); /* initial disarm, etc. */
3630 			spin_lock_irqsave(&ppd->sdma_lock, flags);
3631 			if (__qib_sdma_running(ppd))
3632 				__qib_sdma_process_event(ppd,
3633 					qib_sdma_event_e70_go_idle);
3634 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
3635 		}
3636 		/* this might better in qib_sd7220_presets() */
3637 		set_7220_relock_poll(dd, ibup);
3638 	} else {
3639 		if (qib_compat_ddr_negotiate &&
3640 		    !(ppd->lflags & (QIBL_IB_AUTONEG_FAILED |
3641 				     QIBL_IB_AUTONEG_INPROG)) &&
3642 		    ppd->link_speed_active == QIB_IB_SDR &&
3643 		    (ppd->link_speed_enabled & (QIB_IB_DDR | QIB_IB_SDR)) ==
3644 		    (QIB_IB_DDR | QIB_IB_SDR) &&
3645 		    dd->cspec->autoneg_tries < AUTONEG_TRIES) {
3646 			/* we are SDR, and DDR auto-negotiation enabled */
3647 			++dd->cspec->autoneg_tries;
3648 			if (!ppd->cpspec->ibdeltainprog) {
3649 				ppd->cpspec->ibdeltainprog = 1;
3650 				ppd->cpspec->ibsymsnap = read_7220_creg32(dd,
3651 					cr_ibsymbolerr);
3652 				ppd->cpspec->iblnkerrsnap = read_7220_creg32(dd,
3653 					cr_iblinkerrrecov);
3654 			}
3655 			try_7220_autoneg(ppd);
3656 			ret = 1; /* no other IB status change processing */
3657 		} else if ((ppd->lflags & QIBL_IB_AUTONEG_INPROG) &&
3658 			   ppd->link_speed_active == QIB_IB_SDR) {
3659 			autoneg_7220_send(ppd, 1);
3660 			set_7220_ibspeed_fast(ppd, QIB_IB_DDR);
3661 			udelay(2);
3662 			toggle_7220_rclkrls(dd);
3663 			ret = 1; /* no other IB status change processing */
3664 		} else {
3665 			if ((ppd->lflags & QIBL_IB_AUTONEG_INPROG) &&
3666 			    (ppd->link_speed_active & QIB_IB_DDR)) {
3667 				spin_lock_irqsave(&ppd->lflags_lock, flags);
3668 				ppd->lflags &= ~(QIBL_IB_AUTONEG_INPROG |
3669 						 QIBL_IB_AUTONEG_FAILED);
3670 				spin_unlock_irqrestore(&ppd->lflags_lock,
3671 						       flags);
3672 				dd->cspec->autoneg_tries = 0;
3673 				/* re-enable SDR, for next link down */
3674 				set_7220_ibspeed_fast(ppd,
3675 						      ppd->link_speed_enabled);
3676 				wake_up(&ppd->cpspec->autoneg_wait);
3677 				symadj = 1;
3678 			} else if (ppd->lflags & QIBL_IB_AUTONEG_FAILED) {
3679 				/*
3680 				 * Clear autoneg failure flag, and do setup
3681 				 * so we'll try next time link goes down and
3682 				 * back to INIT (possibly connected to a
3683 				 * different device).
3684 				 */
3685 				spin_lock_irqsave(&ppd->lflags_lock, flags);
3686 				ppd->lflags &= ~QIBL_IB_AUTONEG_FAILED;
3687 				spin_unlock_irqrestore(&ppd->lflags_lock,
3688 						       flags);
3689 				ppd->cpspec->ibcddrctrl |=
3690 					IBA7220_IBC_IBTA_1_2_MASK;
3691 				qib_write_kreg(dd, kr_ncmodectrl, 0);
3692 				symadj = 1;
3693 			}
3694 		}
3695 
3696 		if (!(ppd->lflags & QIBL_IB_AUTONEG_INPROG))
3697 			symadj = 1;
3698 
3699 		if (!ret) {
3700 			ppd->delay_mult = rate_to_delay
3701 			    [(ibcs >> IBA7220_LINKSPEED_SHIFT) & 1]
3702 			    [(ibcs >> IBA7220_LINKWIDTH_SHIFT) & 1];
3703 
3704 			set_7220_relock_poll(dd, ibup);
3705 			spin_lock_irqsave(&ppd->sdma_lock, flags);
3706 			/*
3707 			 * Unlike 7322, the 7220 needs this, due to lack of
3708 			 * interrupt in some cases when we have sdma active
3709 			 * when the link goes down.
3710 			 */
3711 			if (ppd->sdma_state.current_state !=
3712 			    qib_sdma_state_s20_idle)
3713 				__qib_sdma_process_event(ppd,
3714 					qib_sdma_event_e00_go_hw_down);
3715 			spin_unlock_irqrestore(&ppd->sdma_lock, flags);
3716 		}
3717 	}
3718 
3719 	if (symadj) {
3720 		if (ppd->cpspec->ibdeltainprog) {
3721 			ppd->cpspec->ibdeltainprog = 0;
3722 			ppd->cpspec->ibsymdelta += read_7220_creg32(ppd->dd,
3723 				cr_ibsymbolerr) - ppd->cpspec->ibsymsnap;
3724 			ppd->cpspec->iblnkerrdelta += read_7220_creg32(ppd->dd,
3725 				cr_iblinkerrrecov) - ppd->cpspec->iblnkerrsnap;
3726 		}
3727 	} else if (!ibup && qib_compat_ddr_negotiate &&
3728 		   !ppd->cpspec->ibdeltainprog &&
3729 			!(ppd->lflags & QIBL_IB_AUTONEG_INPROG)) {
3730 		ppd->cpspec->ibdeltainprog = 1;
3731 		ppd->cpspec->ibsymsnap = read_7220_creg32(ppd->dd,
3732 							  cr_ibsymbolerr);
3733 		ppd->cpspec->iblnkerrsnap = read_7220_creg32(ppd->dd,
3734 						     cr_iblinkerrrecov);
3735 	}
3736 
3737 	if (!ret)
3738 		qib_setup_7220_setextled(ppd, ibup);
3739 	return ret;
3740 }
3741 
3742 /*
3743  * Does read/modify/write to appropriate registers to
3744  * set output and direction bits selected by mask.
3745  * these are in their canonical positions (e.g. lsb of
3746  * dir will end up in D48 of extctrl on existing chips).
3747  * returns contents of GP Inputs.
3748  */
3749 static int gpio_7220_mod(struct qib_devdata *dd, u32 out, u32 dir, u32 mask)
3750 {
3751 	u64 read_val, new_out;
3752 	unsigned long flags;
3753 
3754 	if (mask) {
3755 		/* some bits being written, lock access to GPIO */
3756 		dir &= mask;
3757 		out &= mask;
3758 		spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
3759 		dd->cspec->extctrl &= ~((u64)mask << SYM_LSB(EXTCtrl, GPIOOe));
3760 		dd->cspec->extctrl |= ((u64) dir << SYM_LSB(EXTCtrl, GPIOOe));
3761 		new_out = (dd->cspec->gpio_out & ~mask) | out;
3762 
3763 		qib_write_kreg(dd, kr_extctrl, dd->cspec->extctrl);
3764 		qib_write_kreg(dd, kr_gpio_out, new_out);
3765 		dd->cspec->gpio_out = new_out;
3766 		spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
3767 	}
3768 	/*
3769 	 * It is unlikely that a read at this time would get valid
3770 	 * data on a pin whose direction line was set in the same
3771 	 * call to this function. We include the read here because
3772 	 * that allows us to potentially combine a change on one pin with
3773 	 * a read on another, and because the old code did something like
3774 	 * this.
3775 	 */
3776 	read_val = qib_read_kreg64(dd, kr_extstatus);
3777 	return SYM_FIELD(read_val, EXTStatus, GPIOIn);
3778 }
3779 
3780 /*
3781  * Read fundamental info we need to use the chip.  These are
3782  * the registers that describe chip capabilities, and are
3783  * saved in shadow registers.
3784  */
3785 static void get_7220_chip_params(struct qib_devdata *dd)
3786 {
3787 	u64 val;
3788 	u32 piobufs;
3789 	int mtu;
3790 
3791 	dd->uregbase = qib_read_kreg32(dd, kr_userregbase);
3792 
3793 	dd->rcvtidcnt = qib_read_kreg32(dd, kr_rcvtidcnt);
3794 	dd->rcvtidbase = qib_read_kreg32(dd, kr_rcvtidbase);
3795 	dd->rcvegrbase = qib_read_kreg32(dd, kr_rcvegrbase);
3796 	dd->palign = qib_read_kreg32(dd, kr_palign);
3797 	dd->piobufbase = qib_read_kreg64(dd, kr_sendpiobufbase);
3798 	dd->pio2k_bufbase = dd->piobufbase & 0xffffffff;
3799 
3800 	val = qib_read_kreg64(dd, kr_sendpiosize);
3801 	dd->piosize2k = val & ~0U;
3802 	dd->piosize4k = val >> 32;
3803 
3804 	mtu = ib_mtu_enum_to_int(qib_ibmtu);
3805 	if (mtu == -1)
3806 		mtu = QIB_DEFAULT_MTU;
3807 	dd->pport->ibmtu = (u32)mtu;
3808 
3809 	val = qib_read_kreg64(dd, kr_sendpiobufcnt);
3810 	dd->piobcnt2k = val & ~0U;
3811 	dd->piobcnt4k = val >> 32;
3812 	/* these may be adjusted in init_chip_wc_pat() */
3813 	dd->pio2kbase = (u32 __iomem *)
3814 		((char __iomem *) dd->kregbase + dd->pio2k_bufbase);
3815 	if (dd->piobcnt4k) {
3816 		dd->pio4kbase = (u32 __iomem *)
3817 			((char __iomem *) dd->kregbase +
3818 			 (dd->piobufbase >> 32));
3819 		/*
3820 		 * 4K buffers take 2 pages; we use roundup just to be
3821 		 * paranoid; we calculate it once here, rather than on
3822 		 * ever buf allocate
3823 		 */
3824 		dd->align4k = ALIGN(dd->piosize4k, dd->palign);
3825 	}
3826 
3827 	piobufs = dd->piobcnt4k + dd->piobcnt2k;
3828 
3829 	dd->pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) /
3830 		(sizeof(u64) * BITS_PER_BYTE / 2);
3831 }
3832 
3833 /*
3834  * The chip base addresses in cspec and cpspec have to be set
3835  * after possible init_chip_wc_pat(), rather than in
3836  * qib_get_7220_chip_params(), so split out as separate function
3837  */
3838 static void set_7220_baseaddrs(struct qib_devdata *dd)
3839 {
3840 	u32 cregbase;
3841 	/* init after possible re-map in init_chip_wc_pat() */
3842 	cregbase = qib_read_kreg32(dd, kr_counterregbase);
3843 	dd->cspec->cregbase = (u64 __iomem *)
3844 		((char __iomem *) dd->kregbase + cregbase);
3845 
3846 	dd->egrtidbase = (u64 __iomem *)
3847 		((char __iomem *) dd->kregbase + dd->rcvegrbase);
3848 }
3849 
3850 
3851 #define SENDCTRL_SHADOWED (SYM_MASK(SendCtrl, SendIntBufAvail) |	\
3852 			   SYM_MASK(SendCtrl, SPioEnable) |		\
3853 			   SYM_MASK(SendCtrl, SSpecialTriggerEn) |	\
3854 			   SYM_MASK(SendCtrl, SendBufAvailUpd) |	\
3855 			   SYM_MASK(SendCtrl, AvailUpdThld) |		\
3856 			   SYM_MASK(SendCtrl, SDmaEnable) |		\
3857 			   SYM_MASK(SendCtrl, SDmaIntEnable) |		\
3858 			   SYM_MASK(SendCtrl, SDmaHalt) |		\
3859 			   SYM_MASK(SendCtrl, SDmaSingleDescriptor))
3860 
3861 static int sendctrl_hook(struct qib_devdata *dd,
3862 			 const struct diag_observer *op,
3863 			 u32 offs, u64 *data, u64 mask, int only_32)
3864 {
3865 	unsigned long flags;
3866 	unsigned idx = offs / sizeof(u64);
3867 	u64 local_data, all_bits;
3868 
3869 	if (idx != kr_sendctrl) {
3870 		qib_dev_err(dd, "SendCtrl Hook called with offs %X, %s-bit\n",
3871 			    offs, only_32 ? "32" : "64");
3872 		return 0;
3873 	}
3874 
3875 	all_bits = ~0ULL;
3876 	if (only_32)
3877 		all_bits >>= 32;
3878 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
3879 	if ((mask & all_bits) != all_bits) {
3880 		/*
3881 		 * At least some mask bits are zero, so we need
3882 		 * to read. The judgement call is whether from
3883 		 * reg or shadow. First-cut: read reg, and complain
3884 		 * if any bits which should be shadowed are different
3885 		 * from their shadowed value.
3886 		 */
3887 		if (only_32)
3888 			local_data = (u64)qib_read_kreg32(dd, idx);
3889 		else
3890 			local_data = qib_read_kreg64(dd, idx);
3891 		qib_dev_err(dd, "Sendctrl -> %X, Shad -> %X\n",
3892 			    (u32)local_data, (u32)dd->sendctrl);
3893 		if ((local_data & SENDCTRL_SHADOWED) !=
3894 		    (dd->sendctrl & SENDCTRL_SHADOWED))
3895 			qib_dev_err(dd, "Sendctrl read: %X shadow is %X\n",
3896 				(u32)local_data, (u32) dd->sendctrl);
3897 		*data = (local_data & ~mask) | (*data & mask);
3898 	}
3899 	if (mask) {
3900 		/*
3901 		 * At least some mask bits are one, so we need
3902 		 * to write, but only shadow some bits.
3903 		 */
3904 		u64 sval, tval; /* Shadowed, transient */
3905 
3906 		/*
3907 		 * New shadow val is bits we don't want to touch,
3908 		 * ORed with bits we do, that are intended for shadow.
3909 		 */
3910 		sval = (dd->sendctrl & ~mask);
3911 		sval |= *data & SENDCTRL_SHADOWED & mask;
3912 		dd->sendctrl = sval;
3913 		tval = sval | (*data & ~SENDCTRL_SHADOWED & mask);
3914 		qib_dev_err(dd, "Sendctrl <- %X, Shad <- %X\n",
3915 			    (u32)tval, (u32)sval);
3916 		qib_write_kreg(dd, kr_sendctrl, tval);
3917 		qib_write_kreg(dd, kr_scratch, 0Ull);
3918 	}
3919 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
3920 
3921 	return only_32 ? 4 : 8;
3922 }
3923 
3924 static const struct diag_observer sendctrl_observer = {
3925 	sendctrl_hook, kr_sendctrl * sizeof(u64),
3926 	kr_sendctrl * sizeof(u64)
3927 };
3928 
3929 /*
3930  * write the final few registers that depend on some of the
3931  * init setup.  Done late in init, just before bringing up
3932  * the serdes.
3933  */
3934 static int qib_late_7220_initreg(struct qib_devdata *dd)
3935 {
3936 	int ret = 0;
3937 	u64 val;
3938 
3939 	qib_write_kreg(dd, kr_rcvhdrentsize, dd->rcvhdrentsize);
3940 	qib_write_kreg(dd, kr_rcvhdrsize, dd->rcvhdrsize);
3941 	qib_write_kreg(dd, kr_rcvhdrcnt, dd->rcvhdrcnt);
3942 	qib_write_kreg(dd, kr_sendpioavailaddr, dd->pioavailregs_phys);
3943 	val = qib_read_kreg64(dd, kr_sendpioavailaddr);
3944 	if (val != dd->pioavailregs_phys) {
3945 		qib_dev_err(dd,
3946 			"Catastrophic software error, SendPIOAvailAddr written as %lx, read back as %llx\n",
3947 			(unsigned long) dd->pioavailregs_phys,
3948 			(unsigned long long) val);
3949 		ret = -EINVAL;
3950 	}
3951 	qib_register_observer(dd, &sendctrl_observer);
3952 	return ret;
3953 }
3954 
3955 static int qib_init_7220_variables(struct qib_devdata *dd)
3956 {
3957 	struct qib_chippport_specific *cpspec;
3958 	struct qib_pportdata *ppd;
3959 	int ret = 0;
3960 	u32 sbufs, updthresh;
3961 
3962 	cpspec = (struct qib_chippport_specific *)(dd + 1);
3963 	ppd = &cpspec->pportdata;
3964 	dd->pport = ppd;
3965 	dd->num_pports = 1;
3966 
3967 	dd->cspec = (struct qib_chip_specific *)(cpspec + dd->num_pports);
3968 	dd->cspec->dd = dd;
3969 	ppd->cpspec = cpspec;
3970 
3971 	spin_lock_init(&dd->cspec->sdepb_lock);
3972 	spin_lock_init(&dd->cspec->rcvmod_lock);
3973 	spin_lock_init(&dd->cspec->gpio_lock);
3974 
3975 	/* we haven't yet set QIB_PRESENT, so use read directly */
3976 	dd->revision = readq(&dd->kregbase[kr_revision]);
3977 
3978 	if ((dd->revision & 0xffffffffU) == 0xffffffffU) {
3979 		qib_dev_err(dd,
3980 			"Revision register read failure, giving up initialization\n");
3981 		ret = -ENODEV;
3982 		goto bail;
3983 	}
3984 	dd->flags |= QIB_PRESENT;  /* now register routines work */
3985 
3986 	dd->majrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3987 				    ChipRevMajor);
3988 	dd->minrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3989 				    ChipRevMinor);
3990 
3991 	get_7220_chip_params(dd);
3992 	qib_7220_boardname(dd);
3993 
3994 	/*
3995 	 * GPIO bits for TWSI data and clock,
3996 	 * used for serial EEPROM.
3997 	 */
3998 	dd->gpio_sda_num = _QIB_GPIO_SDA_NUM;
3999 	dd->gpio_scl_num = _QIB_GPIO_SCL_NUM;
4000 	dd->twsi_eeprom_dev = QIB_TWSI_EEPROM_DEV;
4001 
4002 	dd->flags |= QIB_HAS_INTX | QIB_HAS_LINK_LATENCY |
4003 		QIB_NODMA_RTAIL | QIB_HAS_THRESH_UPDATE;
4004 	dd->flags |= qib_special_trigger ?
4005 		QIB_USE_SPCL_TRIG : QIB_HAS_SEND_DMA;
4006 
4007 	init_waitqueue_head(&cpspec->autoneg_wait);
4008 	INIT_DELAYED_WORK(&cpspec->autoneg_work, autoneg_7220_work);
4009 
4010 	ret = qib_init_pportdata(ppd, dd, 0, 1);
4011 	if (ret)
4012 		goto bail;
4013 	ppd->link_width_supported = IB_WIDTH_1X | IB_WIDTH_4X;
4014 	ppd->link_speed_supported = QIB_IB_SDR | QIB_IB_DDR;
4015 
4016 	ppd->link_width_enabled = ppd->link_width_supported;
4017 	ppd->link_speed_enabled = ppd->link_speed_supported;
4018 	/*
4019 	 * Set the initial values to reasonable default, will be set
4020 	 * for real when link is up.
4021 	 */
4022 	ppd->link_width_active = IB_WIDTH_4X;
4023 	ppd->link_speed_active = QIB_IB_SDR;
4024 	ppd->delay_mult = rate_to_delay[0][1];
4025 	ppd->vls_supported = IB_VL_VL0;
4026 	ppd->vls_operational = ppd->vls_supported;
4027 
4028 	if (!qib_mini_init)
4029 		qib_write_kreg(dd, kr_rcvbthqp, QIB_KD_QP);
4030 
4031 	timer_setup(&ppd->cpspec->chase_timer, reenable_7220_chase, 0);
4032 
4033 	qib_num_cfg_vls = 1; /* if any 7220's, only one VL */
4034 
4035 	dd->rcvhdrentsize = QIB_RCVHDR_ENTSIZE;
4036 	dd->rcvhdrsize = QIB_DFLT_RCVHDRSIZE;
4037 	dd->rhf_offset =
4038 		dd->rcvhdrentsize - sizeof(u64) / sizeof(u32);
4039 
4040 	/* we always allocate at least 2048 bytes for eager buffers */
4041 	ret = ib_mtu_enum_to_int(qib_ibmtu);
4042 	dd->rcvegrbufsize = ret != -1 ? max(ret, 2048) : QIB_DEFAULT_MTU;
4043 	dd->rcvegrbufsize_shift = ilog2(dd->rcvegrbufsize);
4044 
4045 	qib_7220_tidtemplate(dd);
4046 
4047 	/*
4048 	 * We can request a receive interrupt for 1 or
4049 	 * more packets from current offset.  For now, we set this
4050 	 * up for a single packet.
4051 	 */
4052 	dd->rhdrhead_intr_off = 1ULL << 32;
4053 
4054 	/* setup the stats timer; the add_timer is done at end of init */
4055 	timer_setup(&dd->stats_timer, qib_get_7220_faststats, 0);
4056 	dd->stats_timer.expires = jiffies + ACTIVITY_TIMER * HZ;
4057 
4058 	/*
4059 	 * Control[4] has been added to change the arbitration within
4060 	 * the SDMA engine between favoring data fetches over descriptor
4061 	 * fetches.  qib_sdma_fetch_arb==0 gives data fetches priority.
4062 	 */
4063 	if (qib_sdma_fetch_arb)
4064 		dd->control |= 1 << 4;
4065 
4066 	dd->ureg_align = 0x10000;  /* 64KB alignment */
4067 
4068 	dd->piosize2kmax_dwords = (dd->piosize2k >> 2)-1;
4069 	qib_7220_config_ctxts(dd);
4070 	qib_set_ctxtcnt(dd);  /* needed for PAT setup */
4071 
4072 	ret = init_chip_wc_pat(dd, 0);
4073 	if (ret)
4074 		goto bail;
4075 	set_7220_baseaddrs(dd); /* set chip access pointers now */
4076 
4077 	ret = 0;
4078 	if (qib_mini_init)
4079 		goto bail;
4080 
4081 	ret = qib_create_ctxts(dd);
4082 	init_7220_cntrnames(dd);
4083 
4084 	/* use all of 4KB buffers for the kernel SDMA, zero if !SDMA.
4085 	 * reserve the update threshold amount for other kernel use, such
4086 	 * as sending SMI, MAD, and ACKs, or 3, whichever is greater,
4087 	 * unless we aren't enabling SDMA, in which case we want to use
4088 	 * all the 4k bufs for the kernel.
4089 	 * if this was less than the update threshold, we could wait
4090 	 * a long time for an update.  Coded this way because we
4091 	 * sometimes change the update threshold for various reasons,
4092 	 * and we want this to remain robust.
4093 	 */
4094 	updthresh = 8U; /* update threshold */
4095 	if (dd->flags & QIB_HAS_SEND_DMA) {
4096 		dd->cspec->sdmabufcnt =  dd->piobcnt4k;
4097 		sbufs = updthresh > 3 ? updthresh : 3;
4098 	} else {
4099 		dd->cspec->sdmabufcnt = 0;
4100 		sbufs = dd->piobcnt4k;
4101 	}
4102 
4103 	dd->cspec->lastbuf_for_pio = dd->piobcnt2k + dd->piobcnt4k -
4104 		dd->cspec->sdmabufcnt;
4105 	dd->lastctxt_piobuf = dd->cspec->lastbuf_for_pio - sbufs;
4106 	dd->cspec->lastbuf_for_pio--; /* range is <= , not < */
4107 	dd->last_pio = dd->cspec->lastbuf_for_pio;
4108 	dd->pbufsctxt = dd->lastctxt_piobuf /
4109 		(dd->cfgctxts - dd->first_user_ctxt);
4110 
4111 	/*
4112 	 * if we are at 16 user contexts, we will have one 7 sbufs
4113 	 * per context, so drop the update threshold to match.  We
4114 	 * want to update before we actually run out, at low pbufs/ctxt
4115 	 * so give ourselves some margin
4116 	 */
4117 	if ((dd->pbufsctxt - 2) < updthresh)
4118 		updthresh = dd->pbufsctxt - 2;
4119 
4120 	dd->cspec->updthresh_dflt = updthresh;
4121 	dd->cspec->updthresh = updthresh;
4122 
4123 	/* before full enable, no interrupts, no locking needed */
4124 	dd->sendctrl |= (updthresh & SYM_RMASK(SendCtrl, AvailUpdThld))
4125 			     << SYM_LSB(SendCtrl, AvailUpdThld);
4126 
4127 	dd->psxmitwait_supported = 1;
4128 	dd->psxmitwait_check_rate = QIB_7220_PSXMITWAIT_CHECK_RATE;
4129 bail:
4130 	return ret;
4131 }
4132 
4133 static u32 __iomem *qib_7220_getsendbuf(struct qib_pportdata *ppd, u64 pbc,
4134 					u32 *pbufnum)
4135 {
4136 	u32 first, last, plen = pbc & QIB_PBC_LENGTH_MASK;
4137 	struct qib_devdata *dd = ppd->dd;
4138 	u32 __iomem *buf;
4139 
4140 	if (((pbc >> 32) & PBC_7220_VL15_SEND_CTRL) &&
4141 		!(ppd->lflags & (QIBL_IB_AUTONEG_INPROG | QIBL_LINKACTIVE)))
4142 		buf = get_7220_link_buf(ppd, pbufnum);
4143 	else {
4144 		if ((plen + 1) > dd->piosize2kmax_dwords)
4145 			first = dd->piobcnt2k;
4146 		else
4147 			first = 0;
4148 		/* try 4k if all 2k busy, so same last for both sizes */
4149 		last = dd->cspec->lastbuf_for_pio;
4150 		buf = qib_getsendbuf_range(dd, pbufnum, first, last);
4151 	}
4152 	return buf;
4153 }
4154 
4155 /* these 2 "counters" are really control registers, and are always RW */
4156 static void qib_set_cntr_7220_sample(struct qib_pportdata *ppd, u32 intv,
4157 				     u32 start)
4158 {
4159 	write_7220_creg(ppd->dd, cr_psinterval, intv);
4160 	write_7220_creg(ppd->dd, cr_psstart, start);
4161 }
4162 
4163 /*
4164  * NOTE: no real attempt is made to generalize the SDMA stuff.
4165  * At some point "soon" we will have a new more generalized
4166  * set of sdma interface, and then we'll clean this up.
4167  */
4168 
4169 /* Must be called with sdma_lock held, or before init finished */
4170 static void qib_sdma_update_7220_tail(struct qib_pportdata *ppd, u16 tail)
4171 {
4172 	/* Commit writes to memory and advance the tail on the chip */
4173 	wmb();
4174 	ppd->sdma_descq_tail = tail;
4175 	qib_write_kreg(ppd->dd, kr_senddmatail, tail);
4176 }
4177 
4178 static void qib_sdma_set_7220_desc_cnt(struct qib_pportdata *ppd, unsigned cnt)
4179 {
4180 }
4181 
4182 static struct sdma_set_state_action sdma_7220_action_table[] = {
4183 	[qib_sdma_state_s00_hw_down] = {
4184 		.op_enable = 0,
4185 		.op_intenable = 0,
4186 		.op_halt = 0,
4187 		.go_s99_running_tofalse = 1,
4188 	},
4189 	[qib_sdma_state_s10_hw_start_up_wait] = {
4190 		.op_enable = 1,
4191 		.op_intenable = 1,
4192 		.op_halt = 1,
4193 	},
4194 	[qib_sdma_state_s20_idle] = {
4195 		.op_enable = 1,
4196 		.op_intenable = 1,
4197 		.op_halt = 1,
4198 	},
4199 	[qib_sdma_state_s30_sw_clean_up_wait] = {
4200 		.op_enable = 0,
4201 		.op_intenable = 1,
4202 		.op_halt = 0,
4203 	},
4204 	[qib_sdma_state_s40_hw_clean_up_wait] = {
4205 		.op_enable = 1,
4206 		.op_intenable = 1,
4207 		.op_halt = 1,
4208 	},
4209 	[qib_sdma_state_s50_hw_halt_wait] = {
4210 		.op_enable = 1,
4211 		.op_intenable = 1,
4212 		.op_halt = 1,
4213 	},
4214 	[qib_sdma_state_s99_running] = {
4215 		.op_enable = 1,
4216 		.op_intenable = 1,
4217 		.op_halt = 0,
4218 		.go_s99_running_totrue = 1,
4219 	},
4220 };
4221 
4222 static void qib_7220_sdma_init_early(struct qib_pportdata *ppd)
4223 {
4224 	ppd->sdma_state.set_state_action = sdma_7220_action_table;
4225 }
4226 
4227 static int init_sdma_7220_regs(struct qib_pportdata *ppd)
4228 {
4229 	struct qib_devdata *dd = ppd->dd;
4230 	unsigned i, n;
4231 	u64 senddmabufmask[3] = { 0 };
4232 
4233 	/* Set SendDmaBase */
4234 	qib_write_kreg(dd, kr_senddmabase, ppd->sdma_descq_phys);
4235 	qib_sdma_7220_setlengen(ppd);
4236 	qib_sdma_update_7220_tail(ppd, 0); /* Set SendDmaTail */
4237 	/* Set SendDmaHeadAddr */
4238 	qib_write_kreg(dd, kr_senddmaheadaddr, ppd->sdma_head_phys);
4239 
4240 	/*
4241 	 * Reserve all the former "kernel" piobufs, using high number range
4242 	 * so we get as many 4K buffers as possible
4243 	 */
4244 	n = dd->piobcnt2k + dd->piobcnt4k;
4245 	i = n - dd->cspec->sdmabufcnt;
4246 
4247 	for (; i < n; ++i) {
4248 		unsigned word = i / 64;
4249 		unsigned bit = i & 63;
4250 
4251 		senddmabufmask[word] |= 1ULL << bit;
4252 	}
4253 	qib_write_kreg(dd, kr_senddmabufmask0, senddmabufmask[0]);
4254 	qib_write_kreg(dd, kr_senddmabufmask1, senddmabufmask[1]);
4255 	qib_write_kreg(dd, kr_senddmabufmask2, senddmabufmask[2]);
4256 
4257 	ppd->sdma_state.first_sendbuf = i;
4258 	ppd->sdma_state.last_sendbuf = n;
4259 
4260 	return 0;
4261 }
4262 
4263 /* sdma_lock must be held */
4264 static u16 qib_sdma_7220_gethead(struct qib_pportdata *ppd)
4265 {
4266 	struct qib_devdata *dd = ppd->dd;
4267 	int sane;
4268 	int use_dmahead;
4269 	u16 swhead;
4270 	u16 swtail;
4271 	u16 cnt;
4272 	u16 hwhead;
4273 
4274 	use_dmahead = __qib_sdma_running(ppd) &&
4275 		(dd->flags & QIB_HAS_SDMA_TIMEOUT);
4276 retry:
4277 	hwhead = use_dmahead ?
4278 		(u16)le64_to_cpu(*ppd->sdma_head_dma) :
4279 		(u16)qib_read_kreg32(dd, kr_senddmahead);
4280 
4281 	swhead = ppd->sdma_descq_head;
4282 	swtail = ppd->sdma_descq_tail;
4283 	cnt = ppd->sdma_descq_cnt;
4284 
4285 	if (swhead < swtail) {
4286 		/* not wrapped */
4287 		sane = (hwhead >= swhead) & (hwhead <= swtail);
4288 	} else if (swhead > swtail) {
4289 		/* wrapped around */
4290 		sane = ((hwhead >= swhead) && (hwhead < cnt)) ||
4291 			(hwhead <= swtail);
4292 	} else {
4293 		/* empty */
4294 		sane = (hwhead == swhead);
4295 	}
4296 
4297 	if (unlikely(!sane)) {
4298 		if (use_dmahead) {
4299 			/* try one more time, directly from the register */
4300 			use_dmahead = 0;
4301 			goto retry;
4302 		}
4303 		/* assume no progress */
4304 		hwhead = swhead;
4305 	}
4306 
4307 	return hwhead;
4308 }
4309 
4310 static int qib_sdma_7220_busy(struct qib_pportdata *ppd)
4311 {
4312 	u64 hwstatus = qib_read_kreg64(ppd->dd, kr_senddmastatus);
4313 
4314 	return (hwstatus & SYM_MASK(SendDmaStatus, ScoreBoardDrainInProg)) ||
4315 	       (hwstatus & SYM_MASK(SendDmaStatus, AbortInProg)) ||
4316 	       (hwstatus & SYM_MASK(SendDmaStatus, InternalSDmaEnable)) ||
4317 	       !(hwstatus & SYM_MASK(SendDmaStatus, ScbEmpty));
4318 }
4319 
4320 /*
4321  * Compute the amount of delay before sending the next packet if the
4322  * port's send rate differs from the static rate set for the QP.
4323  * Since the delay affects this packet but the amount of the delay is
4324  * based on the length of the previous packet, use the last delay computed
4325  * and save the delay count for this packet to be used next time
4326  * we get here.
4327  */
4328 static u32 qib_7220_setpbc_control(struct qib_pportdata *ppd, u32 plen,
4329 				   u8 srate, u8 vl)
4330 {
4331 	u8 snd_mult = ppd->delay_mult;
4332 	u8 rcv_mult = ib_rate_to_delay[srate];
4333 	u32 ret = ppd->cpspec->last_delay_mult;
4334 
4335 	ppd->cpspec->last_delay_mult = (rcv_mult > snd_mult) ?
4336 		(plen * (rcv_mult - snd_mult) + 1) >> 1 : 0;
4337 
4338 	/* Indicate VL15, if necessary */
4339 	if (vl == 15)
4340 		ret |= PBC_7220_VL15_SEND_CTRL;
4341 	return ret;
4342 }
4343 
4344 static void qib_7220_initvl15_bufs(struct qib_devdata *dd)
4345 {
4346 }
4347 
4348 static void qib_7220_init_ctxt(struct qib_ctxtdata *rcd)
4349 {
4350 	if (!rcd->ctxt) {
4351 		rcd->rcvegrcnt = IBA7220_KRCVEGRCNT;
4352 		rcd->rcvegr_tid_base = 0;
4353 	} else {
4354 		rcd->rcvegrcnt = rcd->dd->cspec->rcvegrcnt;
4355 		rcd->rcvegr_tid_base = IBA7220_KRCVEGRCNT +
4356 			(rcd->ctxt - 1) * rcd->rcvegrcnt;
4357 	}
4358 }
4359 
4360 static void qib_7220_txchk_change(struct qib_devdata *dd, u32 start,
4361 				  u32 len, u32 which, struct qib_ctxtdata *rcd)
4362 {
4363 	int i;
4364 	unsigned long flags;
4365 
4366 	switch (which) {
4367 	case TXCHK_CHG_TYPE_KERN:
4368 		/* see if we need to raise avail update threshold */
4369 		spin_lock_irqsave(&dd->uctxt_lock, flags);
4370 		for (i = dd->first_user_ctxt;
4371 		     dd->cspec->updthresh != dd->cspec->updthresh_dflt
4372 		     && i < dd->cfgctxts; i++)
4373 			if (dd->rcd[i] && dd->rcd[i]->subctxt_cnt &&
4374 			   ((dd->rcd[i]->piocnt / dd->rcd[i]->subctxt_cnt) - 1)
4375 			   < dd->cspec->updthresh_dflt)
4376 				break;
4377 		spin_unlock_irqrestore(&dd->uctxt_lock, flags);
4378 		if (i == dd->cfgctxts) {
4379 			spin_lock_irqsave(&dd->sendctrl_lock, flags);
4380 			dd->cspec->updthresh = dd->cspec->updthresh_dflt;
4381 			dd->sendctrl &= ~SYM_MASK(SendCtrl, AvailUpdThld);
4382 			dd->sendctrl |= (dd->cspec->updthresh &
4383 					 SYM_RMASK(SendCtrl, AvailUpdThld)) <<
4384 					   SYM_LSB(SendCtrl, AvailUpdThld);
4385 			spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
4386 			sendctrl_7220_mod(dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
4387 		}
4388 		break;
4389 	case TXCHK_CHG_TYPE_USER:
4390 		spin_lock_irqsave(&dd->sendctrl_lock, flags);
4391 		if (rcd && rcd->subctxt_cnt && ((rcd->piocnt
4392 			/ rcd->subctxt_cnt) - 1) < dd->cspec->updthresh) {
4393 			dd->cspec->updthresh = (rcd->piocnt /
4394 						rcd->subctxt_cnt) - 1;
4395 			dd->sendctrl &= ~SYM_MASK(SendCtrl, AvailUpdThld);
4396 			dd->sendctrl |= (dd->cspec->updthresh &
4397 					SYM_RMASK(SendCtrl, AvailUpdThld))
4398 					<< SYM_LSB(SendCtrl, AvailUpdThld);
4399 			spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
4400 			sendctrl_7220_mod(dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
4401 		} else
4402 			spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
4403 		break;
4404 	}
4405 }
4406 
4407 static void writescratch(struct qib_devdata *dd, u32 val)
4408 {
4409 	qib_write_kreg(dd, kr_scratch, val);
4410 }
4411 
4412 #define VALID_TS_RD_REG_MASK 0xBF
4413 /**
4414  * qib_7220_tempsense_rd - read register of temp sensor via TWSI
4415  * @dd: the qlogic_ib device
4416  * @regnum: register to read from
4417  *
4418  * returns reg contents (0..255) or < 0 for error
4419  */
4420 static int qib_7220_tempsense_rd(struct qib_devdata *dd, int regnum)
4421 {
4422 	int ret;
4423 	u8 rdata;
4424 
4425 	if (regnum > 7) {
4426 		ret = -EINVAL;
4427 		goto bail;
4428 	}
4429 
4430 	/* return a bogus value for (the one) register we do not have */
4431 	if (!((1 << regnum) & VALID_TS_RD_REG_MASK)) {
4432 		ret = 0;
4433 		goto bail;
4434 	}
4435 
4436 	ret = mutex_lock_interruptible(&dd->eep_lock);
4437 	if (ret)
4438 		goto bail;
4439 
4440 	ret = qib_twsi_blk_rd(dd, QIB_TWSI_TEMP_DEV, regnum, &rdata, 1);
4441 	if (!ret)
4442 		ret = rdata;
4443 
4444 	mutex_unlock(&dd->eep_lock);
4445 
4446 	/*
4447 	 * There are three possibilities here:
4448 	 * ret is actual value (0..255)
4449 	 * ret is -ENXIO or -EINVAL from twsi code or this file
4450 	 * ret is -EINTR from mutex_lock_interruptible.
4451 	 */
4452 bail:
4453 	return ret;
4454 }
4455 
4456 #ifdef CONFIG_INFINIBAND_QIB_DCA
4457 static int qib_7220_notify_dca(struct qib_devdata *dd, unsigned long event)
4458 {
4459 	return 0;
4460 }
4461 #endif
4462 
4463 /* Dummy function, as 7220 boards never disable EEPROM Write */
4464 static int qib_7220_eeprom_wen(struct qib_devdata *dd, int wen)
4465 {
4466 	return 1;
4467 }
4468 
4469 /**
4470  * qib_init_iba7220_funcs - set up the chip-specific function pointers
4471  * @pdev: the pci_dev for qlogic_ib device
4472  * @ent: pci_device_id struct for this dev
4473  *
4474  * This is global, and is called directly at init to set up the
4475  * chip-specific function pointers for later use.
4476  */
4477 struct qib_devdata *qib_init_iba7220_funcs(struct pci_dev *pdev,
4478 					   const struct pci_device_id *ent)
4479 {
4480 	struct qib_devdata *dd;
4481 	int ret;
4482 	u32 boardid, minwidth;
4483 
4484 	dd = qib_alloc_devdata(pdev, sizeof(struct qib_chip_specific) +
4485 		sizeof(struct qib_chippport_specific));
4486 	if (IS_ERR(dd))
4487 		goto bail;
4488 
4489 	dd->f_bringup_serdes    = qib_7220_bringup_serdes;
4490 	dd->f_cleanup           = qib_setup_7220_cleanup;
4491 	dd->f_clear_tids        = qib_7220_clear_tids;
4492 	dd->f_free_irq          = qib_free_irq;
4493 	dd->f_get_base_info     = qib_7220_get_base_info;
4494 	dd->f_get_msgheader     = qib_7220_get_msgheader;
4495 	dd->f_getsendbuf        = qib_7220_getsendbuf;
4496 	dd->f_gpio_mod          = gpio_7220_mod;
4497 	dd->f_eeprom_wen        = qib_7220_eeprom_wen;
4498 	dd->f_hdrqempty         = qib_7220_hdrqempty;
4499 	dd->f_ib_updown         = qib_7220_ib_updown;
4500 	dd->f_init_ctxt         = qib_7220_init_ctxt;
4501 	dd->f_initvl15_bufs     = qib_7220_initvl15_bufs;
4502 	dd->f_intr_fallback     = qib_7220_intr_fallback;
4503 	dd->f_late_initreg      = qib_late_7220_initreg;
4504 	dd->f_setpbc_control    = qib_7220_setpbc_control;
4505 	dd->f_portcntr          = qib_portcntr_7220;
4506 	dd->f_put_tid           = qib_7220_put_tid;
4507 	dd->f_quiet_serdes      = qib_7220_quiet_serdes;
4508 	dd->f_rcvctrl           = rcvctrl_7220_mod;
4509 	dd->f_read_cntrs        = qib_read_7220cntrs;
4510 	dd->f_read_portcntrs    = qib_read_7220portcntrs;
4511 	dd->f_reset             = qib_setup_7220_reset;
4512 	dd->f_init_sdma_regs    = init_sdma_7220_regs;
4513 	dd->f_sdma_busy         = qib_sdma_7220_busy;
4514 	dd->f_sdma_gethead      = qib_sdma_7220_gethead;
4515 	dd->f_sdma_sendctrl     = qib_7220_sdma_sendctrl;
4516 	dd->f_sdma_set_desc_cnt = qib_sdma_set_7220_desc_cnt;
4517 	dd->f_sdma_update_tail  = qib_sdma_update_7220_tail;
4518 	dd->f_sdma_hw_clean_up  = qib_7220_sdma_hw_clean_up;
4519 	dd->f_sdma_hw_start_up  = qib_7220_sdma_hw_start_up;
4520 	dd->f_sdma_init_early   = qib_7220_sdma_init_early;
4521 	dd->f_sendctrl          = sendctrl_7220_mod;
4522 	dd->f_set_armlaunch     = qib_set_7220_armlaunch;
4523 	dd->f_set_cntr_sample   = qib_set_cntr_7220_sample;
4524 	dd->f_iblink_state      = qib_7220_iblink_state;
4525 	dd->f_ibphys_portstate  = qib_7220_phys_portstate;
4526 	dd->f_get_ib_cfg        = qib_7220_get_ib_cfg;
4527 	dd->f_set_ib_cfg        = qib_7220_set_ib_cfg;
4528 	dd->f_set_ib_loopback   = qib_7220_set_loopback;
4529 	dd->f_set_intr_state    = qib_7220_set_intr_state;
4530 	dd->f_setextled         = qib_setup_7220_setextled;
4531 	dd->f_txchk_change      = qib_7220_txchk_change;
4532 	dd->f_update_usrhead    = qib_update_7220_usrhead;
4533 	dd->f_wantpiobuf_intr   = qib_wantpiobuf_7220_intr;
4534 	dd->f_xgxs_reset        = qib_7220_xgxs_reset;
4535 	dd->f_writescratch      = writescratch;
4536 	dd->f_tempsense_rd	= qib_7220_tempsense_rd;
4537 #ifdef CONFIG_INFINIBAND_QIB_DCA
4538 	dd->f_notify_dca = qib_7220_notify_dca;
4539 #endif
4540 	/*
4541 	 * Do remaining pcie setup and save pcie values in dd.
4542 	 * Any error printing is already done by the init code.
4543 	 * On return, we have the chip mapped, but chip registers
4544 	 * are not set up until start of qib_init_7220_variables.
4545 	 */
4546 	ret = qib_pcie_ddinit(dd, pdev, ent);
4547 	if (ret < 0)
4548 		goto bail_free;
4549 
4550 	/* initialize chip-specific variables */
4551 	ret = qib_init_7220_variables(dd);
4552 	if (ret)
4553 		goto bail_cleanup;
4554 
4555 	if (qib_mini_init)
4556 		goto bail;
4557 
4558 	boardid = SYM_FIELD(dd->revision, Revision,
4559 			    BoardID);
4560 	switch (boardid) {
4561 	case 0:
4562 	case 2:
4563 	case 10:
4564 	case 12:
4565 		minwidth = 16; /* x16 capable boards */
4566 		break;
4567 	default:
4568 		minwidth = 8; /* x8 capable boards */
4569 		break;
4570 	}
4571 	if (qib_pcie_params(dd, minwidth, NULL))
4572 		qib_dev_err(dd,
4573 			"Failed to setup PCIe or interrupts; continuing anyway\n");
4574 
4575 	if (qib_read_kreg64(dd, kr_hwerrstatus) &
4576 	    QLOGIC_IB_HWE_SERDESPLLFAILED)
4577 		qib_write_kreg(dd, kr_hwerrclear,
4578 			       QLOGIC_IB_HWE_SERDESPLLFAILED);
4579 
4580 	/* setup interrupt handler (interrupt type handled above) */
4581 	qib_setup_7220_interrupt(dd);
4582 	qib_7220_init_hwerrors(dd);
4583 
4584 	/* clear diagctrl register, in case diags were running and crashed */
4585 	qib_write_kreg(dd, kr_hwdiagctrl, 0);
4586 
4587 	goto bail;
4588 
4589 bail_cleanup:
4590 	qib_pcie_ddcleanup(dd);
4591 bail_free:
4592 	qib_free_devdata(dd);
4593 	dd = ERR_PTR(ret);
4594 bail:
4595 	return dd;
4596 }
4597