1 /*
2  * Copyright (c) 2013 - 2017 Intel Corporation. All rights reserved.
3  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
4  * All rights reserved.
5  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 /*
36  * This file contains all of the code that is specific to the
37  * QLogic_IB 6120 PCIe chip.
38  */
39 
40 #include <linux/interrupt.h>
41 #include <linux/pci.h>
42 #include <linux/delay.h>
43 #include <rdma/ib_verbs.h>
44 
45 #include "qib.h"
46 #include "qib_6120_regs.h"
47 
48 static void qib_6120_setup_setextled(struct qib_pportdata *, u32);
49 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op);
50 static u8 qib_6120_phys_portstate(u64);
51 static u32 qib_6120_iblink_state(u64);
52 
53 /*
54  * This file contains all the chip-specific register information and
55  * access functions for the Intel Intel_IB PCI-Express chip.
56  *
57  */
58 
59 /* KREG_IDX uses machine-generated #defines */
60 #define KREG_IDX(regname) (QIB_6120_##regname##_OFFS / sizeof(u64))
61 
62 /* Use defines to tie machine-generated names to lower-case names */
63 #define kr_extctrl KREG_IDX(EXTCtrl)
64 #define kr_extstatus KREG_IDX(EXTStatus)
65 #define kr_gpio_clear KREG_IDX(GPIOClear)
66 #define kr_gpio_mask KREG_IDX(GPIOMask)
67 #define kr_gpio_out KREG_IDX(GPIOOut)
68 #define kr_gpio_status KREG_IDX(GPIOStatus)
69 #define kr_rcvctrl KREG_IDX(RcvCtrl)
70 #define kr_sendctrl KREG_IDX(SendCtrl)
71 #define kr_partitionkey KREG_IDX(RcvPartitionKey)
72 #define kr_hwdiagctrl KREG_IDX(HwDiagCtrl)
73 #define kr_ibcstatus KREG_IDX(IBCStatus)
74 #define kr_ibcctrl KREG_IDX(IBCCtrl)
75 #define kr_sendbuffererror KREG_IDX(SendBufErr0)
76 #define kr_rcvbthqp KREG_IDX(RcvBTHQP)
77 #define kr_counterregbase KREG_IDX(CntrRegBase)
78 #define kr_palign KREG_IDX(PageAlign)
79 #define kr_rcvegrbase KREG_IDX(RcvEgrBase)
80 #define kr_rcvegrcnt KREG_IDX(RcvEgrCnt)
81 #define kr_rcvhdrcnt KREG_IDX(RcvHdrCnt)
82 #define kr_rcvhdrentsize KREG_IDX(RcvHdrEntSize)
83 #define kr_rcvhdrsize KREG_IDX(RcvHdrSize)
84 #define kr_rcvtidbase KREG_IDX(RcvTIDBase)
85 #define kr_rcvtidcnt KREG_IDX(RcvTIDCnt)
86 #define kr_scratch KREG_IDX(Scratch)
87 #define kr_sendctrl KREG_IDX(SendCtrl)
88 #define kr_sendpioavailaddr KREG_IDX(SendPIOAvailAddr)
89 #define kr_sendpiobufbase KREG_IDX(SendPIOBufBase)
90 #define kr_sendpiobufcnt KREG_IDX(SendPIOBufCnt)
91 #define kr_sendpiosize KREG_IDX(SendPIOSize)
92 #define kr_sendregbase KREG_IDX(SendRegBase)
93 #define kr_userregbase KREG_IDX(UserRegBase)
94 #define kr_control KREG_IDX(Control)
95 #define kr_intclear KREG_IDX(IntClear)
96 #define kr_intmask KREG_IDX(IntMask)
97 #define kr_intstatus KREG_IDX(IntStatus)
98 #define kr_errclear KREG_IDX(ErrClear)
99 #define kr_errmask KREG_IDX(ErrMask)
100 #define kr_errstatus KREG_IDX(ErrStatus)
101 #define kr_hwerrclear KREG_IDX(HwErrClear)
102 #define kr_hwerrmask KREG_IDX(HwErrMask)
103 #define kr_hwerrstatus KREG_IDX(HwErrStatus)
104 #define kr_revision KREG_IDX(Revision)
105 #define kr_portcnt KREG_IDX(PortCnt)
106 #define kr_serdes_cfg0 KREG_IDX(SerdesCfg0)
107 #define kr_serdes_cfg1 (kr_serdes_cfg0 + 1)
108 #define kr_serdes_stat KREG_IDX(SerdesStat)
109 #define kr_xgxs_cfg KREG_IDX(XGXSCfg)
110 
111 /* These must only be written via qib_write_kreg_ctxt() */
112 #define kr_rcvhdraddr KREG_IDX(RcvHdrAddr0)
113 #define kr_rcvhdrtailaddr KREG_IDX(RcvHdrTailAddr0)
114 
115 #define CREG_IDX(regname) ((QIB_6120_##regname##_OFFS - \
116 			QIB_6120_LBIntCnt_OFFS) / sizeof(u64))
117 
118 #define cr_badformat CREG_IDX(RxBadFormatCnt)
119 #define cr_erricrc CREG_IDX(RxICRCErrCnt)
120 #define cr_errlink CREG_IDX(RxLinkProblemCnt)
121 #define cr_errlpcrc CREG_IDX(RxLPCRCErrCnt)
122 #define cr_errpkey CREG_IDX(RxPKeyMismatchCnt)
123 #define cr_rcvflowctrl_err CREG_IDX(RxFlowCtrlErrCnt)
124 #define cr_err_rlen CREG_IDX(RxLenErrCnt)
125 #define cr_errslen CREG_IDX(TxLenErrCnt)
126 #define cr_errtidfull CREG_IDX(RxTIDFullErrCnt)
127 #define cr_errtidvalid CREG_IDX(RxTIDValidErrCnt)
128 #define cr_errvcrc CREG_IDX(RxVCRCErrCnt)
129 #define cr_ibstatuschange CREG_IDX(IBStatusChangeCnt)
130 #define cr_lbint CREG_IDX(LBIntCnt)
131 #define cr_invalidrlen CREG_IDX(RxMaxMinLenErrCnt)
132 #define cr_invalidslen CREG_IDX(TxMaxMinLenErrCnt)
133 #define cr_lbflowstall CREG_IDX(LBFlowStallCnt)
134 #define cr_pktrcv CREG_IDX(RxDataPktCnt)
135 #define cr_pktrcvflowctrl CREG_IDX(RxFlowPktCnt)
136 #define cr_pktsend CREG_IDX(TxDataPktCnt)
137 #define cr_pktsendflow CREG_IDX(TxFlowPktCnt)
138 #define cr_portovfl CREG_IDX(RxP0HdrEgrOvflCnt)
139 #define cr_rcvebp CREG_IDX(RxEBPCnt)
140 #define cr_rcvovfl CREG_IDX(RxBufOvflCnt)
141 #define cr_senddropped CREG_IDX(TxDroppedPktCnt)
142 #define cr_sendstall CREG_IDX(TxFlowStallCnt)
143 #define cr_sendunderrun CREG_IDX(TxUnderrunCnt)
144 #define cr_wordrcv CREG_IDX(RxDwordCnt)
145 #define cr_wordsend CREG_IDX(TxDwordCnt)
146 #define cr_txunsupvl CREG_IDX(TxUnsupVLErrCnt)
147 #define cr_rxdroppkt CREG_IDX(RxDroppedPktCnt)
148 #define cr_iblinkerrrecov CREG_IDX(IBLinkErrRecoveryCnt)
149 #define cr_iblinkdown CREG_IDX(IBLinkDownedCnt)
150 #define cr_ibsymbolerr CREG_IDX(IBSymbolErrCnt)
151 
152 #define SYM_RMASK(regname, fldname) ((u64)              \
153 	QIB_6120_##regname##_##fldname##_RMASK)
154 #define SYM_MASK(regname, fldname) ((u64)               \
155 	QIB_6120_##regname##_##fldname##_RMASK <<       \
156 	 QIB_6120_##regname##_##fldname##_LSB)
157 #define SYM_LSB(regname, fldname) (QIB_6120_##regname##_##fldname##_LSB)
158 
159 #define SYM_FIELD(value, regname, fldname) ((u64) \
160 	(((value) >> SYM_LSB(regname, fldname)) & \
161 	 SYM_RMASK(regname, fldname)))
162 #define ERR_MASK(fldname) SYM_MASK(ErrMask, fldname##Mask)
163 #define HWE_MASK(fldname) SYM_MASK(HwErrMask, fldname##Mask)
164 
165 /* link training states, from IBC */
166 #define IB_6120_LT_STATE_DISABLED        0x00
167 #define IB_6120_LT_STATE_LINKUP          0x01
168 #define IB_6120_LT_STATE_POLLACTIVE      0x02
169 #define IB_6120_LT_STATE_POLLQUIET       0x03
170 #define IB_6120_LT_STATE_SLEEPDELAY      0x04
171 #define IB_6120_LT_STATE_SLEEPQUIET      0x05
172 #define IB_6120_LT_STATE_CFGDEBOUNCE     0x08
173 #define IB_6120_LT_STATE_CFGRCVFCFG      0x09
174 #define IB_6120_LT_STATE_CFGWAITRMT      0x0a
175 #define IB_6120_LT_STATE_CFGIDLE 0x0b
176 #define IB_6120_LT_STATE_RECOVERRETRAIN  0x0c
177 #define IB_6120_LT_STATE_RECOVERWAITRMT  0x0e
178 #define IB_6120_LT_STATE_RECOVERIDLE     0x0f
179 
180 /* link state machine states from IBC */
181 #define IB_6120_L_STATE_DOWN             0x0
182 #define IB_6120_L_STATE_INIT             0x1
183 #define IB_6120_L_STATE_ARM              0x2
184 #define IB_6120_L_STATE_ACTIVE           0x3
185 #define IB_6120_L_STATE_ACT_DEFER        0x4
186 
187 static const u8 qib_6120_physportstate[0x20] = {
188 	[IB_6120_LT_STATE_DISABLED] = IB_PHYSPORTSTATE_DISABLED,
189 	[IB_6120_LT_STATE_LINKUP] = IB_PHYSPORTSTATE_LINKUP,
190 	[IB_6120_LT_STATE_POLLACTIVE] = IB_PHYSPORTSTATE_POLL,
191 	[IB_6120_LT_STATE_POLLQUIET] = IB_PHYSPORTSTATE_POLL,
192 	[IB_6120_LT_STATE_SLEEPDELAY] = IB_PHYSPORTSTATE_SLEEP,
193 	[IB_6120_LT_STATE_SLEEPQUIET] = IB_PHYSPORTSTATE_SLEEP,
194 	[IB_6120_LT_STATE_CFGDEBOUNCE] =
195 		IB_PHYSPORTSTATE_CFG_TRAIN,
196 	[IB_6120_LT_STATE_CFGRCVFCFG] =
197 		IB_PHYSPORTSTATE_CFG_TRAIN,
198 	[IB_6120_LT_STATE_CFGWAITRMT] =
199 		IB_PHYSPORTSTATE_CFG_TRAIN,
200 	[IB_6120_LT_STATE_CFGIDLE] = IB_PHYSPORTSTATE_CFG_TRAIN,
201 	[IB_6120_LT_STATE_RECOVERRETRAIN] =
202 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
203 	[IB_6120_LT_STATE_RECOVERWAITRMT] =
204 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
205 	[IB_6120_LT_STATE_RECOVERIDLE] =
206 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
207 	[0x10] = IB_PHYSPORTSTATE_CFG_TRAIN,
208 	[0x11] = IB_PHYSPORTSTATE_CFG_TRAIN,
209 	[0x12] = IB_PHYSPORTSTATE_CFG_TRAIN,
210 	[0x13] = IB_PHYSPORTSTATE_CFG_TRAIN,
211 	[0x14] = IB_PHYSPORTSTATE_CFG_TRAIN,
212 	[0x15] = IB_PHYSPORTSTATE_CFG_TRAIN,
213 	[0x16] = IB_PHYSPORTSTATE_CFG_TRAIN,
214 	[0x17] = IB_PHYSPORTSTATE_CFG_TRAIN
215 };
216 
217 
218 struct qib_chip_specific {
219 	u64 __iomem *cregbase;
220 	u64 *cntrs;
221 	u64 *portcntrs;
222 	void *dummy_hdrq;   /* used after ctxt close */
223 	dma_addr_t dummy_hdrq_phys;
224 	spinlock_t kernel_tid_lock; /* no back to back kernel TID writes */
225 	spinlock_t user_tid_lock; /* no back to back user TID writes */
226 	spinlock_t rcvmod_lock; /* protect rcvctrl shadow changes */
227 	spinlock_t gpio_lock; /* RMW of shadows/regs for ExtCtrl and GPIO */
228 	u64 hwerrmask;
229 	u64 errormask;
230 	u64 gpio_out; /* shadow of kr_gpio_out, for rmw ops */
231 	u64 gpio_mask; /* shadow the gpio mask register */
232 	u64 extctrl; /* shadow the gpio output enable, etc... */
233 	/*
234 	 * these 5 fields are used to establish deltas for IB symbol
235 	 * errors and linkrecovery errors.  They can be reported on
236 	 * some chips during link negotiation prior to INIT, and with
237 	 * DDR when faking DDR negotiations with non-IBTA switches.
238 	 * The chip counters are adjusted at driver unload if there is
239 	 * a non-zero delta.
240 	 */
241 	u64 ibdeltainprog;
242 	u64 ibsymdelta;
243 	u64 ibsymsnap;
244 	u64 iblnkerrdelta;
245 	u64 iblnkerrsnap;
246 	u64 ibcctrl; /* shadow for kr_ibcctrl */
247 	u32 lastlinkrecov; /* link recovery issue */
248 	u32 cntrnamelen;
249 	u32 portcntrnamelen;
250 	u32 ncntrs;
251 	u32 nportcntrs;
252 	/* used with gpio interrupts to implement IB counters */
253 	u32 rxfc_unsupvl_errs;
254 	u32 overrun_thresh_errs;
255 	/*
256 	 * these count only cases where _successive_ LocalLinkIntegrity
257 	 * errors were seen in the receive headers of IB standard packets
258 	 */
259 	u32 lli_errs;
260 	u32 lli_counter;
261 	u64 lli_thresh;
262 	u64 sword; /* total dwords sent (sample result) */
263 	u64 rword; /* total dwords received (sample result) */
264 	u64 spkts; /* total packets sent (sample result) */
265 	u64 rpkts; /* total packets received (sample result) */
266 	u64 xmit_wait; /* # of ticks no data sent (sample result) */
267 	struct timer_list pma_timer;
268 	struct qib_pportdata *ppd;
269 	char emsgbuf[128];
270 	char bitsmsgbuf[64];
271 	u8 pma_sample_status;
272 };
273 
274 /* ibcctrl bits */
275 #define QLOGIC_IB_IBCC_LINKINITCMD_DISABLE 1
276 /* cycle through TS1/TS2 till OK */
277 #define QLOGIC_IB_IBCC_LINKINITCMD_POLL 2
278 /* wait for TS1, then go on */
279 #define QLOGIC_IB_IBCC_LINKINITCMD_SLEEP 3
280 #define QLOGIC_IB_IBCC_LINKINITCMD_SHIFT 16
281 
282 #define QLOGIC_IB_IBCC_LINKCMD_DOWN 1           /* move to 0x11 */
283 #define QLOGIC_IB_IBCC_LINKCMD_ARMED 2          /* move to 0x21 */
284 #define QLOGIC_IB_IBCC_LINKCMD_ACTIVE 3 /* move to 0x31 */
285 #define QLOGIC_IB_IBCC_LINKCMD_SHIFT 18
286 
287 /*
288  * We could have a single register get/put routine, that takes a group type,
289  * but this is somewhat clearer and cleaner.  It also gives us some error
290  * checking.  64 bit register reads should always work, but are inefficient
291  * on opteron (the northbridge always generates 2 separate HT 32 bit reads),
292  * so we use kreg32 wherever possible.  User register and counter register
293  * reads are always 32 bit reads, so only one form of those routines.
294  */
295 
296 /**
297  * qib_read_ureg32 - read 32-bit virtualized per-context register
298  * @dd: device
299  * @regno: register number
300  * @ctxt: context number
301  *
302  * Return the contents of a register that is virtualized to be per context.
303  * Returns -1 on errors (not distinguishable from valid contents at
304  * runtime; we may add a separate error variable at some point).
305  */
306 static inline u32 qib_read_ureg32(const struct qib_devdata *dd,
307 				  enum qib_ureg regno, int ctxt)
308 {
309 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
310 		return 0;
311 
312 	if (dd->userbase)
313 		return readl(regno + (u64 __iomem *)
314 			     ((char __iomem *)dd->userbase +
315 			      dd->ureg_align * ctxt));
316 	else
317 		return readl(regno + (u64 __iomem *)
318 			     (dd->uregbase +
319 			      (char __iomem *)dd->kregbase +
320 			      dd->ureg_align * ctxt));
321 }
322 
323 /**
324  * qib_write_ureg - write 32-bit virtualized per-context register
325  * @dd: device
326  * @regno: register number
327  * @value: value
328  * @ctxt: context
329  *
330  * Write the contents of a register that is virtualized to be per context.
331  */
332 static inline void qib_write_ureg(const struct qib_devdata *dd,
333 				  enum qib_ureg regno, u64 value, int ctxt)
334 {
335 	u64 __iomem *ubase;
336 
337 	if (dd->userbase)
338 		ubase = (u64 __iomem *)
339 			((char __iomem *) dd->userbase +
340 			 dd->ureg_align * ctxt);
341 	else
342 		ubase = (u64 __iomem *)
343 			(dd->uregbase +
344 			 (char __iomem *) dd->kregbase +
345 			 dd->ureg_align * ctxt);
346 
347 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
348 		writeq(value, &ubase[regno]);
349 }
350 
351 static inline u32 qib_read_kreg32(const struct qib_devdata *dd,
352 				  const u16 regno)
353 {
354 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
355 		return -1;
356 	return readl((u32 __iomem *)&dd->kregbase[regno]);
357 }
358 
359 static inline u64 qib_read_kreg64(const struct qib_devdata *dd,
360 				  const u16 regno)
361 {
362 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
363 		return -1;
364 
365 	return readq(&dd->kregbase[regno]);
366 }
367 
368 static inline void qib_write_kreg(const struct qib_devdata *dd,
369 				  const u16 regno, u64 value)
370 {
371 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
372 		writeq(value, &dd->kregbase[regno]);
373 }
374 
375 /**
376  * qib_write_kreg_ctxt - write a device's per-ctxt 64-bit kernel register
377  * @dd: the qlogic_ib device
378  * @regno: the register number to write
379  * @ctxt: the context containing the register
380  * @value: the value to write
381  */
382 static inline void qib_write_kreg_ctxt(const struct qib_devdata *dd,
383 				       const u16 regno, unsigned ctxt,
384 				       u64 value)
385 {
386 	qib_write_kreg(dd, regno + ctxt, value);
387 }
388 
389 static inline void write_6120_creg(const struct qib_devdata *dd,
390 				   u16 regno, u64 value)
391 {
392 	if (dd->cspec->cregbase && (dd->flags & QIB_PRESENT))
393 		writeq(value, &dd->cspec->cregbase[regno]);
394 }
395 
396 static inline u64 read_6120_creg(const struct qib_devdata *dd, u16 regno)
397 {
398 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
399 		return 0;
400 	return readq(&dd->cspec->cregbase[regno]);
401 }
402 
403 static inline u32 read_6120_creg32(const struct qib_devdata *dd, u16 regno)
404 {
405 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
406 		return 0;
407 	return readl(&dd->cspec->cregbase[regno]);
408 }
409 
410 /* kr_control bits */
411 #define QLOGIC_IB_C_RESET 1U
412 
413 /* kr_intstatus, kr_intclear, kr_intmask bits */
414 #define QLOGIC_IB_I_RCVURG_MASK ((1U << 5) - 1)
415 #define QLOGIC_IB_I_RCVURG_SHIFT 0
416 #define QLOGIC_IB_I_RCVAVAIL_MASK ((1U << 5) - 1)
417 #define QLOGIC_IB_I_RCVAVAIL_SHIFT 12
418 
419 #define QLOGIC_IB_C_FREEZEMODE 0x00000002
420 #define QLOGIC_IB_C_LINKENABLE 0x00000004
421 #define QLOGIC_IB_I_ERROR               0x0000000080000000ULL
422 #define QLOGIC_IB_I_SPIOSENT            0x0000000040000000ULL
423 #define QLOGIC_IB_I_SPIOBUFAVAIL        0x0000000020000000ULL
424 #define QLOGIC_IB_I_GPIO                0x0000000010000000ULL
425 #define QLOGIC_IB_I_BITSEXTANT \
426 		((QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT) | \
427 		(QLOGIC_IB_I_RCVAVAIL_MASK << \
428 		 QLOGIC_IB_I_RCVAVAIL_SHIFT) | \
429 		QLOGIC_IB_I_ERROR | QLOGIC_IB_I_SPIOSENT | \
430 		QLOGIC_IB_I_SPIOBUFAVAIL | QLOGIC_IB_I_GPIO)
431 
432 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
433 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK  0x000000000000003fULL
434 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT 0
435 #define QLOGIC_IB_HWE_PCIEPOISONEDTLP      0x0000000010000000ULL
436 #define QLOGIC_IB_HWE_PCIECPLTIMEOUT       0x0000000020000000ULL
437 #define QLOGIC_IB_HWE_PCIEBUSPARITYXTLH    0x0000000040000000ULL
438 #define QLOGIC_IB_HWE_PCIEBUSPARITYXADM    0x0000000080000000ULL
439 #define QLOGIC_IB_HWE_PCIEBUSPARITYRADM    0x0000000100000000ULL
440 #define QLOGIC_IB_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
441 #define QLOGIC_IB_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
442 #define QLOGIC_IB_HWE_PCIE1PLLFAILED       0x0400000000000000ULL
443 #define QLOGIC_IB_HWE_PCIE0PLLFAILED       0x0800000000000000ULL
444 #define QLOGIC_IB_HWE_SERDESPLLFAILED      0x1000000000000000ULL
445 
446 
447 /* kr_extstatus bits */
448 #define QLOGIC_IB_EXTS_FREQSEL 0x2
449 #define QLOGIC_IB_EXTS_SERDESSEL 0x4
450 #define QLOGIC_IB_EXTS_MEMBIST_ENDTEST     0x0000000000004000
451 #define QLOGIC_IB_EXTS_MEMBIST_FOUND       0x0000000000008000
452 
453 /* kr_xgxsconfig bits */
454 #define QLOGIC_IB_XGXS_RESET          0x5ULL
455 
456 #define _QIB_GPIO_SDA_NUM 1
457 #define _QIB_GPIO_SCL_NUM 0
458 
459 /* Bits in GPIO for the added IB link interrupts */
460 #define GPIO_RXUVL_BIT 3
461 #define GPIO_OVRUN_BIT 4
462 #define GPIO_LLI_BIT 5
463 #define GPIO_ERRINTR_MASK 0x38
464 
465 
466 #define QLOGIC_IB_RT_BUFSIZE_MASK 0xe0000000ULL
467 #define QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid) \
468 	((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) >> 29) + 11 - 1)
469 #define QLOGIC_IB_RT_BUFSIZE(tid) (1 << QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid))
470 #define QLOGIC_IB_RT_IS_VALID(tid) \
471 	(((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) && \
472 	 ((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) != QLOGIC_IB_RT_BUFSIZE_MASK)))
473 #define QLOGIC_IB_RT_ADDR_MASK 0x1FFFFFFFULL /* 29 bits valid */
474 #define QLOGIC_IB_RT_ADDR_SHIFT 10
475 
476 #define QLOGIC_IB_R_INTRAVAIL_SHIFT 16
477 #define QLOGIC_IB_R_TAILUPD_SHIFT 31
478 #define IBA6120_R_PKEY_DIS_SHIFT 30
479 
480 #define PBC_6120_VL15_SEND_CTRL (1ULL << 31) /* pbc; VL15; link_buf only */
481 
482 #define IBCBUSFRSPCPARITYERR HWE_MASK(IBCBusFromSPCParityErr)
483 #define IBCBUSTOSPCPARITYERR HWE_MASK(IBCBusToSPCParityErr)
484 
485 #define SYM_MASK_BIT(regname, fldname, bit) ((u64) \
486 	((1ULL << (SYM_LSB(regname, fldname) + (bit)))))
487 
488 #define TXEMEMPARITYERR_PIOBUF \
489 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 0)
490 #define TXEMEMPARITYERR_PIOPBC \
491 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 1)
492 #define TXEMEMPARITYERR_PIOLAUNCHFIFO \
493 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 2)
494 
495 #define RXEMEMPARITYERR_RCVBUF \
496 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 0)
497 #define RXEMEMPARITYERR_LOOKUPQ \
498 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 1)
499 #define RXEMEMPARITYERR_EXPTID \
500 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 2)
501 #define RXEMEMPARITYERR_EAGERTID \
502 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 3)
503 #define RXEMEMPARITYERR_FLAGBUF \
504 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 4)
505 #define RXEMEMPARITYERR_DATAINFO \
506 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 5)
507 #define RXEMEMPARITYERR_HDRINFO \
508 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 6)
509 
510 /* 6120 specific hardware errors... */
511 static const struct qib_hwerror_msgs qib_6120_hwerror_msgs[] = {
512 	/* generic hardware errors */
513 	QLOGIC_IB_HWE_MSG(IBCBUSFRSPCPARITYERR, "QIB2IB Parity"),
514 	QLOGIC_IB_HWE_MSG(IBCBUSTOSPCPARITYERR, "IB2QIB Parity"),
515 
516 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOBUF,
517 			  "TXE PIOBUF Memory Parity"),
518 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOPBC,
519 			  "TXE PIOPBC Memory Parity"),
520 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOLAUNCHFIFO,
521 			  "TXE PIOLAUNCHFIFO Memory Parity"),
522 
523 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_RCVBUF,
524 			  "RXE RCVBUF Memory Parity"),
525 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_LOOKUPQ,
526 			  "RXE LOOKUPQ Memory Parity"),
527 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EAGERTID,
528 			  "RXE EAGERTID Memory Parity"),
529 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EXPTID,
530 			  "RXE EXPTID Memory Parity"),
531 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_FLAGBUF,
532 			  "RXE FLAGBUF Memory Parity"),
533 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_DATAINFO,
534 			  "RXE DATAINFO Memory Parity"),
535 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_HDRINFO,
536 			  "RXE HDRINFO Memory Parity"),
537 
538 	/* chip-specific hardware errors */
539 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEPOISONEDTLP,
540 			  "PCIe Poisoned TLP"),
541 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLTIMEOUT,
542 			  "PCIe completion timeout"),
543 	/*
544 	 * In practice, it's unlikely wthat we'll see PCIe PLL, or bus
545 	 * parity or memory parity error failures, because most likely we
546 	 * won't be able to talk to the core of the chip.  Nonetheless, we
547 	 * might see them, if they are in parts of the PCIe core that aren't
548 	 * essential.
549 	 */
550 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE1PLLFAILED,
551 			  "PCIePLL1"),
552 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE0PLLFAILED,
553 			  "PCIePLL0"),
554 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXTLH,
555 			  "PCIe XTLH core parity"),
556 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXADM,
557 			  "PCIe ADM TX core parity"),
558 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYRADM,
559 			  "PCIe ADM RX core parity"),
560 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_SERDESPLLFAILED,
561 			  "SerDes PLL"),
562 };
563 
564 #define TXE_PIO_PARITY (TXEMEMPARITYERR_PIOBUF | TXEMEMPARITYERR_PIOPBC)
565 #define _QIB_PLL_FAIL (QLOGIC_IB_HWE_COREPLL_FBSLIP |   \
566 		QLOGIC_IB_HWE_COREPLL_RFSLIP)
567 
568 	/* variables for sanity checking interrupt and errors */
569 #define IB_HWE_BITSEXTANT \
570 	(HWE_MASK(RXEMemParityErr) |					\
571 	 HWE_MASK(TXEMemParityErr) |					\
572 	 (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<			\
573 	  QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) |			\
574 	 QLOGIC_IB_HWE_PCIE1PLLFAILED |					\
575 	 QLOGIC_IB_HWE_PCIE0PLLFAILED |					\
576 	 QLOGIC_IB_HWE_PCIEPOISONEDTLP |				\
577 	 QLOGIC_IB_HWE_PCIECPLTIMEOUT |					\
578 	 QLOGIC_IB_HWE_PCIEBUSPARITYXTLH |				\
579 	 QLOGIC_IB_HWE_PCIEBUSPARITYXADM |				\
580 	 QLOGIC_IB_HWE_PCIEBUSPARITYRADM |				\
581 	 HWE_MASK(PowerOnBISTFailed) |					\
582 	 QLOGIC_IB_HWE_COREPLL_FBSLIP |					\
583 	 QLOGIC_IB_HWE_COREPLL_RFSLIP |					\
584 	 QLOGIC_IB_HWE_SERDESPLLFAILED |				\
585 	 HWE_MASK(IBCBusToSPCParityErr) |				\
586 	 HWE_MASK(IBCBusFromSPCParityErr))
587 
588 #define IB_E_BITSEXTANT \
589 	(ERR_MASK(RcvFormatErr) | ERR_MASK(RcvVCRCErr) |		\
590 	 ERR_MASK(RcvICRCErr) | ERR_MASK(RcvMinPktLenErr) |		\
591 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvLongPktLenErr) |	\
592 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvUnexpectedCharErr) | \
593 	 ERR_MASK(RcvUnsupportedVLErr) | ERR_MASK(RcvEBPErr) |		\
594 	 ERR_MASK(RcvIBFlowErr) | ERR_MASK(RcvBadVersionErr) |		\
595 	 ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) |		\
596 	 ERR_MASK(RcvBadTidErr) | ERR_MASK(RcvHdrLenErr) |		\
597 	 ERR_MASK(RcvHdrErr) | ERR_MASK(RcvIBLostLinkErr) |		\
598 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendMaxPktLenErr) |	\
599 	 ERR_MASK(SendUnderRunErr) | ERR_MASK(SendPktLenErr) |		\
600 	 ERR_MASK(SendDroppedSmpPktErr) |				\
601 	 ERR_MASK(SendDroppedDataPktErr) |				\
602 	 ERR_MASK(SendPioArmLaunchErr) |				\
603 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
604 	 ERR_MASK(SendUnsupportedVLErr) | ERR_MASK(IBStatusChanged) |	\
605 	 ERR_MASK(InvalidAddrErr) | ERR_MASK(ResetNegated) |		\
606 	 ERR_MASK(HardwareErr))
607 
608 #define QLOGIC_IB_E_PKTERRS ( \
609 		ERR_MASK(SendPktLenErr) |				\
610 		ERR_MASK(SendDroppedDataPktErr) |			\
611 		ERR_MASK(RcvVCRCErr) |					\
612 		ERR_MASK(RcvICRCErr) |					\
613 		ERR_MASK(RcvShortPktLenErr) |				\
614 		ERR_MASK(RcvEBPErr))
615 
616 /* These are all rcv-related errors which we want to count for stats */
617 #define E_SUM_PKTERRS						\
618 	(ERR_MASK(RcvHdrLenErr) | ERR_MASK(RcvBadTidErr) |		\
619 	 ERR_MASK(RcvBadVersionErr) | ERR_MASK(RcvHdrErr) |		\
620 	 ERR_MASK(RcvLongPktLenErr) | ERR_MASK(RcvShortPktLenErr) |	\
621 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
622 	 ERR_MASK(RcvFormatErr) | ERR_MASK(RcvUnsupportedVLErr) |	\
623 	 ERR_MASK(RcvUnexpectedCharErr) | ERR_MASK(RcvEBPErr))
624 
625 /* These are all send-related errors which we want to count for stats */
626 #define E_SUM_ERRS							\
627 	(ERR_MASK(SendPioArmLaunchErr) |				\
628 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
629 	 ERR_MASK(SendDroppedDataPktErr) |				\
630 	 ERR_MASK(SendDroppedSmpPktErr) |				\
631 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnsupportedVLErr) |	\
632 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
633 	 ERR_MASK(InvalidAddrErr))
634 
635 /*
636  * this is similar to E_SUM_ERRS, but can't ignore armlaunch, don't ignore
637  * errors not related to freeze and cancelling buffers.  Can't ignore
638  * armlaunch because could get more while still cleaning up, and need
639  * to cancel those as they happen.
640  */
641 #define E_SPKT_ERRS_IGNORE \
642 	(ERR_MASK(SendDroppedDataPktErr) |				\
643 	 ERR_MASK(SendDroppedSmpPktErr) |				\
644 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendMinPktLenErr) |	\
645 	 ERR_MASK(SendPktLenErr))
646 
647 /*
648  * these are errors that can occur when the link changes state while
649  * a packet is being sent or received.  This doesn't cover things
650  * like EBP or VCRC that can be the result of a sending having the
651  * link change state, so we receive a "known bad" packet.
652  */
653 #define E_SUM_LINK_PKTERRS		\
654 	(ERR_MASK(SendDroppedDataPktErr) |				\
655 	 ERR_MASK(SendDroppedSmpPktErr) |				\
656 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
657 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
658 	 ERR_MASK(RcvUnexpectedCharErr))
659 
660 static void qib_6120_put_tid_2(struct qib_devdata *, u64 __iomem *,
661 			       u32, unsigned long);
662 
663 /*
664  * On platforms using this chip, and not having ordered WC stores, we
665  * can get TXE parity errors due to speculative reads to the PIO buffers,
666  * and this, due to a chip issue can result in (many) false parity error
667  * reports.  So it's a debug print on those, and an info print on systems
668  * where the speculative reads don't occur.
669  */
670 static void qib_6120_txe_recover(struct qib_devdata *dd)
671 {
672 	if (!qib_unordered_wc())
673 		qib_devinfo(dd->pcidev,
674 			    "Recovering from TXE PIO parity error\n");
675 }
676 
677 /* enable/disable chip from delivering interrupts */
678 static void qib_6120_set_intr_state(struct qib_devdata *dd, u32 enable)
679 {
680 	if (enable) {
681 		if (dd->flags & QIB_BADINTR)
682 			return;
683 		qib_write_kreg(dd, kr_intmask, ~0ULL);
684 		/* force re-interrupt of any pending interrupts. */
685 		qib_write_kreg(dd, kr_intclear, 0ULL);
686 	} else
687 		qib_write_kreg(dd, kr_intmask, 0ULL);
688 }
689 
690 /*
691  * Try to cleanup as much as possible for anything that might have gone
692  * wrong while in freeze mode, such as pio buffers being written by user
693  * processes (causing armlaunch), send errors due to going into freeze mode,
694  * etc., and try to avoid causing extra interrupts while doing so.
695  * Forcibly update the in-memory pioavail register copies after cleanup
696  * because the chip won't do it while in freeze mode (the register values
697  * themselves are kept correct).
698  * Make sure that we don't lose any important interrupts by using the chip
699  * feature that says that writing 0 to a bit in *clear that is set in
700  * *status will cause an interrupt to be generated again (if allowed by
701  * the *mask value).
702  * This is in chip-specific code because of all of the register accesses,
703  * even though the details are similar on most chips
704  */
705 static void qib_6120_clear_freeze(struct qib_devdata *dd)
706 {
707 	/* disable error interrupts, to avoid confusion */
708 	qib_write_kreg(dd, kr_errmask, 0ULL);
709 
710 	/* also disable interrupts; errormask is sometimes overwritten */
711 	qib_6120_set_intr_state(dd, 0);
712 
713 	qib_cancel_sends(dd->pport);
714 
715 	/* clear the freeze, and be sure chip saw it */
716 	qib_write_kreg(dd, kr_control, dd->control);
717 	qib_read_kreg32(dd, kr_scratch);
718 
719 	/* force in-memory update now we are out of freeze */
720 	qib_force_pio_avail_update(dd);
721 
722 	/*
723 	 * force new interrupt if any hwerr, error or interrupt bits are
724 	 * still set, and clear "safe" send packet errors related to freeze
725 	 * and cancelling sends.  Re-enable error interrupts before possible
726 	 * force of re-interrupt on pending interrupts.
727 	 */
728 	qib_write_kreg(dd, kr_hwerrclear, 0ULL);
729 	qib_write_kreg(dd, kr_errclear, E_SPKT_ERRS_IGNORE);
730 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
731 	qib_6120_set_intr_state(dd, 1);
732 }
733 
734 /**
735  * qib_handle_6120_hwerrors - display hardware errors.
736  * @dd: the qlogic_ib device
737  * @msg: the output buffer
738  * @msgl: the size of the output buffer
739  *
740  * Use same msg buffer as regular errors to avoid excessive stack
741  * use.  Most hardware errors are catastrophic, but for right now,
742  * we'll print them and continue.  Reuse the same message buffer as
743  * handle_6120_errors() to avoid excessive stack usage.
744  */
745 static void qib_handle_6120_hwerrors(struct qib_devdata *dd, char *msg,
746 				     size_t msgl)
747 {
748 	u64 hwerrs;
749 	u32 bits, ctrl;
750 	int isfatal = 0;
751 	char *bitsmsg;
752 
753 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
754 	if (!hwerrs)
755 		return;
756 	if (hwerrs == ~0ULL) {
757 		qib_dev_err(dd,
758 			"Read of hardware error status failed (all bits set); ignoring\n");
759 		return;
760 	}
761 	qib_stats.sps_hwerrs++;
762 
763 	/* Always clear the error status register, except MEMBISTFAIL,
764 	 * regardless of whether we continue or stop using the chip.
765 	 * We want that set so we know it failed, even across driver reload.
766 	 * We'll still ignore it in the hwerrmask.  We do this partly for
767 	 * diagnostics, but also for support */
768 	qib_write_kreg(dd, kr_hwerrclear,
769 		       hwerrs & ~HWE_MASK(PowerOnBISTFailed));
770 
771 	hwerrs &= dd->cspec->hwerrmask;
772 
773 	/*
774 	 * Make sure we get this much out, unless told to be quiet,
775 	 * or it's occurred within the last 5 seconds.
776 	 */
777 	if (hwerrs & ~(TXE_PIO_PARITY | RXEMEMPARITYERR_EAGERTID))
778 		qib_devinfo(dd->pcidev,
779 			"Hardware error: hwerr=0x%llx (cleared)\n",
780 			(unsigned long long) hwerrs);
781 
782 	if (hwerrs & ~IB_HWE_BITSEXTANT)
783 		qib_dev_err(dd,
784 			"hwerror interrupt with unknown errors %llx set\n",
785 			(unsigned long long)(hwerrs & ~IB_HWE_BITSEXTANT));
786 
787 	ctrl = qib_read_kreg32(dd, kr_control);
788 	if ((ctrl & QLOGIC_IB_C_FREEZEMODE) && !dd->diag_client) {
789 		/*
790 		 * Parity errors in send memory are recoverable,
791 		 * just cancel the send (if indicated in * sendbuffererror),
792 		 * count the occurrence, unfreeze (if no other handled
793 		 * hardware error bits are set), and continue. They can
794 		 * occur if a processor speculative read is done to the PIO
795 		 * buffer while we are sending a packet, for example.
796 		 */
797 		if (hwerrs & TXE_PIO_PARITY) {
798 			qib_6120_txe_recover(dd);
799 			hwerrs &= ~TXE_PIO_PARITY;
800 		}
801 
802 		if (!hwerrs)
803 			qib_6120_clear_freeze(dd);
804 		else
805 			isfatal = 1;
806 	}
807 
808 	*msg = '\0';
809 
810 	if (hwerrs & HWE_MASK(PowerOnBISTFailed)) {
811 		isfatal = 1;
812 		strlcat(msg,
813 			"[Memory BIST test failed, InfiniPath hardware unusable]",
814 			msgl);
815 		/* ignore from now on, so disable until driver reloaded */
816 		dd->cspec->hwerrmask &= ~HWE_MASK(PowerOnBISTFailed);
817 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
818 	}
819 
820 	qib_format_hwerrors(hwerrs, qib_6120_hwerror_msgs,
821 			    ARRAY_SIZE(qib_6120_hwerror_msgs), msg, msgl);
822 
823 	bitsmsg = dd->cspec->bitsmsgbuf;
824 	if (hwerrs & (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<
825 		      QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT)) {
826 		bits = (u32) ((hwerrs >>
827 			       QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) &
828 			      QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK);
829 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
830 			 "[PCIe Mem Parity Errs %x] ", bits);
831 		strlcat(msg, bitsmsg, msgl);
832 	}
833 
834 	if (hwerrs & _QIB_PLL_FAIL) {
835 		isfatal = 1;
836 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
837 			 "[PLL failed (%llx), InfiniPath hardware unusable]",
838 			 (unsigned long long) hwerrs & _QIB_PLL_FAIL);
839 		strlcat(msg, bitsmsg, msgl);
840 		/* ignore from now on, so disable until driver reloaded */
841 		dd->cspec->hwerrmask &= ~(hwerrs & _QIB_PLL_FAIL);
842 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
843 	}
844 
845 	if (hwerrs & QLOGIC_IB_HWE_SERDESPLLFAILED) {
846 		/*
847 		 * If it occurs, it is left masked since the external
848 		 * interface is unused
849 		 */
850 		dd->cspec->hwerrmask &= ~QLOGIC_IB_HWE_SERDESPLLFAILED;
851 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
852 	}
853 
854 	if (hwerrs)
855 		/*
856 		 * if any set that we aren't ignoring; only
857 		 * make the complaint once, in case it's stuck
858 		 * or recurring, and we get here multiple
859 		 * times.
860 		 */
861 		qib_dev_err(dd, "%s hardware error\n", msg);
862 	else
863 		*msg = 0; /* recovered from all of them */
864 
865 	if (isfatal && !dd->diag_client) {
866 		qib_dev_err(dd,
867 			"Fatal Hardware Error, no longer usable, SN %.16s\n",
868 			dd->serial);
869 		/*
870 		 * for /sys status file and user programs to print; if no
871 		 * trailing brace is copied, we'll know it was truncated.
872 		 */
873 		if (dd->freezemsg)
874 			snprintf(dd->freezemsg, dd->freezelen,
875 				 "{%s}", msg);
876 		qib_disable_after_error(dd);
877 	}
878 }
879 
880 /*
881  * Decode the error status into strings, deciding whether to always
882  * print * it or not depending on "normal packet errors" vs everything
883  * else.   Return 1 if "real" errors, otherwise 0 if only packet
884  * errors, so caller can decide what to print with the string.
885  */
886 static int qib_decode_6120_err(struct qib_devdata *dd, char *buf, size_t blen,
887 			       u64 err)
888 {
889 	int iserr = 1;
890 
891 	*buf = '\0';
892 	if (err & QLOGIC_IB_E_PKTERRS) {
893 		if (!(err & ~QLOGIC_IB_E_PKTERRS))
894 			iserr = 0;
895 		if ((err & ERR_MASK(RcvICRCErr)) &&
896 		    !(err&(ERR_MASK(RcvVCRCErr)|ERR_MASK(RcvEBPErr))))
897 			strlcat(buf, "CRC ", blen);
898 		if (!iserr)
899 			goto done;
900 	}
901 	if (err & ERR_MASK(RcvHdrLenErr))
902 		strlcat(buf, "rhdrlen ", blen);
903 	if (err & ERR_MASK(RcvBadTidErr))
904 		strlcat(buf, "rbadtid ", blen);
905 	if (err & ERR_MASK(RcvBadVersionErr))
906 		strlcat(buf, "rbadversion ", blen);
907 	if (err & ERR_MASK(RcvHdrErr))
908 		strlcat(buf, "rhdr ", blen);
909 	if (err & ERR_MASK(RcvLongPktLenErr))
910 		strlcat(buf, "rlongpktlen ", blen);
911 	if (err & ERR_MASK(RcvMaxPktLenErr))
912 		strlcat(buf, "rmaxpktlen ", blen);
913 	if (err & ERR_MASK(RcvMinPktLenErr))
914 		strlcat(buf, "rminpktlen ", blen);
915 	if (err & ERR_MASK(SendMinPktLenErr))
916 		strlcat(buf, "sminpktlen ", blen);
917 	if (err & ERR_MASK(RcvFormatErr))
918 		strlcat(buf, "rformaterr ", blen);
919 	if (err & ERR_MASK(RcvUnsupportedVLErr))
920 		strlcat(buf, "runsupvl ", blen);
921 	if (err & ERR_MASK(RcvUnexpectedCharErr))
922 		strlcat(buf, "runexpchar ", blen);
923 	if (err & ERR_MASK(RcvIBFlowErr))
924 		strlcat(buf, "ribflow ", blen);
925 	if (err & ERR_MASK(SendUnderRunErr))
926 		strlcat(buf, "sunderrun ", blen);
927 	if (err & ERR_MASK(SendPioArmLaunchErr))
928 		strlcat(buf, "spioarmlaunch ", blen);
929 	if (err & ERR_MASK(SendUnexpectedPktNumErr))
930 		strlcat(buf, "sunexperrpktnum ", blen);
931 	if (err & ERR_MASK(SendDroppedSmpPktErr))
932 		strlcat(buf, "sdroppedsmppkt ", blen);
933 	if (err & ERR_MASK(SendMaxPktLenErr))
934 		strlcat(buf, "smaxpktlen ", blen);
935 	if (err & ERR_MASK(SendUnsupportedVLErr))
936 		strlcat(buf, "sunsupVL ", blen);
937 	if (err & ERR_MASK(InvalidAddrErr))
938 		strlcat(buf, "invalidaddr ", blen);
939 	if (err & ERR_MASK(RcvEgrFullErr))
940 		strlcat(buf, "rcvegrfull ", blen);
941 	if (err & ERR_MASK(RcvHdrFullErr))
942 		strlcat(buf, "rcvhdrfull ", blen);
943 	if (err & ERR_MASK(IBStatusChanged))
944 		strlcat(buf, "ibcstatuschg ", blen);
945 	if (err & ERR_MASK(RcvIBLostLinkErr))
946 		strlcat(buf, "riblostlink ", blen);
947 	if (err & ERR_MASK(HardwareErr))
948 		strlcat(buf, "hardware ", blen);
949 	if (err & ERR_MASK(ResetNegated))
950 		strlcat(buf, "reset ", blen);
951 done:
952 	return iserr;
953 }
954 
955 /*
956  * Called when we might have an error that is specific to a particular
957  * PIO buffer, and may need to cancel that buffer, so it can be re-used.
958  */
959 static void qib_disarm_6120_senderrbufs(struct qib_pportdata *ppd)
960 {
961 	unsigned long sbuf[2];
962 	struct qib_devdata *dd = ppd->dd;
963 
964 	/*
965 	 * It's possible that sendbuffererror could have bits set; might
966 	 * have already done this as a result of hardware error handling.
967 	 */
968 	sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
969 	sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
970 
971 	if (sbuf[0] || sbuf[1])
972 		qib_disarm_piobufs_set(dd, sbuf,
973 				       dd->piobcnt2k + dd->piobcnt4k);
974 }
975 
976 static int chk_6120_linkrecovery(struct qib_devdata *dd, u64 ibcs)
977 {
978 	int ret = 1;
979 	u32 ibstate = qib_6120_iblink_state(ibcs);
980 	u32 linkrecov = read_6120_creg32(dd, cr_iblinkerrrecov);
981 
982 	if (linkrecov != dd->cspec->lastlinkrecov) {
983 		/* and no more until active again */
984 		dd->cspec->lastlinkrecov = 0;
985 		qib_set_linkstate(dd->pport, QIB_IB_LINKDOWN);
986 		ret = 0;
987 	}
988 	if (ibstate == IB_PORT_ACTIVE)
989 		dd->cspec->lastlinkrecov =
990 			read_6120_creg32(dd, cr_iblinkerrrecov);
991 	return ret;
992 }
993 
994 static void handle_6120_errors(struct qib_devdata *dd, u64 errs)
995 {
996 	char *msg;
997 	u64 ignore_this_time = 0;
998 	u64 iserr = 0;
999 	struct qib_pportdata *ppd = dd->pport;
1000 	u64 mask;
1001 
1002 	/* don't report errors that are masked */
1003 	errs &= dd->cspec->errormask;
1004 	msg = dd->cspec->emsgbuf;
1005 
1006 	/* do these first, they are most important */
1007 	if (errs & ERR_MASK(HardwareErr))
1008 		qib_handle_6120_hwerrors(dd, msg, sizeof(dd->cspec->emsgbuf));
1009 
1010 	if (errs & ~IB_E_BITSEXTANT)
1011 		qib_dev_err(dd,
1012 			"error interrupt with unknown errors %llx set\n",
1013 			(unsigned long long) (errs & ~IB_E_BITSEXTANT));
1014 
1015 	if (errs & E_SUM_ERRS) {
1016 		qib_disarm_6120_senderrbufs(ppd);
1017 		if ((errs & E_SUM_LINK_PKTERRS) &&
1018 		    !(ppd->lflags & QIBL_LINKACTIVE)) {
1019 			/*
1020 			 * This can happen when trying to bring the link
1021 			 * up, but the IB link changes state at the "wrong"
1022 			 * time. The IB logic then complains that the packet
1023 			 * isn't valid.  We don't want to confuse people, so
1024 			 * we just don't print them, except at debug
1025 			 */
1026 			ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1027 		}
1028 	} else if ((errs & E_SUM_LINK_PKTERRS) &&
1029 		   !(ppd->lflags & QIBL_LINKACTIVE)) {
1030 		/*
1031 		 * This can happen when SMA is trying to bring the link
1032 		 * up, but the IB link changes state at the "wrong" time.
1033 		 * The IB logic then complains that the packet isn't
1034 		 * valid.  We don't want to confuse people, so we just
1035 		 * don't print them, except at debug
1036 		 */
1037 		ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1038 	}
1039 
1040 	qib_write_kreg(dd, kr_errclear, errs);
1041 
1042 	errs &= ~ignore_this_time;
1043 	if (!errs)
1044 		goto done;
1045 
1046 	/*
1047 	 * The ones we mask off are handled specially below
1048 	 * or above.
1049 	 */
1050 	mask = ERR_MASK(IBStatusChanged) | ERR_MASK(RcvEgrFullErr) |
1051 		ERR_MASK(RcvHdrFullErr) | ERR_MASK(HardwareErr);
1052 	qib_decode_6120_err(dd, msg, sizeof(dd->cspec->emsgbuf), errs & ~mask);
1053 
1054 	if (errs & E_SUM_PKTERRS)
1055 		qib_stats.sps_rcverrs++;
1056 	if (errs & E_SUM_ERRS)
1057 		qib_stats.sps_txerrs++;
1058 
1059 	iserr = errs & ~(E_SUM_PKTERRS | QLOGIC_IB_E_PKTERRS);
1060 
1061 	if (errs & ERR_MASK(IBStatusChanged)) {
1062 		u64 ibcs = qib_read_kreg64(dd, kr_ibcstatus);
1063 		u32 ibstate = qib_6120_iblink_state(ibcs);
1064 		int handle = 1;
1065 
1066 		if (ibstate != IB_PORT_INIT && dd->cspec->lastlinkrecov)
1067 			handle = chk_6120_linkrecovery(dd, ibcs);
1068 		/*
1069 		 * Since going into a recovery state causes the link state
1070 		 * to go down and since recovery is transitory, it is better
1071 		 * if we "miss" ever seeing the link training state go into
1072 		 * recovery (i.e., ignore this transition for link state
1073 		 * special handling purposes) without updating lastibcstat.
1074 		 */
1075 		if (handle && qib_6120_phys_portstate(ibcs) ==
1076 					    IB_PHYSPORTSTATE_LINK_ERR_RECOVER)
1077 			handle = 0;
1078 		if (handle)
1079 			qib_handle_e_ibstatuschanged(ppd, ibcs);
1080 	}
1081 
1082 	if (errs & ERR_MASK(ResetNegated)) {
1083 		qib_dev_err(dd,
1084 			"Got reset, requires re-init (unload and reload driver)\n");
1085 		dd->flags &= ~QIB_INITTED;  /* needs re-init */
1086 		/* mark as having had error */
1087 		*dd->devstatusp |= QIB_STATUS_HWERROR;
1088 		*dd->pport->statusp &= ~QIB_STATUS_IB_CONF;
1089 	}
1090 
1091 	if (*msg && iserr)
1092 		qib_dev_porterr(dd, ppd->port, "%s error\n", msg);
1093 
1094 	if (ppd->state_wanted & ppd->lflags)
1095 		wake_up_interruptible(&ppd->state_wait);
1096 
1097 	/*
1098 	 * If there were hdrq or egrfull errors, wake up any processes
1099 	 * waiting in poll.  We used to try to check which contexts had
1100 	 * the overflow, but given the cost of that and the chip reads
1101 	 * to support it, it's better to just wake everybody up if we
1102 	 * get an overflow; waiters can poll again if it's not them.
1103 	 */
1104 	if (errs & (ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr))) {
1105 		qib_handle_urcv(dd, ~0U);
1106 		if (errs & ERR_MASK(RcvEgrFullErr))
1107 			qib_stats.sps_buffull++;
1108 		else
1109 			qib_stats.sps_hdrfull++;
1110 	}
1111 done:
1112 	return;
1113 }
1114 
1115 /**
1116  * qib_6120_init_hwerrors - enable hardware errors
1117  * @dd: the qlogic_ib device
1118  *
1119  * now that we have finished initializing everything that might reasonably
1120  * cause a hardware error, and cleared those errors bits as they occur,
1121  * we can enable hardware errors in the mask (potentially enabling
1122  * freeze mode), and enable hardware errors as errors (along with
1123  * everything else) in errormask
1124  */
1125 static void qib_6120_init_hwerrors(struct qib_devdata *dd)
1126 {
1127 	u64 val;
1128 	u64 extsval;
1129 
1130 	extsval = qib_read_kreg64(dd, kr_extstatus);
1131 
1132 	if (!(extsval & QLOGIC_IB_EXTS_MEMBIST_ENDTEST))
1133 		qib_dev_err(dd, "MemBIST did not complete!\n");
1134 
1135 	/* init so all hwerrors interrupt, and enter freeze, ajdust below */
1136 	val = ~0ULL;
1137 	if (dd->minrev < 2) {
1138 		/*
1139 		 * Avoid problem with internal interface bus parity
1140 		 * checking. Fixed in Rev2.
1141 		 */
1142 		val &= ~QLOGIC_IB_HWE_PCIEBUSPARITYRADM;
1143 	}
1144 	/* avoid some intel cpu's speculative read freeze mode issue */
1145 	val &= ~TXEMEMPARITYERR_PIOBUF;
1146 
1147 	dd->cspec->hwerrmask = val;
1148 
1149 	qib_write_kreg(dd, kr_hwerrclear, ~HWE_MASK(PowerOnBISTFailed));
1150 	qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1151 
1152 	/* clear all */
1153 	qib_write_kreg(dd, kr_errclear, ~0ULL);
1154 	/* enable errors that are masked, at least this first time. */
1155 	qib_write_kreg(dd, kr_errmask, ~0ULL);
1156 	dd->cspec->errormask = qib_read_kreg64(dd, kr_errmask);
1157 	/* clear any interrupts up to this point (ints still not enabled) */
1158 	qib_write_kreg(dd, kr_intclear, ~0ULL);
1159 
1160 	qib_write_kreg(dd, kr_rcvbthqp,
1161 		       dd->qpn_mask << (QIB_6120_RcvBTHQP_BTHQP_Mask_LSB - 1) |
1162 		       QIB_KD_QP);
1163 }
1164 
1165 /*
1166  * Disable and enable the armlaunch error.  Used for PIO bandwidth testing
1167  * on chips that are count-based, rather than trigger-based.  There is no
1168  * reference counting, but that's also fine, given the intended use.
1169  * Only chip-specific because it's all register accesses
1170  */
1171 static void qib_set_6120_armlaunch(struct qib_devdata *dd, u32 enable)
1172 {
1173 	if (enable) {
1174 		qib_write_kreg(dd, kr_errclear,
1175 			       ERR_MASK(SendPioArmLaunchErr));
1176 		dd->cspec->errormask |= ERR_MASK(SendPioArmLaunchErr);
1177 	} else
1178 		dd->cspec->errormask &= ~ERR_MASK(SendPioArmLaunchErr);
1179 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1180 }
1181 
1182 /*
1183  * Formerly took parameter <which> in pre-shifted,
1184  * pre-merged form with LinkCmd and LinkInitCmd
1185  * together, and assuming the zero was NOP.
1186  */
1187 static void qib_set_ib_6120_lstate(struct qib_pportdata *ppd, u16 linkcmd,
1188 				   u16 linitcmd)
1189 {
1190 	u64 mod_wd;
1191 	struct qib_devdata *dd = ppd->dd;
1192 	unsigned long flags;
1193 
1194 	if (linitcmd == QLOGIC_IB_IBCC_LINKINITCMD_DISABLE) {
1195 		/*
1196 		 * If we are told to disable, note that so link-recovery
1197 		 * code does not attempt to bring us back up.
1198 		 */
1199 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1200 		ppd->lflags |= QIBL_IB_LINK_DISABLED;
1201 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1202 	} else if (linitcmd || linkcmd == QLOGIC_IB_IBCC_LINKCMD_DOWN) {
1203 		/*
1204 		 * Any other linkinitcmd will lead to LINKDOWN and then
1205 		 * to INIT (if all is well), so clear flag to let
1206 		 * link-recovery code attempt to bring us back up.
1207 		 */
1208 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1209 		ppd->lflags &= ~QIBL_IB_LINK_DISABLED;
1210 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1211 	}
1212 
1213 	mod_wd = (linkcmd << QLOGIC_IB_IBCC_LINKCMD_SHIFT) |
1214 		(linitcmd << QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1215 
1216 	qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl | mod_wd);
1217 	/* write to chip to prevent back-to-back writes of control reg */
1218 	qib_write_kreg(dd, kr_scratch, 0);
1219 }
1220 
1221 /**
1222  * qib_6120_bringup_serdes - bring up the serdes
1223  * @ppd: the qlogic_ib device
1224  */
1225 static int qib_6120_bringup_serdes(struct qib_pportdata *ppd)
1226 {
1227 	struct qib_devdata *dd = ppd->dd;
1228 	u64 val, config1, prev_val, hwstat, ibc;
1229 
1230 	/* Put IBC in reset, sends disabled */
1231 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1232 	qib_write_kreg(dd, kr_control, 0ULL);
1233 
1234 	dd->cspec->ibdeltainprog = 1;
1235 	dd->cspec->ibsymsnap = read_6120_creg32(dd, cr_ibsymbolerr);
1236 	dd->cspec->iblnkerrsnap = read_6120_creg32(dd, cr_iblinkerrrecov);
1237 
1238 	/* flowcontrolwatermark is in units of KBytes */
1239 	ibc = 0x5ULL << SYM_LSB(IBCCtrl, FlowCtrlWaterMark);
1240 	/*
1241 	 * How often flowctrl sent.  More or less in usecs; balance against
1242 	 * watermark value, so that in theory senders always get a flow
1243 	 * control update in time to not let the IB link go idle.
1244 	 */
1245 	ibc |= 0x3ULL << SYM_LSB(IBCCtrl, FlowCtrlPeriod);
1246 	/* max error tolerance */
1247 	dd->cspec->lli_thresh = 0xf;
1248 	ibc |= (u64) dd->cspec->lli_thresh << SYM_LSB(IBCCtrl, PhyerrThreshold);
1249 	/* use "real" buffer space for */
1250 	ibc |= 4ULL << SYM_LSB(IBCCtrl, CreditScale);
1251 	/* IB credit flow control. */
1252 	ibc |= 0xfULL << SYM_LSB(IBCCtrl, OverrunThreshold);
1253 	/*
1254 	 * set initial max size pkt IBC will send, including ICRC; it's the
1255 	 * PIO buffer size in dwords, less 1; also see qib_set_mtu()
1256 	 */
1257 	ibc |= ((u64)(ppd->ibmaxlen >> 2) + 1) << SYM_LSB(IBCCtrl, MaxPktLen);
1258 	dd->cspec->ibcctrl = ibc; /* without linkcmd or linkinitcmd! */
1259 
1260 	/* initially come up waiting for TS1, without sending anything. */
1261 	val = dd->cspec->ibcctrl | (QLOGIC_IB_IBCC_LINKINITCMD_DISABLE <<
1262 		QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1263 	qib_write_kreg(dd, kr_ibcctrl, val);
1264 
1265 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1266 	config1 = qib_read_kreg64(dd, kr_serdes_cfg1);
1267 
1268 	/*
1269 	 * Force reset on, also set rxdetect enable.  Must do before reading
1270 	 * serdesstatus at least for simulation, or some of the bits in
1271 	 * serdes status will come back as undefined and cause simulation
1272 	 * failures
1273 	 */
1274 	val |= SYM_MASK(SerdesCfg0, ResetPLL) |
1275 		SYM_MASK(SerdesCfg0, RxDetEnX) |
1276 		(SYM_MASK(SerdesCfg0, L1PwrDnA) |
1277 		 SYM_MASK(SerdesCfg0, L1PwrDnB) |
1278 		 SYM_MASK(SerdesCfg0, L1PwrDnC) |
1279 		 SYM_MASK(SerdesCfg0, L1PwrDnD));
1280 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1281 	/* be sure chip saw it */
1282 	qib_read_kreg64(dd, kr_scratch);
1283 	udelay(5);              /* need pll reset set at least for a bit */
1284 	/*
1285 	 * after PLL is reset, set the per-lane Resets and TxIdle and
1286 	 * clear the PLL reset and rxdetect (to get falling edge).
1287 	 * Leave L1PWR bits set (permanently)
1288 	 */
1289 	val &= ~(SYM_MASK(SerdesCfg0, RxDetEnX) |
1290 		 SYM_MASK(SerdesCfg0, ResetPLL) |
1291 		 (SYM_MASK(SerdesCfg0, L1PwrDnA) |
1292 		  SYM_MASK(SerdesCfg0, L1PwrDnB) |
1293 		  SYM_MASK(SerdesCfg0, L1PwrDnC) |
1294 		  SYM_MASK(SerdesCfg0, L1PwrDnD)));
1295 	val |= (SYM_MASK(SerdesCfg0, ResetA) |
1296 		SYM_MASK(SerdesCfg0, ResetB) |
1297 		SYM_MASK(SerdesCfg0, ResetC) |
1298 		SYM_MASK(SerdesCfg0, ResetD)) |
1299 		SYM_MASK(SerdesCfg0, TxIdeEnX);
1300 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1301 	/* be sure chip saw it */
1302 	(void) qib_read_kreg64(dd, kr_scratch);
1303 	/* need PLL reset clear for at least 11 usec before lane
1304 	 * resets cleared; give it a few more to be sure */
1305 	udelay(15);
1306 	val &= ~((SYM_MASK(SerdesCfg0, ResetA) |
1307 		  SYM_MASK(SerdesCfg0, ResetB) |
1308 		  SYM_MASK(SerdesCfg0, ResetC) |
1309 		  SYM_MASK(SerdesCfg0, ResetD)) |
1310 		 SYM_MASK(SerdesCfg0, TxIdeEnX));
1311 
1312 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1313 	/* be sure chip saw it */
1314 	(void) qib_read_kreg64(dd, kr_scratch);
1315 
1316 	val = qib_read_kreg64(dd, kr_xgxs_cfg);
1317 	prev_val = val;
1318 	if (val & QLOGIC_IB_XGXS_RESET)
1319 		val &= ~QLOGIC_IB_XGXS_RESET;
1320 	if (SYM_FIELD(val, XGXSCfg, polarity_inv) != ppd->rx_pol_inv) {
1321 		/* need to compensate for Tx inversion in partner */
1322 		val &= ~SYM_MASK(XGXSCfg, polarity_inv);
1323 		val |= (u64)ppd->rx_pol_inv << SYM_LSB(XGXSCfg, polarity_inv);
1324 	}
1325 	if (val != prev_val)
1326 		qib_write_kreg(dd, kr_xgxs_cfg, val);
1327 
1328 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1329 
1330 	/* clear current and de-emphasis bits */
1331 	config1 &= ~0x0ffffffff00ULL;
1332 	/* set current to 20ma */
1333 	config1 |= 0x00000000000ULL;
1334 	/* set de-emphasis to -5.68dB */
1335 	config1 |= 0x0cccc000000ULL;
1336 	qib_write_kreg(dd, kr_serdes_cfg1, config1);
1337 
1338 	/* base and port guid same for single port */
1339 	ppd->guid = dd->base_guid;
1340 
1341 	/*
1342 	 * the process of setting and un-resetting the serdes normally
1343 	 * causes a serdes PLL error, so check for that and clear it
1344 	 * here.  Also clearr hwerr bit in errstatus, but not others.
1345 	 */
1346 	hwstat = qib_read_kreg64(dd, kr_hwerrstatus);
1347 	if (hwstat) {
1348 		/* should just have PLL, clear all set, in an case */
1349 		qib_write_kreg(dd, kr_hwerrclear, hwstat);
1350 		qib_write_kreg(dd, kr_errclear, ERR_MASK(HardwareErr));
1351 	}
1352 
1353 	dd->control |= QLOGIC_IB_C_LINKENABLE;
1354 	dd->control &= ~QLOGIC_IB_C_FREEZEMODE;
1355 	qib_write_kreg(dd, kr_control, dd->control);
1356 
1357 	return 0;
1358 }
1359 
1360 /**
1361  * qib_6120_quiet_serdes - set serdes to txidle
1362  * @ppd: physical port of the qlogic_ib device
1363  * Called when driver is being unloaded
1364  */
1365 static void qib_6120_quiet_serdes(struct qib_pportdata *ppd)
1366 {
1367 	struct qib_devdata *dd = ppd->dd;
1368 	u64 val;
1369 
1370 	qib_set_ib_6120_lstate(ppd, 0, QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1371 
1372 	/* disable IBC */
1373 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1374 	qib_write_kreg(dd, kr_control,
1375 		       dd->control | QLOGIC_IB_C_FREEZEMODE);
1376 
1377 	if (dd->cspec->ibsymdelta || dd->cspec->iblnkerrdelta ||
1378 	    dd->cspec->ibdeltainprog) {
1379 		u64 diagc;
1380 
1381 		/* enable counter writes */
1382 		diagc = qib_read_kreg64(dd, kr_hwdiagctrl);
1383 		qib_write_kreg(dd, kr_hwdiagctrl,
1384 			       diagc | SYM_MASK(HwDiagCtrl, CounterWrEnable));
1385 
1386 		if (dd->cspec->ibsymdelta || dd->cspec->ibdeltainprog) {
1387 			val = read_6120_creg32(dd, cr_ibsymbolerr);
1388 			if (dd->cspec->ibdeltainprog)
1389 				val -= val - dd->cspec->ibsymsnap;
1390 			val -= dd->cspec->ibsymdelta;
1391 			write_6120_creg(dd, cr_ibsymbolerr, val);
1392 		}
1393 		if (dd->cspec->iblnkerrdelta || dd->cspec->ibdeltainprog) {
1394 			val = read_6120_creg32(dd, cr_iblinkerrrecov);
1395 			if (dd->cspec->ibdeltainprog)
1396 				val -= val - dd->cspec->iblnkerrsnap;
1397 			val -= dd->cspec->iblnkerrdelta;
1398 			write_6120_creg(dd, cr_iblinkerrrecov, val);
1399 		}
1400 
1401 		/* and disable counter writes */
1402 		qib_write_kreg(dd, kr_hwdiagctrl, diagc);
1403 	}
1404 
1405 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1406 	val |= SYM_MASK(SerdesCfg0, TxIdeEnX);
1407 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1408 }
1409 
1410 /**
1411  * qib_6120_setup_setextled - set the state of the two external LEDs
1412  * @ppd: the qlogic_ib device
1413  * @on: whether the link is up or not
1414  *
1415  * The exact combo of LEDs if on is true is determined by looking
1416  * at the ibcstatus.
1417  * These LEDs indicate the physical and logical state of IB link.
1418  * For this chip (at least with recommended board pinouts), LED1
1419  * is Yellow (logical state) and LED2 is Green (physical state),
1420  *
1421  * Note:  We try to match the Mellanox HCA LED behavior as best
1422  * we can.  Green indicates physical link state is OK (something is
1423  * plugged in, and we can train).
1424  * Amber indicates the link is logically up (ACTIVE).
1425  * Mellanox further blinks the amber LED to indicate data packet
1426  * activity, but we have no hardware support for that, so it would
1427  * require waking up every 10-20 msecs and checking the counters
1428  * on the chip, and then turning the LED off if appropriate.  That's
1429  * visible overhead, so not something we will do.
1430  *
1431  */
1432 static void qib_6120_setup_setextled(struct qib_pportdata *ppd, u32 on)
1433 {
1434 	u64 extctl, val, lst, ltst;
1435 	unsigned long flags;
1436 	struct qib_devdata *dd = ppd->dd;
1437 
1438 	/*
1439 	 * The diags use the LED to indicate diag info, so we leave
1440 	 * the external LED alone when the diags are running.
1441 	 */
1442 	if (dd->diag_client)
1443 		return;
1444 
1445 	/* Allow override of LED display for, e.g. Locating system in rack */
1446 	if (ppd->led_override) {
1447 		ltst = (ppd->led_override & QIB_LED_PHYS) ?
1448 			IB_PHYSPORTSTATE_LINKUP : IB_PHYSPORTSTATE_DISABLED,
1449 		lst = (ppd->led_override & QIB_LED_LOG) ?
1450 			IB_PORT_ACTIVE : IB_PORT_DOWN;
1451 	} else if (on) {
1452 		val = qib_read_kreg64(dd, kr_ibcstatus);
1453 		ltst = qib_6120_phys_portstate(val);
1454 		lst = qib_6120_iblink_state(val);
1455 	} else {
1456 		ltst = 0;
1457 		lst = 0;
1458 	}
1459 
1460 	spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
1461 	extctl = dd->cspec->extctrl & ~(SYM_MASK(EXTCtrl, LEDPriPortGreenOn) |
1462 				 SYM_MASK(EXTCtrl, LEDPriPortYellowOn));
1463 
1464 	if (ltst == IB_PHYSPORTSTATE_LINKUP)
1465 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortYellowOn);
1466 	if (lst == IB_PORT_ACTIVE)
1467 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortGreenOn);
1468 	dd->cspec->extctrl = extctl;
1469 	qib_write_kreg(dd, kr_extctrl, extctl);
1470 	spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
1471 }
1472 
1473 /**
1474  * qib_6120_setup_cleanup - clean up any per-chip chip-specific stuff
1475  * @dd: the qlogic_ib device
1476  *
1477  * This is called during driver unload.
1478 */
1479 static void qib_6120_setup_cleanup(struct qib_devdata *dd)
1480 {
1481 	qib_free_irq(dd);
1482 	kfree(dd->cspec->cntrs);
1483 	kfree(dd->cspec->portcntrs);
1484 	if (dd->cspec->dummy_hdrq) {
1485 		dma_free_coherent(&dd->pcidev->dev,
1486 				  ALIGN(dd->rcvhdrcnt *
1487 					dd->rcvhdrentsize *
1488 					sizeof(u32), PAGE_SIZE),
1489 				  dd->cspec->dummy_hdrq,
1490 				  dd->cspec->dummy_hdrq_phys);
1491 		dd->cspec->dummy_hdrq = NULL;
1492 	}
1493 }
1494 
1495 static void qib_wantpiobuf_6120_intr(struct qib_devdata *dd, u32 needint)
1496 {
1497 	unsigned long flags;
1498 
1499 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
1500 	if (needint)
1501 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOIntBufAvail);
1502 	else
1503 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOIntBufAvail);
1504 	qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
1505 	qib_write_kreg(dd, kr_scratch, 0ULL);
1506 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
1507 }
1508 
1509 /*
1510  * handle errors and unusual events first, separate function
1511  * to improve cache hits for fast path interrupt handling
1512  */
1513 static noinline void unlikely_6120_intr(struct qib_devdata *dd, u64 istat)
1514 {
1515 	if (unlikely(istat & ~QLOGIC_IB_I_BITSEXTANT))
1516 		qib_dev_err(dd, "interrupt with unknown interrupts %Lx set\n",
1517 			    istat & ~QLOGIC_IB_I_BITSEXTANT);
1518 
1519 	if (istat & QLOGIC_IB_I_ERROR) {
1520 		u64 estat = 0;
1521 
1522 		qib_stats.sps_errints++;
1523 		estat = qib_read_kreg64(dd, kr_errstatus);
1524 		if (!estat)
1525 			qib_devinfo(dd->pcidev,
1526 				"error interrupt (%Lx), but no error bits set!\n",
1527 				istat);
1528 		handle_6120_errors(dd, estat);
1529 	}
1530 
1531 	if (istat & QLOGIC_IB_I_GPIO) {
1532 		u32 gpiostatus;
1533 		u32 to_clear = 0;
1534 
1535 		/*
1536 		 * GPIO_3..5 on IBA6120 Rev2 chips indicate
1537 		 * errors that we need to count.
1538 		 */
1539 		gpiostatus = qib_read_kreg32(dd, kr_gpio_status);
1540 		/* First the error-counter case. */
1541 		if (gpiostatus & GPIO_ERRINTR_MASK) {
1542 			/* want to clear the bits we see asserted. */
1543 			to_clear |= (gpiostatus & GPIO_ERRINTR_MASK);
1544 
1545 			/*
1546 			 * Count appropriately, clear bits out of our copy,
1547 			 * as they have been "handled".
1548 			 */
1549 			if (gpiostatus & (1 << GPIO_RXUVL_BIT))
1550 				dd->cspec->rxfc_unsupvl_errs++;
1551 			if (gpiostatus & (1 << GPIO_OVRUN_BIT))
1552 				dd->cspec->overrun_thresh_errs++;
1553 			if (gpiostatus & (1 << GPIO_LLI_BIT))
1554 				dd->cspec->lli_errs++;
1555 			gpiostatus &= ~GPIO_ERRINTR_MASK;
1556 		}
1557 		if (gpiostatus) {
1558 			/*
1559 			 * Some unexpected bits remain. If they could have
1560 			 * caused the interrupt, complain and clear.
1561 			 * To avoid repetition of this condition, also clear
1562 			 * the mask. It is almost certainly due to error.
1563 			 */
1564 			const u32 mask = qib_read_kreg32(dd, kr_gpio_mask);
1565 
1566 			/*
1567 			 * Also check that the chip reflects our shadow,
1568 			 * and report issues, If they caused the interrupt.
1569 			 * we will suppress by refreshing from the shadow.
1570 			 */
1571 			if (mask & gpiostatus) {
1572 				to_clear |= (gpiostatus & mask);
1573 				dd->cspec->gpio_mask &= ~(gpiostatus & mask);
1574 				qib_write_kreg(dd, kr_gpio_mask,
1575 					       dd->cspec->gpio_mask);
1576 			}
1577 		}
1578 		if (to_clear)
1579 			qib_write_kreg(dd, kr_gpio_clear, (u64) to_clear);
1580 	}
1581 }
1582 
1583 static irqreturn_t qib_6120intr(int irq, void *data)
1584 {
1585 	struct qib_devdata *dd = data;
1586 	irqreturn_t ret;
1587 	u32 istat, ctxtrbits, rmask, crcs = 0;
1588 	unsigned i;
1589 
1590 	if ((dd->flags & (QIB_PRESENT | QIB_BADINTR)) != QIB_PRESENT) {
1591 		/*
1592 		 * This return value is not great, but we do not want the
1593 		 * interrupt core code to remove our interrupt handler
1594 		 * because we don't appear to be handling an interrupt
1595 		 * during a chip reset.
1596 		 */
1597 		ret = IRQ_HANDLED;
1598 		goto bail;
1599 	}
1600 
1601 	istat = qib_read_kreg32(dd, kr_intstatus);
1602 
1603 	if (unlikely(!istat)) {
1604 		ret = IRQ_NONE; /* not our interrupt, or already handled */
1605 		goto bail;
1606 	}
1607 	if (unlikely(istat == -1)) {
1608 		qib_bad_intrstatus(dd);
1609 		/* don't know if it was our interrupt or not */
1610 		ret = IRQ_NONE;
1611 		goto bail;
1612 	}
1613 
1614 	this_cpu_inc(*dd->int_counter);
1615 
1616 	if (unlikely(istat & (~QLOGIC_IB_I_BITSEXTANT |
1617 			      QLOGIC_IB_I_GPIO | QLOGIC_IB_I_ERROR)))
1618 		unlikely_6120_intr(dd, istat);
1619 
1620 	/*
1621 	 * Clear the interrupt bits we found set, relatively early, so we
1622 	 * "know" know the chip will have seen this by the time we process
1623 	 * the queue, and will re-interrupt if necessary.  The processor
1624 	 * itself won't take the interrupt again until we return.
1625 	 */
1626 	qib_write_kreg(dd, kr_intclear, istat);
1627 
1628 	/*
1629 	 * Handle kernel receive queues before checking for pio buffers
1630 	 * available since receives can overflow; piobuf waiters can afford
1631 	 * a few extra cycles, since they were waiting anyway.
1632 	 */
1633 	ctxtrbits = istat &
1634 		((QLOGIC_IB_I_RCVAVAIL_MASK << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1635 		 (QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT));
1636 	if (ctxtrbits) {
1637 		rmask = (1U << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1638 			(1U << QLOGIC_IB_I_RCVURG_SHIFT);
1639 		for (i = 0; i < dd->first_user_ctxt; i++) {
1640 			if (ctxtrbits & rmask) {
1641 				ctxtrbits &= ~rmask;
1642 				crcs += qib_kreceive(dd->rcd[i],
1643 						     &dd->cspec->lli_counter,
1644 						     NULL);
1645 			}
1646 			rmask <<= 1;
1647 		}
1648 		if (crcs) {
1649 			u32 cntr = dd->cspec->lli_counter;
1650 
1651 			cntr += crcs;
1652 			if (cntr) {
1653 				if (cntr > dd->cspec->lli_thresh) {
1654 					dd->cspec->lli_counter = 0;
1655 					dd->cspec->lli_errs++;
1656 				} else
1657 					dd->cspec->lli_counter += cntr;
1658 			}
1659 		}
1660 
1661 
1662 		if (ctxtrbits) {
1663 			ctxtrbits =
1664 				(ctxtrbits >> QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1665 				(ctxtrbits >> QLOGIC_IB_I_RCVURG_SHIFT);
1666 			qib_handle_urcv(dd, ctxtrbits);
1667 		}
1668 	}
1669 
1670 	if ((istat & QLOGIC_IB_I_SPIOBUFAVAIL) && (dd->flags & QIB_INITTED))
1671 		qib_ib_piobufavail(dd);
1672 
1673 	ret = IRQ_HANDLED;
1674 bail:
1675 	return ret;
1676 }
1677 
1678 /*
1679  * Set up our chip-specific interrupt handler
1680  * The interrupt type has already been setup, so
1681  * we just need to do the registration and error checking.
1682  */
1683 static void qib_setup_6120_interrupt(struct qib_devdata *dd)
1684 {
1685 	int ret;
1686 
1687 	/*
1688 	 * If the chip supports added error indication via GPIO pins,
1689 	 * enable interrupts on those bits so the interrupt routine
1690 	 * can count the events. Also set flag so interrupt routine
1691 	 * can know they are expected.
1692 	 */
1693 	if (SYM_FIELD(dd->revision, Revision_R,
1694 		      ChipRevMinor) > 1) {
1695 		/* Rev2+ reports extra errors via internal GPIO pins */
1696 		dd->cspec->gpio_mask |= GPIO_ERRINTR_MASK;
1697 		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1698 	}
1699 
1700 	ret = pci_request_irq(dd->pcidev, 0, qib_6120intr, NULL, dd,
1701 			      QIB_DRV_NAME);
1702 	if (ret)
1703 		qib_dev_err(dd,
1704 			    "Couldn't setup interrupt (irq=%d): %d\n",
1705 			    pci_irq_vector(dd->pcidev, 0), ret);
1706 }
1707 
1708 /**
1709  * pe_boardname - fill in the board name
1710  * @dd: the qlogic_ib device
1711  *
1712  * info is based on the board revision register
1713  */
1714 static void pe_boardname(struct qib_devdata *dd)
1715 {
1716 	u32 boardid;
1717 
1718 	boardid = SYM_FIELD(dd->revision, Revision,
1719 			    BoardID);
1720 
1721 	switch (boardid) {
1722 	case 2:
1723 		dd->boardname = "InfiniPath_QLE7140";
1724 		break;
1725 	default:
1726 		qib_dev_err(dd, "Unknown 6120 board with ID %u\n", boardid);
1727 		dd->boardname = "Unknown_InfiniPath_6120";
1728 		break;
1729 	}
1730 
1731 	if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
1732 		qib_dev_err(dd,
1733 			    "Unsupported InfiniPath hardware revision %u.%u!\n",
1734 			    dd->majrev, dd->minrev);
1735 
1736 	snprintf(dd->boardversion, sizeof(dd->boardversion),
1737 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
1738 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
1739 		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, Arch),
1740 		 dd->majrev, dd->minrev,
1741 		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, SW));
1742 }
1743 
1744 /*
1745  * This routine sleeps, so it can only be called from user context, not
1746  * from interrupt context.  If we need interrupt context, we can split
1747  * it into two routines.
1748  */
1749 static int qib_6120_setup_reset(struct qib_devdata *dd)
1750 {
1751 	u64 val;
1752 	int i;
1753 	int ret;
1754 	u16 cmdval;
1755 	u8 int_line, clinesz;
1756 
1757 	qib_pcie_getcmd(dd, &cmdval, &int_line, &clinesz);
1758 
1759 	/* Use ERROR so it shows up in logs, etc. */
1760 	qib_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->unit);
1761 
1762 	/* no interrupts till re-initted */
1763 	qib_6120_set_intr_state(dd, 0);
1764 
1765 	dd->cspec->ibdeltainprog = 0;
1766 	dd->cspec->ibsymdelta = 0;
1767 	dd->cspec->iblnkerrdelta = 0;
1768 
1769 	/*
1770 	 * Keep chip from being accessed until we are ready.  Use
1771 	 * writeq() directly, to allow the write even though QIB_PRESENT
1772 	 * isn't set.
1773 	 */
1774 	dd->flags &= ~(QIB_INITTED | QIB_PRESENT);
1775 	/* so we check interrupts work again */
1776 	dd->z_int_counter = qib_int_counter(dd);
1777 	val = dd->control | QLOGIC_IB_C_RESET;
1778 	writeq(val, &dd->kregbase[kr_control]);
1779 	mb(); /* prevent compiler re-ordering around actual reset */
1780 
1781 	for (i = 1; i <= 5; i++) {
1782 		/*
1783 		 * Allow MBIST, etc. to complete; longer on each retry.
1784 		 * We sometimes get machine checks from bus timeout if no
1785 		 * response, so for now, make it *really* long.
1786 		 */
1787 		msleep(1000 + (1 + i) * 2000);
1788 
1789 		qib_pcie_reenable(dd, cmdval, int_line, clinesz);
1790 
1791 		/*
1792 		 * Use readq directly, so we don't need to mark it as PRESENT
1793 		 * until we get a successful indication that all is well.
1794 		 */
1795 		val = readq(&dd->kregbase[kr_revision]);
1796 		if (val == dd->revision) {
1797 			dd->flags |= QIB_PRESENT; /* it's back */
1798 			ret = qib_reinit_intr(dd);
1799 			goto bail;
1800 		}
1801 	}
1802 	ret = 0; /* failed */
1803 
1804 bail:
1805 	if (ret) {
1806 		if (qib_pcie_params(dd, dd->lbus_width, NULL))
1807 			qib_dev_err(dd,
1808 				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
1809 		/* clear the reset error, init error/hwerror mask */
1810 		qib_6120_init_hwerrors(dd);
1811 		/* for Rev2 error interrupts; nop for rev 1 */
1812 		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1813 		/* clear the reset error, init error/hwerror mask */
1814 		qib_6120_init_hwerrors(dd);
1815 	}
1816 	return ret;
1817 }
1818 
1819 /**
1820  * qib_6120_put_tid - write a TID in chip
1821  * @dd: the qlogic_ib device
1822  * @tidptr: pointer to the expected TID (in chip) to update
1823  * @type: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1824  * for expected
1825  * @pa: physical address of in memory buffer; tidinvalid if freeing
1826  *
1827  * This exists as a separate routine to allow for special locking etc.
1828  * It's used for both the full cleanup on exit, as well as the normal
1829  * setup and teardown.
1830  */
1831 static void qib_6120_put_tid(struct qib_devdata *dd, u64 __iomem *tidptr,
1832 			     u32 type, unsigned long pa)
1833 {
1834 	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1835 	unsigned long flags;
1836 	int tidx;
1837 	spinlock_t *tidlockp; /* select appropriate spinlock */
1838 
1839 	if (!dd->kregbase)
1840 		return;
1841 
1842 	if (pa != dd->tidinvalid) {
1843 		if (pa & ((1U << 11) - 1)) {
1844 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1845 				    pa);
1846 			return;
1847 		}
1848 		pa >>= 11;
1849 		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1850 			qib_dev_err(dd,
1851 				"Physical page address 0x%lx larger than supported\n",
1852 				pa);
1853 			return;
1854 		}
1855 
1856 		if (type == RCVHQ_RCV_TYPE_EAGER)
1857 			pa |= dd->tidtemplate;
1858 		else /* for now, always full 4KB page */
1859 			pa |= 2 << 29;
1860 	}
1861 
1862 	/*
1863 	 * Avoid chip issue by writing the scratch register
1864 	 * before and after the TID, and with an io write barrier.
1865 	 * We use a spinlock around the writes, so they can't intermix
1866 	 * with other TID (eager or expected) writes (the chip problem
1867 	 * is triggered by back to back TID writes). Unfortunately, this
1868 	 * call can be done from interrupt level for the ctxt 0 eager TIDs,
1869 	 * so we have to use irqsave locks.
1870 	 */
1871 	/*
1872 	 * Assumes tidptr always > egrtidbase
1873 	 * if type == RCVHQ_RCV_TYPE_EAGER.
1874 	 */
1875 	tidx = tidptr - dd->egrtidbase;
1876 
1877 	tidlockp = (type == RCVHQ_RCV_TYPE_EAGER && tidx < dd->rcvhdrcnt)
1878 		? &dd->cspec->kernel_tid_lock : &dd->cspec->user_tid_lock;
1879 	spin_lock_irqsave(tidlockp, flags);
1880 	qib_write_kreg(dd, kr_scratch, 0xfeeddeaf);
1881 	writel(pa, tidp32);
1882 	qib_write_kreg(dd, kr_scratch, 0xdeadbeef);
1883 	spin_unlock_irqrestore(tidlockp, flags);
1884 }
1885 
1886 /**
1887  * qib_6120_put_tid_2 - write a TID in chip, Revision 2 or higher
1888  * @dd: the qlogic_ib device
1889  * @tidptr: pointer to the expected TID (in chip) to update
1890  * @type: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1891  * for expected
1892  * @pa: physical address of in memory buffer; tidinvalid if freeing
1893  *
1894  * This exists as a separate routine to allow for selection of the
1895  * appropriate "flavor". The static calls in cleanup just use the
1896  * revision-agnostic form, as they are not performance critical.
1897  */
1898 static void qib_6120_put_tid_2(struct qib_devdata *dd, u64 __iomem *tidptr,
1899 			       u32 type, unsigned long pa)
1900 {
1901 	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1902 
1903 	if (!dd->kregbase)
1904 		return;
1905 
1906 	if (pa != dd->tidinvalid) {
1907 		if (pa & ((1U << 11) - 1)) {
1908 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1909 				    pa);
1910 			return;
1911 		}
1912 		pa >>= 11;
1913 		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1914 			qib_dev_err(dd,
1915 				"Physical page address 0x%lx larger than supported\n",
1916 				pa);
1917 			return;
1918 		}
1919 
1920 		if (type == RCVHQ_RCV_TYPE_EAGER)
1921 			pa |= dd->tidtemplate;
1922 		else /* for now, always full 4KB page */
1923 			pa |= 2 << 29;
1924 	}
1925 	writel(pa, tidp32);
1926 }
1927 
1928 
1929 /**
1930  * qib_6120_clear_tids - clear all TID entries for a context, expected and eager
1931  * @dd: the qlogic_ib device
1932  * @rcd: the context
1933  *
1934  * clear all TID entries for a context, expected and eager.
1935  * Used from qib_close().  On this chip, TIDs are only 32 bits,
1936  * not 64, but they are still on 64 bit boundaries, so tidbase
1937  * is declared as u64 * for the pointer math, even though we write 32 bits
1938  */
1939 static void qib_6120_clear_tids(struct qib_devdata *dd,
1940 				struct qib_ctxtdata *rcd)
1941 {
1942 	u64 __iomem *tidbase;
1943 	unsigned long tidinv;
1944 	u32 ctxt;
1945 	int i;
1946 
1947 	if (!dd->kregbase || !rcd)
1948 		return;
1949 
1950 	ctxt = rcd->ctxt;
1951 
1952 	tidinv = dd->tidinvalid;
1953 	tidbase = (u64 __iomem *)
1954 		((char __iomem *)(dd->kregbase) +
1955 		 dd->rcvtidbase +
1956 		 ctxt * dd->rcvtidcnt * sizeof(*tidbase));
1957 
1958 	for (i = 0; i < dd->rcvtidcnt; i++)
1959 		/* use func pointer because could be one of two funcs */
1960 		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
1961 				  tidinv);
1962 
1963 	tidbase = (u64 __iomem *)
1964 		((char __iomem *)(dd->kregbase) +
1965 		 dd->rcvegrbase +
1966 		 rcd->rcvegr_tid_base * sizeof(*tidbase));
1967 
1968 	for (i = 0; i < rcd->rcvegrcnt; i++)
1969 		/* use func pointer because could be one of two funcs */
1970 		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EAGER,
1971 				  tidinv);
1972 }
1973 
1974 /**
1975  * qib_6120_tidtemplate - setup constants for TID updates
1976  * @dd: the qlogic_ib device
1977  *
1978  * We setup stuff that we use a lot, to avoid calculating each time
1979  */
1980 static void qib_6120_tidtemplate(struct qib_devdata *dd)
1981 {
1982 	u32 egrsize = dd->rcvegrbufsize;
1983 
1984 	/*
1985 	 * For now, we always allocate 4KB buffers (at init) so we can
1986 	 * receive max size packets.  We may want a module parameter to
1987 	 * specify 2KB or 4KB and/or make be per ctxt instead of per device
1988 	 * for those who want to reduce memory footprint.  Note that the
1989 	 * rcvhdrentsize size must be large enough to hold the largest
1990 	 * IB header (currently 96 bytes) that we expect to handle (plus of
1991 	 * course the 2 dwords of RHF).
1992 	 */
1993 	if (egrsize == 2048)
1994 		dd->tidtemplate = 1U << 29;
1995 	else if (egrsize == 4096)
1996 		dd->tidtemplate = 2U << 29;
1997 	dd->tidinvalid = 0;
1998 }
1999 
2000 int __attribute__((weak)) qib_unordered_wc(void)
2001 {
2002 	return 0;
2003 }
2004 
2005 /**
2006  * qib_6120_get_base_info - set chip-specific flags for user code
2007  * @rcd: the qlogic_ib ctxt
2008  * @kinfo: qib_base_info pointer
2009  *
2010  * We set the PCIE flag because the lower bandwidth on PCIe vs
2011  * HyperTransport can affect some user packet algorithms.
2012  */
2013 static int qib_6120_get_base_info(struct qib_ctxtdata *rcd,
2014 				  struct qib_base_info *kinfo)
2015 {
2016 	if (qib_unordered_wc())
2017 		kinfo->spi_runtime_flags |= QIB_RUNTIME_FORCE_WC_ORDER;
2018 
2019 	kinfo->spi_runtime_flags |= QIB_RUNTIME_PCIE |
2020 		QIB_RUNTIME_FORCE_PIOAVAIL | QIB_RUNTIME_PIO_REGSWAPPED;
2021 	return 0;
2022 }
2023 
2024 
2025 static struct qib_message_header *
2026 qib_6120_get_msgheader(struct qib_devdata *dd, __le32 *rhf_addr)
2027 {
2028 	return (struct qib_message_header *)
2029 		&rhf_addr[sizeof(u64) / sizeof(u32)];
2030 }
2031 
2032 static void qib_6120_config_ctxts(struct qib_devdata *dd)
2033 {
2034 	dd->ctxtcnt = qib_read_kreg32(dd, kr_portcnt);
2035 	if (qib_n_krcv_queues > 1) {
2036 		dd->first_user_ctxt = qib_n_krcv_queues * dd->num_pports;
2037 		if (dd->first_user_ctxt > dd->ctxtcnt)
2038 			dd->first_user_ctxt = dd->ctxtcnt;
2039 		dd->qpn_mask = dd->first_user_ctxt <= 2 ? 2 : 6;
2040 	} else
2041 		dd->first_user_ctxt = dd->num_pports;
2042 	dd->n_krcv_queues = dd->first_user_ctxt;
2043 }
2044 
2045 static void qib_update_6120_usrhead(struct qib_ctxtdata *rcd, u64 hd,
2046 				    u32 updegr, u32 egrhd, u32 npkts)
2047 {
2048 	if (updegr)
2049 		qib_write_ureg(rcd->dd, ur_rcvegrindexhead, egrhd, rcd->ctxt);
2050 	qib_write_ureg(rcd->dd, ur_rcvhdrhead, hd, rcd->ctxt);
2051 }
2052 
2053 static u32 qib_6120_hdrqempty(struct qib_ctxtdata *rcd)
2054 {
2055 	u32 head, tail;
2056 
2057 	head = qib_read_ureg32(rcd->dd, ur_rcvhdrhead, rcd->ctxt);
2058 	if (rcd->rcvhdrtail_kvaddr)
2059 		tail = qib_get_rcvhdrtail(rcd);
2060 	else
2061 		tail = qib_read_ureg32(rcd->dd, ur_rcvhdrtail, rcd->ctxt);
2062 	return head == tail;
2063 }
2064 
2065 /*
2066  * Used when we close any ctxt, for DMA already in flight
2067  * at close.  Can't be done until we know hdrq size, so not
2068  * early in chip init.
2069  */
2070 static void alloc_dummy_hdrq(struct qib_devdata *dd)
2071 {
2072 	dd->cspec->dummy_hdrq = dma_alloc_coherent(&dd->pcidev->dev,
2073 					dd->rcd[0]->rcvhdrq_size,
2074 					&dd->cspec->dummy_hdrq_phys,
2075 					GFP_ATOMIC);
2076 	if (!dd->cspec->dummy_hdrq) {
2077 		qib_devinfo(dd->pcidev, "Couldn't allocate dummy hdrq\n");
2078 		/* fallback to just 0'ing */
2079 		dd->cspec->dummy_hdrq_phys = 0UL;
2080 	}
2081 }
2082 
2083 /*
2084  * Modify the RCVCTRL register in chip-specific way. This
2085  * is a function because bit positions and (future) register
2086  * location is chip-specific, but the needed operations are
2087  * generic. <op> is a bit-mask because we often want to
2088  * do multiple modifications.
2089  */
2090 static void rcvctrl_6120_mod(struct qib_pportdata *ppd, unsigned int op,
2091 			     int ctxt)
2092 {
2093 	struct qib_devdata *dd = ppd->dd;
2094 	u64 mask, val;
2095 	unsigned long flags;
2096 
2097 	spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2098 
2099 	if (op & QIB_RCVCTRL_TAILUPD_ENB)
2100 		dd->rcvctrl |= (1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2101 	if (op & QIB_RCVCTRL_TAILUPD_DIS)
2102 		dd->rcvctrl &= ~(1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2103 	if (op & QIB_RCVCTRL_PKEY_ENB)
2104 		dd->rcvctrl &= ~(1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2105 	if (op & QIB_RCVCTRL_PKEY_DIS)
2106 		dd->rcvctrl |= (1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2107 	if (ctxt < 0)
2108 		mask = (1ULL << dd->ctxtcnt) - 1;
2109 	else
2110 		mask = (1ULL << ctxt);
2111 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2112 		/* always done for specific ctxt */
2113 		dd->rcvctrl |= (mask << SYM_LSB(RcvCtrl, PortEnable));
2114 		if (!(dd->flags & QIB_NODMA_RTAIL))
2115 			dd->rcvctrl |= 1ULL << QLOGIC_IB_R_TAILUPD_SHIFT;
2116 		/* Write these registers before the context is enabled. */
2117 		qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2118 			dd->rcd[ctxt]->rcvhdrqtailaddr_phys);
2119 		qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2120 			dd->rcd[ctxt]->rcvhdrq_phys);
2121 
2122 		if (ctxt == 0 && !dd->cspec->dummy_hdrq)
2123 			alloc_dummy_hdrq(dd);
2124 	}
2125 	if (op & QIB_RCVCTRL_CTXT_DIS)
2126 		dd->rcvctrl &= ~(mask << SYM_LSB(RcvCtrl, PortEnable));
2127 	if (op & QIB_RCVCTRL_INTRAVAIL_ENB)
2128 		dd->rcvctrl |= (mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2129 	if (op & QIB_RCVCTRL_INTRAVAIL_DIS)
2130 		dd->rcvctrl &= ~(mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2131 	qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2132 	if ((op & QIB_RCVCTRL_INTRAVAIL_ENB) && dd->rhdrhead_intr_off) {
2133 		/* arm rcv interrupt */
2134 		val = qib_read_ureg32(dd, ur_rcvhdrhead, ctxt) |
2135 			dd->rhdrhead_intr_off;
2136 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2137 	}
2138 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2139 		/*
2140 		 * Init the context registers also; if we were
2141 		 * disabled, tail and head should both be zero
2142 		 * already from the enable, but since we don't
2143 		 * know, we have to do it explicitly.
2144 		 */
2145 		val = qib_read_ureg32(dd, ur_rcvegrindextail, ctxt);
2146 		qib_write_ureg(dd, ur_rcvegrindexhead, val, ctxt);
2147 
2148 		val = qib_read_ureg32(dd, ur_rcvhdrtail, ctxt);
2149 		dd->rcd[ctxt]->head = val;
2150 		/* If kctxt, interrupt on next receive. */
2151 		if (ctxt < dd->first_user_ctxt)
2152 			val |= dd->rhdrhead_intr_off;
2153 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2154 	}
2155 	if (op & QIB_RCVCTRL_CTXT_DIS) {
2156 		/*
2157 		 * Be paranoid, and never write 0's to these, just use an
2158 		 * unused page.  Of course,
2159 		 * rcvhdraddr points to a large chunk of memory, so this
2160 		 * could still trash things, but at least it won't trash
2161 		 * page 0, and by disabling the ctxt, it should stop "soon",
2162 		 * even if a packet or two is in already in flight after we
2163 		 * disabled the ctxt.  Only 6120 has this issue.
2164 		 */
2165 		if (ctxt >= 0) {
2166 			qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2167 					    dd->cspec->dummy_hdrq_phys);
2168 			qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2169 					    dd->cspec->dummy_hdrq_phys);
2170 		} else {
2171 			unsigned i;
2172 
2173 			for (i = 0; i < dd->cfgctxts; i++) {
2174 				qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr,
2175 					    i, dd->cspec->dummy_hdrq_phys);
2176 				qib_write_kreg_ctxt(dd, kr_rcvhdraddr,
2177 					    i, dd->cspec->dummy_hdrq_phys);
2178 			}
2179 		}
2180 	}
2181 	spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2182 }
2183 
2184 /*
2185  * Modify the SENDCTRL register in chip-specific way. This
2186  * is a function there may be multiple such registers with
2187  * slightly different layouts. Only operations actually used
2188  * are implemented yet.
2189  * Chip requires no back-back sendctrl writes, so write
2190  * scratch register after writing sendctrl
2191  */
2192 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op)
2193 {
2194 	struct qib_devdata *dd = ppd->dd;
2195 	u64 tmp_dd_sendctrl;
2196 	unsigned long flags;
2197 
2198 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
2199 
2200 	/* First the ones that are "sticky", saved in shadow */
2201 	if (op & QIB_SENDCTRL_CLEAR)
2202 		dd->sendctrl = 0;
2203 	if (op & QIB_SENDCTRL_SEND_DIS)
2204 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOEnable);
2205 	else if (op & QIB_SENDCTRL_SEND_ENB)
2206 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOEnable);
2207 	if (op & QIB_SENDCTRL_AVAIL_DIS)
2208 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2209 	else if (op & QIB_SENDCTRL_AVAIL_ENB)
2210 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOBufAvailUpd);
2211 
2212 	if (op & QIB_SENDCTRL_DISARM_ALL) {
2213 		u32 i, last;
2214 
2215 		tmp_dd_sendctrl = dd->sendctrl;
2216 		/*
2217 		 * disarm any that are not yet launched, disabling sends
2218 		 * and updates until done.
2219 		 */
2220 		last = dd->piobcnt2k + dd->piobcnt4k;
2221 		tmp_dd_sendctrl &=
2222 			~(SYM_MASK(SendCtrl, PIOEnable) |
2223 			  SYM_MASK(SendCtrl, PIOBufAvailUpd));
2224 		for (i = 0; i < last; i++) {
2225 			qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl |
2226 				       SYM_MASK(SendCtrl, Disarm) | i);
2227 			qib_write_kreg(dd, kr_scratch, 0);
2228 		}
2229 	}
2230 
2231 	tmp_dd_sendctrl = dd->sendctrl;
2232 
2233 	if (op & QIB_SENDCTRL_FLUSH)
2234 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Abort);
2235 	if (op & QIB_SENDCTRL_DISARM)
2236 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Disarm) |
2237 			((op & QIB_6120_SendCtrl_DisarmPIOBuf_RMASK) <<
2238 			 SYM_LSB(SendCtrl, DisarmPIOBuf));
2239 	if (op & QIB_SENDCTRL_AVAIL_BLIP)
2240 		tmp_dd_sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2241 
2242 	qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl);
2243 	qib_write_kreg(dd, kr_scratch, 0);
2244 
2245 	if (op & QIB_SENDCTRL_AVAIL_BLIP) {
2246 		qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
2247 		qib_write_kreg(dd, kr_scratch, 0);
2248 	}
2249 
2250 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
2251 
2252 	if (op & QIB_SENDCTRL_FLUSH) {
2253 		u32 v;
2254 		/*
2255 		 * ensure writes have hit chip, then do a few
2256 		 * more reads, to allow DMA of pioavail registers
2257 		 * to occur, so in-memory copy is in sync with
2258 		 * the chip.  Not always safe to sleep.
2259 		 */
2260 		v = qib_read_kreg32(dd, kr_scratch);
2261 		qib_write_kreg(dd, kr_scratch, v);
2262 		v = qib_read_kreg32(dd, kr_scratch);
2263 		qib_write_kreg(dd, kr_scratch, v);
2264 		qib_read_kreg32(dd, kr_scratch);
2265 	}
2266 }
2267 
2268 /**
2269  * qib_portcntr_6120 - read a per-port counter
2270  * @ppd: the qlogic_ib device
2271  * @reg: the counter to snapshot
2272  */
2273 static u64 qib_portcntr_6120(struct qib_pportdata *ppd, u32 reg)
2274 {
2275 	u64 ret = 0ULL;
2276 	struct qib_devdata *dd = ppd->dd;
2277 	u16 creg;
2278 	/* 0xffff for unimplemented or synthesized counters */
2279 	static const u16 xlator[] = {
2280 		[QIBPORTCNTR_PKTSEND] = cr_pktsend,
2281 		[QIBPORTCNTR_WORDSEND] = cr_wordsend,
2282 		[QIBPORTCNTR_PSXMITDATA] = 0xffff,
2283 		[QIBPORTCNTR_PSXMITPKTS] = 0xffff,
2284 		[QIBPORTCNTR_PSXMITWAIT] = 0xffff,
2285 		[QIBPORTCNTR_SENDSTALL] = cr_sendstall,
2286 		[QIBPORTCNTR_PKTRCV] = cr_pktrcv,
2287 		[QIBPORTCNTR_PSRCVDATA] = 0xffff,
2288 		[QIBPORTCNTR_PSRCVPKTS] = 0xffff,
2289 		[QIBPORTCNTR_RCVEBP] = cr_rcvebp,
2290 		[QIBPORTCNTR_RCVOVFL] = cr_rcvovfl,
2291 		[QIBPORTCNTR_WORDRCV] = cr_wordrcv,
2292 		[QIBPORTCNTR_RXDROPPKT] = cr_rxdroppkt,
2293 		[QIBPORTCNTR_RXLOCALPHYERR] = 0xffff,
2294 		[QIBPORTCNTR_RXVLERR] = 0xffff,
2295 		[QIBPORTCNTR_ERRICRC] = cr_erricrc,
2296 		[QIBPORTCNTR_ERRVCRC] = cr_errvcrc,
2297 		[QIBPORTCNTR_ERRLPCRC] = cr_errlpcrc,
2298 		[QIBPORTCNTR_BADFORMAT] = cr_badformat,
2299 		[QIBPORTCNTR_ERR_RLEN] = cr_err_rlen,
2300 		[QIBPORTCNTR_IBSYMBOLERR] = cr_ibsymbolerr,
2301 		[QIBPORTCNTR_INVALIDRLEN] = cr_invalidrlen,
2302 		[QIBPORTCNTR_UNSUPVL] = cr_txunsupvl,
2303 		[QIBPORTCNTR_EXCESSBUFOVFL] = 0xffff,
2304 		[QIBPORTCNTR_ERRLINK] = cr_errlink,
2305 		[QIBPORTCNTR_IBLINKDOWN] = cr_iblinkdown,
2306 		[QIBPORTCNTR_IBLINKERRRECOV] = cr_iblinkerrrecov,
2307 		[QIBPORTCNTR_LLI] = 0xffff,
2308 		[QIBPORTCNTR_PSINTERVAL] = 0xffff,
2309 		[QIBPORTCNTR_PSSTART] = 0xffff,
2310 		[QIBPORTCNTR_PSSTAT] = 0xffff,
2311 		[QIBPORTCNTR_VL15PKTDROP] = 0xffff,
2312 		[QIBPORTCNTR_ERRPKEY] = cr_errpkey,
2313 		[QIBPORTCNTR_KHDROVFL] = 0xffff,
2314 	};
2315 
2316 	if (reg >= ARRAY_SIZE(xlator)) {
2317 		qib_devinfo(ppd->dd->pcidev,
2318 			 "Unimplemented portcounter %u\n", reg);
2319 		goto done;
2320 	}
2321 	creg = xlator[reg];
2322 
2323 	/* handle counters requests not implemented as chip counters */
2324 	if (reg == QIBPORTCNTR_LLI)
2325 		ret = dd->cspec->lli_errs;
2326 	else if (reg == QIBPORTCNTR_EXCESSBUFOVFL)
2327 		ret = dd->cspec->overrun_thresh_errs;
2328 	else if (reg == QIBPORTCNTR_KHDROVFL) {
2329 		int i;
2330 
2331 		/* sum over all kernel contexts */
2332 		for (i = 0; i < dd->first_user_ctxt; i++)
2333 			ret += read_6120_creg32(dd, cr_portovfl + i);
2334 	} else if (reg == QIBPORTCNTR_PSSTAT)
2335 		ret = dd->cspec->pma_sample_status;
2336 	if (creg == 0xffff)
2337 		goto done;
2338 
2339 	/*
2340 	 * only fast incrementing counters are 64bit; use 32 bit reads to
2341 	 * avoid two independent reads when on opteron
2342 	 */
2343 	if (creg == cr_wordsend || creg == cr_wordrcv ||
2344 	    creg == cr_pktsend || creg == cr_pktrcv)
2345 		ret = read_6120_creg(dd, creg);
2346 	else
2347 		ret = read_6120_creg32(dd, creg);
2348 	if (creg == cr_ibsymbolerr) {
2349 		if (dd->cspec->ibdeltainprog)
2350 			ret -= ret - dd->cspec->ibsymsnap;
2351 		ret -= dd->cspec->ibsymdelta;
2352 	} else if (creg == cr_iblinkerrrecov) {
2353 		if (dd->cspec->ibdeltainprog)
2354 			ret -= ret - dd->cspec->iblnkerrsnap;
2355 		ret -= dd->cspec->iblnkerrdelta;
2356 	}
2357 	if (reg == QIBPORTCNTR_RXDROPPKT) /* add special cased count */
2358 		ret += dd->cspec->rxfc_unsupvl_errs;
2359 
2360 done:
2361 	return ret;
2362 }
2363 
2364 /*
2365  * Device counter names (not port-specific), one line per stat,
2366  * single string.  Used by utilities like ipathstats to print the stats
2367  * in a way which works for different versions of drivers, without changing
2368  * the utility.  Names need to be 12 chars or less (w/o newline), for proper
2369  * display by utility.
2370  * Non-error counters are first.
2371  * Start of "error" conters is indicated by a leading "E " on the first
2372  * "error" counter, and doesn't count in label length.
2373  * The EgrOvfl list needs to be last so we truncate them at the configured
2374  * context count for the device.
2375  * cntr6120indices contains the corresponding register indices.
2376  */
2377 static const char cntr6120names[] =
2378 	"Interrupts\n"
2379 	"HostBusStall\n"
2380 	"E RxTIDFull\n"
2381 	"RxTIDInvalid\n"
2382 	"Ctxt0EgrOvfl\n"
2383 	"Ctxt1EgrOvfl\n"
2384 	"Ctxt2EgrOvfl\n"
2385 	"Ctxt3EgrOvfl\n"
2386 	"Ctxt4EgrOvfl\n";
2387 
2388 static const size_t cntr6120indices[] = {
2389 	cr_lbint,
2390 	cr_lbflowstall,
2391 	cr_errtidfull,
2392 	cr_errtidvalid,
2393 	cr_portovfl + 0,
2394 	cr_portovfl + 1,
2395 	cr_portovfl + 2,
2396 	cr_portovfl + 3,
2397 	cr_portovfl + 4,
2398 };
2399 
2400 /*
2401  * same as cntr6120names and cntr6120indices, but for port-specific counters.
2402  * portcntr6120indices is somewhat complicated by some registers needing
2403  * adjustments of various kinds, and those are ORed with _PORT_VIRT_FLAG
2404  */
2405 static const char portcntr6120names[] =
2406 	"TxPkt\n"
2407 	"TxFlowPkt\n"
2408 	"TxWords\n"
2409 	"RxPkt\n"
2410 	"RxFlowPkt\n"
2411 	"RxWords\n"
2412 	"TxFlowStall\n"
2413 	"E IBStatusChng\n"
2414 	"IBLinkDown\n"
2415 	"IBLnkRecov\n"
2416 	"IBRxLinkErr\n"
2417 	"IBSymbolErr\n"
2418 	"RxLLIErr\n"
2419 	"RxBadFormat\n"
2420 	"RxBadLen\n"
2421 	"RxBufOvrfl\n"
2422 	"RxEBP\n"
2423 	"RxFlowCtlErr\n"
2424 	"RxICRCerr\n"
2425 	"RxLPCRCerr\n"
2426 	"RxVCRCerr\n"
2427 	"RxInvalLen\n"
2428 	"RxInvalPKey\n"
2429 	"RxPktDropped\n"
2430 	"TxBadLength\n"
2431 	"TxDropped\n"
2432 	"TxInvalLen\n"
2433 	"TxUnderrun\n"
2434 	"TxUnsupVL\n"
2435 	;
2436 
2437 #define _PORT_VIRT_FLAG 0x8000 /* "virtual", need adjustments */
2438 static const size_t portcntr6120indices[] = {
2439 	QIBPORTCNTR_PKTSEND | _PORT_VIRT_FLAG,
2440 	cr_pktsendflow,
2441 	QIBPORTCNTR_WORDSEND | _PORT_VIRT_FLAG,
2442 	QIBPORTCNTR_PKTRCV | _PORT_VIRT_FLAG,
2443 	cr_pktrcvflowctrl,
2444 	QIBPORTCNTR_WORDRCV | _PORT_VIRT_FLAG,
2445 	QIBPORTCNTR_SENDSTALL | _PORT_VIRT_FLAG,
2446 	cr_ibstatuschange,
2447 	QIBPORTCNTR_IBLINKDOWN | _PORT_VIRT_FLAG,
2448 	QIBPORTCNTR_IBLINKERRRECOV | _PORT_VIRT_FLAG,
2449 	QIBPORTCNTR_ERRLINK | _PORT_VIRT_FLAG,
2450 	QIBPORTCNTR_IBSYMBOLERR | _PORT_VIRT_FLAG,
2451 	QIBPORTCNTR_LLI | _PORT_VIRT_FLAG,
2452 	QIBPORTCNTR_BADFORMAT | _PORT_VIRT_FLAG,
2453 	QIBPORTCNTR_ERR_RLEN | _PORT_VIRT_FLAG,
2454 	QIBPORTCNTR_RCVOVFL | _PORT_VIRT_FLAG,
2455 	QIBPORTCNTR_RCVEBP | _PORT_VIRT_FLAG,
2456 	cr_rcvflowctrl_err,
2457 	QIBPORTCNTR_ERRICRC | _PORT_VIRT_FLAG,
2458 	QIBPORTCNTR_ERRLPCRC | _PORT_VIRT_FLAG,
2459 	QIBPORTCNTR_ERRVCRC | _PORT_VIRT_FLAG,
2460 	QIBPORTCNTR_INVALIDRLEN | _PORT_VIRT_FLAG,
2461 	QIBPORTCNTR_ERRPKEY | _PORT_VIRT_FLAG,
2462 	QIBPORTCNTR_RXDROPPKT | _PORT_VIRT_FLAG,
2463 	cr_invalidslen,
2464 	cr_senddropped,
2465 	cr_errslen,
2466 	cr_sendunderrun,
2467 	cr_txunsupvl,
2468 };
2469 
2470 /* do all the setup to make the counter reads efficient later */
2471 static void init_6120_cntrnames(struct qib_devdata *dd)
2472 {
2473 	int i, j = 0;
2474 	char *s;
2475 
2476 	for (i = 0, s = (char *)cntr6120names; s && j <= dd->cfgctxts;
2477 	     i++) {
2478 		/* we always have at least one counter before the egrovfl */
2479 		if (!j && !strncmp("Ctxt0EgrOvfl", s + 1, 12))
2480 			j = 1;
2481 		s = strchr(s + 1, '\n');
2482 		if (s && j)
2483 			j++;
2484 	}
2485 	dd->cspec->ncntrs = i;
2486 	if (!s)
2487 		/* full list; size is without terminating null */
2488 		dd->cspec->cntrnamelen = sizeof(cntr6120names) - 1;
2489 	else
2490 		dd->cspec->cntrnamelen = 1 + s - cntr6120names;
2491 	dd->cspec->cntrs = kmalloc_array(dd->cspec->ncntrs, sizeof(u64),
2492 					 GFP_KERNEL);
2493 
2494 	for (i = 0, s = (char *)portcntr6120names; s; i++)
2495 		s = strchr(s + 1, '\n');
2496 	dd->cspec->nportcntrs = i - 1;
2497 	dd->cspec->portcntrnamelen = sizeof(portcntr6120names) - 1;
2498 	dd->cspec->portcntrs = kmalloc_array(dd->cspec->nportcntrs,
2499 					     sizeof(u64),
2500 					     GFP_KERNEL);
2501 }
2502 
2503 static u32 qib_read_6120cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
2504 			      u64 **cntrp)
2505 {
2506 	u32 ret;
2507 
2508 	if (namep) {
2509 		ret = dd->cspec->cntrnamelen;
2510 		if (pos >= ret)
2511 			ret = 0; /* final read after getting everything */
2512 		else
2513 			*namep = (char *)cntr6120names;
2514 	} else {
2515 		u64 *cntr = dd->cspec->cntrs;
2516 		int i;
2517 
2518 		ret = dd->cspec->ncntrs * sizeof(u64);
2519 		if (!cntr || pos >= ret) {
2520 			/* everything read, or couldn't get memory */
2521 			ret = 0;
2522 			goto done;
2523 		}
2524 		if (pos >= ret) {
2525 			ret = 0; /* final read after getting everything */
2526 			goto done;
2527 		}
2528 		*cntrp = cntr;
2529 		for (i = 0; i < dd->cspec->ncntrs; i++)
2530 			*cntr++ = read_6120_creg32(dd, cntr6120indices[i]);
2531 	}
2532 done:
2533 	return ret;
2534 }
2535 
2536 static u32 qib_read_6120portcntrs(struct qib_devdata *dd, loff_t pos, u32 port,
2537 				  char **namep, u64 **cntrp)
2538 {
2539 	u32 ret;
2540 
2541 	if (namep) {
2542 		ret = dd->cspec->portcntrnamelen;
2543 		if (pos >= ret)
2544 			ret = 0; /* final read after getting everything */
2545 		else
2546 			*namep = (char *)portcntr6120names;
2547 	} else {
2548 		u64 *cntr = dd->cspec->portcntrs;
2549 		struct qib_pportdata *ppd = &dd->pport[port];
2550 		int i;
2551 
2552 		ret = dd->cspec->nportcntrs * sizeof(u64);
2553 		if (!cntr || pos >= ret) {
2554 			/* everything read, or couldn't get memory */
2555 			ret = 0;
2556 			goto done;
2557 		}
2558 		*cntrp = cntr;
2559 		for (i = 0; i < dd->cspec->nportcntrs; i++) {
2560 			if (portcntr6120indices[i] & _PORT_VIRT_FLAG)
2561 				*cntr++ = qib_portcntr_6120(ppd,
2562 					portcntr6120indices[i] &
2563 					~_PORT_VIRT_FLAG);
2564 			else
2565 				*cntr++ = read_6120_creg32(dd,
2566 					   portcntr6120indices[i]);
2567 		}
2568 	}
2569 done:
2570 	return ret;
2571 }
2572 
2573 static void qib_chk_6120_errormask(struct qib_devdata *dd)
2574 {
2575 	static u32 fixed;
2576 	u32 ctrl;
2577 	unsigned long errormask;
2578 	unsigned long hwerrs;
2579 
2580 	if (!dd->cspec->errormask || !(dd->flags & QIB_INITTED))
2581 		return;
2582 
2583 	errormask = qib_read_kreg64(dd, kr_errmask);
2584 
2585 	if (errormask == dd->cspec->errormask)
2586 		return;
2587 	fixed++;
2588 
2589 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
2590 	ctrl = qib_read_kreg32(dd, kr_control);
2591 
2592 	qib_write_kreg(dd, kr_errmask,
2593 		dd->cspec->errormask);
2594 
2595 	if ((hwerrs & dd->cspec->hwerrmask) ||
2596 	    (ctrl & QLOGIC_IB_C_FREEZEMODE)) {
2597 		qib_write_kreg(dd, kr_hwerrclear, 0ULL);
2598 		qib_write_kreg(dd, kr_errclear, 0ULL);
2599 		/* force re-interrupt of pending events, just in case */
2600 		qib_write_kreg(dd, kr_intclear, 0ULL);
2601 		qib_devinfo(dd->pcidev,
2602 			 "errormask fixed(%u) %lx->%lx, ctrl %x hwerr %lx\n",
2603 			 fixed, errormask, (unsigned long)dd->cspec->errormask,
2604 			 ctrl, hwerrs);
2605 	}
2606 }
2607 
2608 /**
2609  * qib_get_6120_faststats - get word counters from chip before they overflow
2610  * @t: contains a pointer to the qlogic_ib device qib_devdata
2611  *
2612  * This needs more work; in particular, decision on whether we really
2613  * need traffic_wds done the way it is
2614  * called from add_timer
2615  */
2616 static void qib_get_6120_faststats(struct timer_list *t)
2617 {
2618 	struct qib_devdata *dd = from_timer(dd, t, stats_timer);
2619 	struct qib_pportdata *ppd = dd->pport;
2620 	unsigned long flags;
2621 	u64 traffic_wds;
2622 
2623 	/*
2624 	 * don't access the chip while running diags, or memory diags can
2625 	 * fail
2626 	 */
2627 	if (!(dd->flags & QIB_INITTED) || dd->diag_client)
2628 		/* but re-arm the timer, for diags case; won't hurt other */
2629 		goto done;
2630 
2631 	/*
2632 	 * We now try to maintain an activity timer, based on traffic
2633 	 * exceeding a threshold, so we need to check the word-counts
2634 	 * even if they are 64-bit.
2635 	 */
2636 	traffic_wds = qib_portcntr_6120(ppd, cr_wordsend) +
2637 		qib_portcntr_6120(ppd, cr_wordrcv);
2638 	spin_lock_irqsave(&dd->eep_st_lock, flags);
2639 	traffic_wds -= dd->traffic_wds;
2640 	dd->traffic_wds += traffic_wds;
2641 	spin_unlock_irqrestore(&dd->eep_st_lock, flags);
2642 
2643 	qib_chk_6120_errormask(dd);
2644 done:
2645 	mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
2646 }
2647 
2648 /* no interrupt fallback for these chips */
2649 static int qib_6120_nointr_fallback(struct qib_devdata *dd)
2650 {
2651 	return 0;
2652 }
2653 
2654 /*
2655  * reset the XGXS (between serdes and IBC).  Slightly less intrusive
2656  * than resetting the IBC or external link state, and useful in some
2657  * cases to cause some retraining.  To do this right, we reset IBC
2658  * as well.
2659  */
2660 static void qib_6120_xgxs_reset(struct qib_pportdata *ppd)
2661 {
2662 	u64 val, prev_val;
2663 	struct qib_devdata *dd = ppd->dd;
2664 
2665 	prev_val = qib_read_kreg64(dd, kr_xgxs_cfg);
2666 	val = prev_val | QLOGIC_IB_XGXS_RESET;
2667 	prev_val &= ~QLOGIC_IB_XGXS_RESET; /* be sure */
2668 	qib_write_kreg(dd, kr_control,
2669 		       dd->control & ~QLOGIC_IB_C_LINKENABLE);
2670 	qib_write_kreg(dd, kr_xgxs_cfg, val);
2671 	qib_read_kreg32(dd, kr_scratch);
2672 	qib_write_kreg(dd, kr_xgxs_cfg, prev_val);
2673 	qib_write_kreg(dd, kr_control, dd->control);
2674 }
2675 
2676 static int qib_6120_get_ib_cfg(struct qib_pportdata *ppd, int which)
2677 {
2678 	int ret;
2679 
2680 	switch (which) {
2681 	case QIB_IB_CFG_LWID:
2682 		ret = ppd->link_width_active;
2683 		break;
2684 
2685 	case QIB_IB_CFG_SPD:
2686 		ret = ppd->link_speed_active;
2687 		break;
2688 
2689 	case QIB_IB_CFG_LWID_ENB:
2690 		ret = ppd->link_width_enabled;
2691 		break;
2692 
2693 	case QIB_IB_CFG_SPD_ENB:
2694 		ret = ppd->link_speed_enabled;
2695 		break;
2696 
2697 	case QIB_IB_CFG_OP_VLS:
2698 		ret = ppd->vls_operational;
2699 		break;
2700 
2701 	case QIB_IB_CFG_VL_HIGH_CAP:
2702 		ret = 0;
2703 		break;
2704 
2705 	case QIB_IB_CFG_VL_LOW_CAP:
2706 		ret = 0;
2707 		break;
2708 
2709 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2710 		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2711 				OverrunThreshold);
2712 		break;
2713 
2714 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2715 		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2716 				PhyerrThreshold);
2717 		break;
2718 
2719 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2720 		/* will only take effect when the link state changes */
2721 		ret = (ppd->dd->cspec->ibcctrl &
2722 		       SYM_MASK(IBCCtrl, LinkDownDefaultState)) ?
2723 			IB_LINKINITCMD_SLEEP : IB_LINKINITCMD_POLL;
2724 		break;
2725 
2726 	case QIB_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2727 		ret = 0; /* no heartbeat on this chip */
2728 		break;
2729 
2730 	case QIB_IB_CFG_PMA_TICKS:
2731 		ret = 250; /* 1 usec. */
2732 		break;
2733 
2734 	default:
2735 		ret =  -EINVAL;
2736 		break;
2737 	}
2738 	return ret;
2739 }
2740 
2741 /*
2742  * We assume range checking is already done, if needed.
2743  */
2744 static int qib_6120_set_ib_cfg(struct qib_pportdata *ppd, int which, u32 val)
2745 {
2746 	struct qib_devdata *dd = ppd->dd;
2747 	int ret = 0;
2748 	u64 val64;
2749 	u16 lcmd, licmd;
2750 
2751 	switch (which) {
2752 	case QIB_IB_CFG_LWID_ENB:
2753 		ppd->link_width_enabled = val;
2754 		break;
2755 
2756 	case QIB_IB_CFG_SPD_ENB:
2757 		ppd->link_speed_enabled = val;
2758 		break;
2759 
2760 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2761 		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2762 				  OverrunThreshold);
2763 		if (val64 != val) {
2764 			dd->cspec->ibcctrl &=
2765 				~SYM_MASK(IBCCtrl, OverrunThreshold);
2766 			dd->cspec->ibcctrl |= (u64) val <<
2767 				SYM_LSB(IBCCtrl, OverrunThreshold);
2768 			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2769 			qib_write_kreg(dd, kr_scratch, 0);
2770 		}
2771 		break;
2772 
2773 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2774 		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2775 				  PhyerrThreshold);
2776 		if (val64 != val) {
2777 			dd->cspec->ibcctrl &=
2778 				~SYM_MASK(IBCCtrl, PhyerrThreshold);
2779 			dd->cspec->ibcctrl |= (u64) val <<
2780 				SYM_LSB(IBCCtrl, PhyerrThreshold);
2781 			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2782 			qib_write_kreg(dd, kr_scratch, 0);
2783 		}
2784 		break;
2785 
2786 	case QIB_IB_CFG_PKEYS: /* update pkeys */
2787 		val64 = (u64) ppd->pkeys[0] | ((u64) ppd->pkeys[1] << 16) |
2788 			((u64) ppd->pkeys[2] << 32) |
2789 			((u64) ppd->pkeys[3] << 48);
2790 		qib_write_kreg(dd, kr_partitionkey, val64);
2791 		break;
2792 
2793 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2794 		/* will only take effect when the link state changes */
2795 		if (val == IB_LINKINITCMD_POLL)
2796 			dd->cspec->ibcctrl &=
2797 				~SYM_MASK(IBCCtrl, LinkDownDefaultState);
2798 		else /* SLEEP */
2799 			dd->cspec->ibcctrl |=
2800 				SYM_MASK(IBCCtrl, LinkDownDefaultState);
2801 		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2802 		qib_write_kreg(dd, kr_scratch, 0);
2803 		break;
2804 
2805 	case QIB_IB_CFG_MTU: /* update the MTU in IBC */
2806 		/*
2807 		 * Update our housekeeping variables, and set IBC max
2808 		 * size, same as init code; max IBC is max we allow in
2809 		 * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2810 		 * Set even if it's unchanged, print debug message only
2811 		 * on changes.
2812 		 */
2813 		val = (ppd->ibmaxlen >> 2) + 1;
2814 		dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, MaxPktLen);
2815 		dd->cspec->ibcctrl |= (u64)val <<
2816 			SYM_LSB(IBCCtrl, MaxPktLen);
2817 		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2818 		qib_write_kreg(dd, kr_scratch, 0);
2819 		break;
2820 
2821 	case QIB_IB_CFG_LSTATE: /* set the IB link state */
2822 		switch (val & 0xffff0000) {
2823 		case IB_LINKCMD_DOWN:
2824 			lcmd = QLOGIC_IB_IBCC_LINKCMD_DOWN;
2825 			if (!dd->cspec->ibdeltainprog) {
2826 				dd->cspec->ibdeltainprog = 1;
2827 				dd->cspec->ibsymsnap =
2828 					read_6120_creg32(dd, cr_ibsymbolerr);
2829 				dd->cspec->iblnkerrsnap =
2830 					read_6120_creg32(dd, cr_iblinkerrrecov);
2831 			}
2832 			break;
2833 
2834 		case IB_LINKCMD_ARMED:
2835 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ARMED;
2836 			break;
2837 
2838 		case IB_LINKCMD_ACTIVE:
2839 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ACTIVE;
2840 			break;
2841 
2842 		default:
2843 			ret = -EINVAL;
2844 			qib_dev_err(dd, "bad linkcmd req 0x%x\n", val >> 16);
2845 			goto bail;
2846 		}
2847 		switch (val & 0xffff) {
2848 		case IB_LINKINITCMD_NOP:
2849 			licmd = 0;
2850 			break;
2851 
2852 		case IB_LINKINITCMD_POLL:
2853 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_POLL;
2854 			break;
2855 
2856 		case IB_LINKINITCMD_SLEEP:
2857 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_SLEEP;
2858 			break;
2859 
2860 		case IB_LINKINITCMD_DISABLE:
2861 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_DISABLE;
2862 			break;
2863 
2864 		default:
2865 			ret = -EINVAL;
2866 			qib_dev_err(dd, "bad linkinitcmd req 0x%x\n",
2867 				    val & 0xffff);
2868 			goto bail;
2869 		}
2870 		qib_set_ib_6120_lstate(ppd, lcmd, licmd);
2871 		goto bail;
2872 
2873 	case QIB_IB_CFG_HRTBT:
2874 		ret = -EINVAL;
2875 		break;
2876 
2877 	default:
2878 		ret = -EINVAL;
2879 	}
2880 bail:
2881 	return ret;
2882 }
2883 
2884 static int qib_6120_set_loopback(struct qib_pportdata *ppd, const char *what)
2885 {
2886 	int ret = 0;
2887 
2888 	if (!strncmp(what, "ibc", 3)) {
2889 		ppd->dd->cspec->ibcctrl |= SYM_MASK(IBCCtrl, Loopback);
2890 		qib_devinfo(ppd->dd->pcidev, "Enabling IB%u:%u IBC loopback\n",
2891 			 ppd->dd->unit, ppd->port);
2892 	} else if (!strncmp(what, "off", 3)) {
2893 		ppd->dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, Loopback);
2894 		qib_devinfo(ppd->dd->pcidev,
2895 			"Disabling IB%u:%u IBC loopback (normal)\n",
2896 			ppd->dd->unit, ppd->port);
2897 	} else
2898 		ret = -EINVAL;
2899 	if (!ret) {
2900 		qib_write_kreg(ppd->dd, kr_ibcctrl, ppd->dd->cspec->ibcctrl);
2901 		qib_write_kreg(ppd->dd, kr_scratch, 0);
2902 	}
2903 	return ret;
2904 }
2905 
2906 static void pma_6120_timer(struct timer_list *t)
2907 {
2908 	struct qib_chip_specific *cs = from_timer(cs, t, pma_timer);
2909 	struct qib_pportdata *ppd = cs->ppd;
2910 	struct qib_ibport *ibp = &ppd->ibport_data;
2911 	unsigned long flags;
2912 
2913 	spin_lock_irqsave(&ibp->rvp.lock, flags);
2914 	if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED) {
2915 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2916 		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2917 				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2918 		mod_timer(&cs->pma_timer,
2919 		      jiffies + usecs_to_jiffies(ibp->rvp.pma_sample_interval));
2920 	} else if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
2921 		u64 ta, tb, tc, td, te;
2922 
2923 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2924 		qib_snapshot_counters(ppd, &ta, &tb, &tc, &td, &te);
2925 
2926 		cs->sword = ta - cs->sword;
2927 		cs->rword = tb - cs->rword;
2928 		cs->spkts = tc - cs->spkts;
2929 		cs->rpkts = td - cs->rpkts;
2930 		cs->xmit_wait = te - cs->xmit_wait;
2931 	}
2932 	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
2933 }
2934 
2935 /*
2936  * Note that the caller has the ibp->rvp.lock held.
2937  */
2938 static void qib_set_cntr_6120_sample(struct qib_pportdata *ppd, u32 intv,
2939 				     u32 start)
2940 {
2941 	struct qib_chip_specific *cs = ppd->dd->cspec;
2942 
2943 	if (start && intv) {
2944 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_STARTED;
2945 		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(start));
2946 	} else if (intv) {
2947 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2948 		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2949 				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2950 		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(intv));
2951 	} else {
2952 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2953 		cs->sword = 0;
2954 		cs->rword = 0;
2955 		cs->spkts = 0;
2956 		cs->rpkts = 0;
2957 		cs->xmit_wait = 0;
2958 	}
2959 }
2960 
2961 static u32 qib_6120_iblink_state(u64 ibcs)
2962 {
2963 	u32 state = (u32)SYM_FIELD(ibcs, IBCStatus, LinkState);
2964 
2965 	switch (state) {
2966 	case IB_6120_L_STATE_INIT:
2967 		state = IB_PORT_INIT;
2968 		break;
2969 	case IB_6120_L_STATE_ARM:
2970 		state = IB_PORT_ARMED;
2971 		break;
2972 	case IB_6120_L_STATE_ACTIVE:
2973 	case IB_6120_L_STATE_ACT_DEFER:
2974 		state = IB_PORT_ACTIVE;
2975 		break;
2976 	default:
2977 		fallthrough;
2978 	case IB_6120_L_STATE_DOWN:
2979 		state = IB_PORT_DOWN;
2980 		break;
2981 	}
2982 	return state;
2983 }
2984 
2985 /* returns the IBTA port state, rather than the IBC link training state */
2986 static u8 qib_6120_phys_portstate(u64 ibcs)
2987 {
2988 	u8 state = (u8)SYM_FIELD(ibcs, IBCStatus, LinkTrainingState);
2989 	return qib_6120_physportstate[state];
2990 }
2991 
2992 static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs)
2993 {
2994 	unsigned long flags;
2995 
2996 	spin_lock_irqsave(&ppd->lflags_lock, flags);
2997 	ppd->lflags &= ~QIBL_IB_FORCE_NOTIFY;
2998 	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
2999 
3000 	if (ibup) {
3001 		if (ppd->dd->cspec->ibdeltainprog) {
3002 			ppd->dd->cspec->ibdeltainprog = 0;
3003 			ppd->dd->cspec->ibsymdelta +=
3004 				read_6120_creg32(ppd->dd, cr_ibsymbolerr) -
3005 					ppd->dd->cspec->ibsymsnap;
3006 			ppd->dd->cspec->iblnkerrdelta +=
3007 				read_6120_creg32(ppd->dd, cr_iblinkerrrecov) -
3008 					ppd->dd->cspec->iblnkerrsnap;
3009 		}
3010 		qib_hol_init(ppd);
3011 	} else {
3012 		ppd->dd->cspec->lli_counter = 0;
3013 		if (!ppd->dd->cspec->ibdeltainprog) {
3014 			ppd->dd->cspec->ibdeltainprog = 1;
3015 			ppd->dd->cspec->ibsymsnap =
3016 				read_6120_creg32(ppd->dd, cr_ibsymbolerr);
3017 			ppd->dd->cspec->iblnkerrsnap =
3018 				read_6120_creg32(ppd->dd, cr_iblinkerrrecov);
3019 		}
3020 		qib_hol_down(ppd);
3021 	}
3022 
3023 	qib_6120_setup_setextled(ppd, ibup);
3024 
3025 	return 0;
3026 }
3027 
3028 /* Does read/modify/write to appropriate registers to
3029  * set output and direction bits selected by mask.
3030  * these are in their canonical positions (e.g. lsb of
3031  * dir will end up in D48 of extctrl on existing chips).
3032  * returns contents of GP Inputs.
3033  */
3034 static int gpio_6120_mod(struct qib_devdata *dd, u32 out, u32 dir, u32 mask)
3035 {
3036 	u64 read_val, new_out;
3037 	unsigned long flags;
3038 
3039 	if (mask) {
3040 		/* some bits being written, lock access to GPIO */
3041 		dir &= mask;
3042 		out &= mask;
3043 		spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
3044 		dd->cspec->extctrl &= ~((u64)mask << SYM_LSB(EXTCtrl, GPIOOe));
3045 		dd->cspec->extctrl |= ((u64) dir << SYM_LSB(EXTCtrl, GPIOOe));
3046 		new_out = (dd->cspec->gpio_out & ~mask) | out;
3047 
3048 		qib_write_kreg(dd, kr_extctrl, dd->cspec->extctrl);
3049 		qib_write_kreg(dd, kr_gpio_out, new_out);
3050 		dd->cspec->gpio_out = new_out;
3051 		spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
3052 	}
3053 	/*
3054 	 * It is unlikely that a read at this time would get valid
3055 	 * data on a pin whose direction line was set in the same
3056 	 * call to this function. We include the read here because
3057 	 * that allows us to potentially combine a change on one pin with
3058 	 * a read on another, and because the old code did something like
3059 	 * this.
3060 	 */
3061 	read_val = qib_read_kreg64(dd, kr_extstatus);
3062 	return SYM_FIELD(read_val, EXTStatus, GPIOIn);
3063 }
3064 
3065 /*
3066  * Read fundamental info we need to use the chip.  These are
3067  * the registers that describe chip capabilities, and are
3068  * saved in shadow registers.
3069  */
3070 static void get_6120_chip_params(struct qib_devdata *dd)
3071 {
3072 	u64 val;
3073 	u32 piobufs;
3074 	int mtu;
3075 
3076 	dd->uregbase = qib_read_kreg32(dd, kr_userregbase);
3077 
3078 	dd->rcvtidcnt = qib_read_kreg32(dd, kr_rcvtidcnt);
3079 	dd->rcvtidbase = qib_read_kreg32(dd, kr_rcvtidbase);
3080 	dd->rcvegrbase = qib_read_kreg32(dd, kr_rcvegrbase);
3081 	dd->palign = qib_read_kreg32(dd, kr_palign);
3082 	dd->piobufbase = qib_read_kreg64(dd, kr_sendpiobufbase);
3083 	dd->pio2k_bufbase = dd->piobufbase & 0xffffffff;
3084 
3085 	dd->rcvhdrcnt = qib_read_kreg32(dd, kr_rcvegrcnt);
3086 
3087 	val = qib_read_kreg64(dd, kr_sendpiosize);
3088 	dd->piosize2k = val & ~0U;
3089 	dd->piosize4k = val >> 32;
3090 
3091 	mtu = ib_mtu_enum_to_int(qib_ibmtu);
3092 	if (mtu == -1)
3093 		mtu = QIB_DEFAULT_MTU;
3094 	dd->pport->ibmtu = (u32)mtu;
3095 
3096 	val = qib_read_kreg64(dd, kr_sendpiobufcnt);
3097 	dd->piobcnt2k = val & ~0U;
3098 	dd->piobcnt4k = val >> 32;
3099 	dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
3100 	/* these may be adjusted in init_chip_wc_pat() */
3101 	dd->pio2kbase = (u32 __iomem *)
3102 		(((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
3103 	if (dd->piobcnt4k) {
3104 		dd->pio4kbase = (u32 __iomem *)
3105 			(((char __iomem *) dd->kregbase) +
3106 			 (dd->piobufbase >> 32));
3107 		/*
3108 		 * 4K buffers take 2 pages; we use roundup just to be
3109 		 * paranoid; we calculate it once here, rather than on
3110 		 * ever buf allocate
3111 		 */
3112 		dd->align4k = ALIGN(dd->piosize4k, dd->palign);
3113 	}
3114 
3115 	piobufs = dd->piobcnt4k + dd->piobcnt2k;
3116 
3117 	dd->pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) /
3118 		(sizeof(u64) * BITS_PER_BYTE / 2);
3119 }
3120 
3121 /*
3122  * The chip base addresses in cspec and cpspec have to be set
3123  * after possible init_chip_wc_pat(), rather than in
3124  * get_6120_chip_params(), so split out as separate function
3125  */
3126 static void set_6120_baseaddrs(struct qib_devdata *dd)
3127 {
3128 	u32 cregbase;
3129 
3130 	cregbase = qib_read_kreg32(dd, kr_counterregbase);
3131 	dd->cspec->cregbase = (u64 __iomem *)
3132 		((char __iomem *) dd->kregbase + cregbase);
3133 
3134 	dd->egrtidbase = (u64 __iomem *)
3135 		((char __iomem *) dd->kregbase + dd->rcvegrbase);
3136 }
3137 
3138 /*
3139  * Write the final few registers that depend on some of the
3140  * init setup.  Done late in init, just before bringing up
3141  * the serdes.
3142  */
3143 static int qib_late_6120_initreg(struct qib_devdata *dd)
3144 {
3145 	int ret = 0;
3146 	u64 val;
3147 
3148 	qib_write_kreg(dd, kr_rcvhdrentsize, dd->rcvhdrentsize);
3149 	qib_write_kreg(dd, kr_rcvhdrsize, dd->rcvhdrsize);
3150 	qib_write_kreg(dd, kr_rcvhdrcnt, dd->rcvhdrcnt);
3151 	qib_write_kreg(dd, kr_sendpioavailaddr, dd->pioavailregs_phys);
3152 	val = qib_read_kreg64(dd, kr_sendpioavailaddr);
3153 	if (val != dd->pioavailregs_phys) {
3154 		qib_dev_err(dd,
3155 			"Catastrophic software error, SendPIOAvailAddr written as %lx, read back as %llx\n",
3156 			(unsigned long) dd->pioavailregs_phys,
3157 			(unsigned long long) val);
3158 		ret = -EINVAL;
3159 	}
3160 	return ret;
3161 }
3162 
3163 static int init_6120_variables(struct qib_devdata *dd)
3164 {
3165 	int ret = 0;
3166 	struct qib_pportdata *ppd;
3167 	u32 sbufs;
3168 
3169 	ppd = (struct qib_pportdata *)(dd + 1);
3170 	dd->pport = ppd;
3171 	dd->num_pports = 1;
3172 
3173 	dd->cspec = (struct qib_chip_specific *)(ppd + dd->num_pports);
3174 	dd->cspec->ppd = ppd;
3175 	ppd->cpspec = NULL; /* not used in this chip */
3176 
3177 	spin_lock_init(&dd->cspec->kernel_tid_lock);
3178 	spin_lock_init(&dd->cspec->user_tid_lock);
3179 	spin_lock_init(&dd->cspec->rcvmod_lock);
3180 	spin_lock_init(&dd->cspec->gpio_lock);
3181 
3182 	/* we haven't yet set QIB_PRESENT, so use read directly */
3183 	dd->revision = readq(&dd->kregbase[kr_revision]);
3184 
3185 	if ((dd->revision & 0xffffffffU) == 0xffffffffU) {
3186 		qib_dev_err(dd,
3187 			"Revision register read failure, giving up initialization\n");
3188 		ret = -ENODEV;
3189 		goto bail;
3190 	}
3191 	dd->flags |= QIB_PRESENT;  /* now register routines work */
3192 
3193 	dd->majrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3194 				    ChipRevMajor);
3195 	dd->minrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3196 				    ChipRevMinor);
3197 
3198 	get_6120_chip_params(dd);
3199 	pe_boardname(dd); /* fill in boardname */
3200 
3201 	/*
3202 	 * GPIO bits for TWSI data and clock,
3203 	 * used for serial EEPROM.
3204 	 */
3205 	dd->gpio_sda_num = _QIB_GPIO_SDA_NUM;
3206 	dd->gpio_scl_num = _QIB_GPIO_SCL_NUM;
3207 	dd->twsi_eeprom_dev = QIB_TWSI_NO_DEV;
3208 
3209 	if (qib_unordered_wc())
3210 		dd->flags |= QIB_PIO_FLUSH_WC;
3211 
3212 	ret = qib_init_pportdata(ppd, dd, 0, 1);
3213 	if (ret)
3214 		goto bail;
3215 	ppd->link_width_supported = IB_WIDTH_1X | IB_WIDTH_4X;
3216 	ppd->link_speed_supported = QIB_IB_SDR;
3217 	ppd->link_width_enabled = IB_WIDTH_4X;
3218 	ppd->link_speed_enabled = ppd->link_speed_supported;
3219 	/* these can't change for this chip, so set once */
3220 	ppd->link_width_active = ppd->link_width_enabled;
3221 	ppd->link_speed_active = ppd->link_speed_enabled;
3222 	ppd->vls_supported = IB_VL_VL0;
3223 	ppd->vls_operational = ppd->vls_supported;
3224 
3225 	dd->rcvhdrentsize = QIB_RCVHDR_ENTSIZE;
3226 	dd->rcvhdrsize = QIB_DFLT_RCVHDRSIZE;
3227 	dd->rhf_offset = 0;
3228 
3229 	/* we always allocate at least 2048 bytes for eager buffers */
3230 	ret = ib_mtu_enum_to_int(qib_ibmtu);
3231 	dd->rcvegrbufsize = ret != -1 ? max(ret, 2048) : QIB_DEFAULT_MTU;
3232 	dd->rcvegrbufsize_shift = ilog2(dd->rcvegrbufsize);
3233 
3234 	qib_6120_tidtemplate(dd);
3235 
3236 	/*
3237 	 * We can request a receive interrupt for 1 or
3238 	 * more packets from current offset.  For now, we set this
3239 	 * up for a single packet.
3240 	 */
3241 	dd->rhdrhead_intr_off = 1ULL << 32;
3242 
3243 	/* setup the stats timer; the add_timer is done at end of init */
3244 	timer_setup(&dd->stats_timer, qib_get_6120_faststats, 0);
3245 	timer_setup(&dd->cspec->pma_timer, pma_6120_timer, 0);
3246 
3247 	dd->ureg_align = qib_read_kreg32(dd, kr_palign);
3248 
3249 	dd->piosize2kmax_dwords = dd->piosize2k >> 2;
3250 	qib_6120_config_ctxts(dd);
3251 	qib_set_ctxtcnt(dd);
3252 
3253 	ret = init_chip_wc_pat(dd, 0);
3254 	if (ret)
3255 		goto bail;
3256 	set_6120_baseaddrs(dd); /* set chip access pointers now */
3257 
3258 	ret = 0;
3259 	if (qib_mini_init)
3260 		goto bail;
3261 
3262 	qib_num_cfg_vls = 1; /* if any 6120's, only one VL */
3263 
3264 	ret = qib_create_ctxts(dd);
3265 	init_6120_cntrnames(dd);
3266 
3267 	/* use all of 4KB buffers for the kernel, otherwise 16 */
3268 	sbufs = dd->piobcnt4k ?  dd->piobcnt4k : 16;
3269 
3270 	dd->lastctxt_piobuf = dd->piobcnt2k + dd->piobcnt4k - sbufs;
3271 	dd->pbufsctxt = dd->lastctxt_piobuf /
3272 		(dd->cfgctxts - dd->first_user_ctxt);
3273 
3274 	if (ret)
3275 		goto bail;
3276 bail:
3277 	return ret;
3278 }
3279 
3280 /*
3281  * For this chip, we want to use the same buffer every time
3282  * when we are trying to bring the link up (they are always VL15
3283  * packets).  At that link state the packet should always go out immediately
3284  * (or at least be discarded at the tx interface if the link is down).
3285  * If it doesn't, and the buffer isn't available, that means some other
3286  * sender has gotten ahead of us, and is preventing our packet from going
3287  * out.  In that case, we flush all packets, and try again.  If that still
3288  * fails, we fail the request, and hope things work the next time around.
3289  *
3290  * We don't need very complicated heuristics on whether the packet had
3291  * time to go out or not, since even at SDR 1X, it goes out in very short
3292  * time periods, covered by the chip reads done here and as part of the
3293  * flush.
3294  */
3295 static u32 __iomem *get_6120_link_buf(struct qib_pportdata *ppd, u32 *bnum)
3296 {
3297 	u32 __iomem *buf;
3298 	u32 lbuf = ppd->dd->piobcnt2k + ppd->dd->piobcnt4k - 1;
3299 
3300 	/*
3301 	 * always blip to get avail list updated, since it's almost
3302 	 * always needed, and is fairly cheap.
3303 	 */
3304 	sendctrl_6120_mod(ppd->dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
3305 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3306 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3307 	if (buf)
3308 		goto done;
3309 
3310 	sendctrl_6120_mod(ppd, QIB_SENDCTRL_DISARM_ALL | QIB_SENDCTRL_FLUSH |
3311 			  QIB_SENDCTRL_AVAIL_BLIP);
3312 	ppd->dd->upd_pio_shadow  = 1; /* update our idea of what's busy */
3313 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3314 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3315 done:
3316 	return buf;
3317 }
3318 
3319 static u32 __iomem *qib_6120_getsendbuf(struct qib_pportdata *ppd, u64 pbc,
3320 					u32 *pbufnum)
3321 {
3322 	u32 first, last, plen = pbc & QIB_PBC_LENGTH_MASK;
3323 	struct qib_devdata *dd = ppd->dd;
3324 	u32 __iomem *buf;
3325 
3326 	if (((pbc >> 32) & PBC_6120_VL15_SEND_CTRL) &&
3327 		!(ppd->lflags & (QIBL_IB_AUTONEG_INPROG | QIBL_LINKACTIVE)))
3328 		buf = get_6120_link_buf(ppd, pbufnum);
3329 	else {
3330 
3331 		if ((plen + 1) > dd->piosize2kmax_dwords)
3332 			first = dd->piobcnt2k;
3333 		else
3334 			first = 0;
3335 		/* try 4k if all 2k busy, so same last for both sizes */
3336 		last = dd->piobcnt2k + dd->piobcnt4k - 1;
3337 		buf = qib_getsendbuf_range(dd, pbufnum, first, last);
3338 	}
3339 	return buf;
3340 }
3341 
3342 static int init_sdma_6120_regs(struct qib_pportdata *ppd)
3343 {
3344 	return -ENODEV;
3345 }
3346 
3347 static u16 qib_sdma_6120_gethead(struct qib_pportdata *ppd)
3348 {
3349 	return 0;
3350 }
3351 
3352 static int qib_sdma_6120_busy(struct qib_pportdata *ppd)
3353 {
3354 	return 0;
3355 }
3356 
3357 static void qib_sdma_update_6120_tail(struct qib_pportdata *ppd, u16 tail)
3358 {
3359 }
3360 
3361 static void qib_6120_sdma_sendctrl(struct qib_pportdata *ppd, unsigned op)
3362 {
3363 }
3364 
3365 static void qib_sdma_set_6120_desc_cnt(struct qib_pportdata *ppd, unsigned cnt)
3366 {
3367 }
3368 
3369 /*
3370  * the pbc doesn't need a VL15 indicator, but we need it for link_buf.
3371  * The chip ignores the bit if set.
3372  */
3373 static u32 qib_6120_setpbc_control(struct qib_pportdata *ppd, u32 plen,
3374 				   u8 srate, u8 vl)
3375 {
3376 	return vl == 15 ? PBC_6120_VL15_SEND_CTRL : 0;
3377 }
3378 
3379 static void qib_6120_initvl15_bufs(struct qib_devdata *dd)
3380 {
3381 }
3382 
3383 static void qib_6120_init_ctxt(struct qib_ctxtdata *rcd)
3384 {
3385 	rcd->rcvegrcnt = rcd->dd->rcvhdrcnt;
3386 	rcd->rcvegr_tid_base = rcd->ctxt * rcd->rcvegrcnt;
3387 }
3388 
3389 static void qib_6120_txchk_change(struct qib_devdata *dd, u32 start,
3390 	u32 len, u32 avail, struct qib_ctxtdata *rcd)
3391 {
3392 }
3393 
3394 static void writescratch(struct qib_devdata *dd, u32 val)
3395 {
3396 	(void) qib_write_kreg(dd, kr_scratch, val);
3397 }
3398 
3399 static int qib_6120_tempsense_rd(struct qib_devdata *dd, int regnum)
3400 {
3401 	return -ENXIO;
3402 }
3403 
3404 #ifdef CONFIG_INFINIBAND_QIB_DCA
3405 static int qib_6120_notify_dca(struct qib_devdata *dd, unsigned long event)
3406 {
3407 	return 0;
3408 }
3409 #endif
3410 
3411 /* Dummy function, as 6120 boards never disable EEPROM Write */
3412 static int qib_6120_eeprom_wen(struct qib_devdata *dd, int wen)
3413 {
3414 	return 1;
3415 }
3416 
3417 /**
3418  * qib_init_iba6120_funcs - set up the chip-specific function pointers
3419  * @pdev: pci_dev of the qlogic_ib device
3420  * @ent: pci_device_id matching this chip
3421  *
3422  * This is global, and is called directly at init to set up the
3423  * chip-specific function pointers for later use.
3424  *
3425  * It also allocates/partially-inits the qib_devdata struct for
3426  * this device.
3427  */
3428 struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *pdev,
3429 					   const struct pci_device_id *ent)
3430 {
3431 	struct qib_devdata *dd;
3432 	int ret;
3433 
3434 	dd = qib_alloc_devdata(pdev, sizeof(struct qib_pportdata) +
3435 			       sizeof(struct qib_chip_specific));
3436 	if (IS_ERR(dd))
3437 		goto bail;
3438 
3439 	dd->f_bringup_serdes    = qib_6120_bringup_serdes;
3440 	dd->f_cleanup           = qib_6120_setup_cleanup;
3441 	dd->f_clear_tids        = qib_6120_clear_tids;
3442 	dd->f_free_irq          = qib_free_irq;
3443 	dd->f_get_base_info     = qib_6120_get_base_info;
3444 	dd->f_get_msgheader     = qib_6120_get_msgheader;
3445 	dd->f_getsendbuf        = qib_6120_getsendbuf;
3446 	dd->f_gpio_mod          = gpio_6120_mod;
3447 	dd->f_eeprom_wen	= qib_6120_eeprom_wen;
3448 	dd->f_hdrqempty         = qib_6120_hdrqempty;
3449 	dd->f_ib_updown         = qib_6120_ib_updown;
3450 	dd->f_init_ctxt         = qib_6120_init_ctxt;
3451 	dd->f_initvl15_bufs     = qib_6120_initvl15_bufs;
3452 	dd->f_intr_fallback     = qib_6120_nointr_fallback;
3453 	dd->f_late_initreg      = qib_late_6120_initreg;
3454 	dd->f_setpbc_control    = qib_6120_setpbc_control;
3455 	dd->f_portcntr          = qib_portcntr_6120;
3456 	dd->f_put_tid           = (dd->minrev >= 2) ?
3457 				      qib_6120_put_tid_2 :
3458 				      qib_6120_put_tid;
3459 	dd->f_quiet_serdes      = qib_6120_quiet_serdes;
3460 	dd->f_rcvctrl           = rcvctrl_6120_mod;
3461 	dd->f_read_cntrs        = qib_read_6120cntrs;
3462 	dd->f_read_portcntrs    = qib_read_6120portcntrs;
3463 	dd->f_reset             = qib_6120_setup_reset;
3464 	dd->f_init_sdma_regs    = init_sdma_6120_regs;
3465 	dd->f_sdma_busy         = qib_sdma_6120_busy;
3466 	dd->f_sdma_gethead      = qib_sdma_6120_gethead;
3467 	dd->f_sdma_sendctrl     = qib_6120_sdma_sendctrl;
3468 	dd->f_sdma_set_desc_cnt = qib_sdma_set_6120_desc_cnt;
3469 	dd->f_sdma_update_tail  = qib_sdma_update_6120_tail;
3470 	dd->f_sendctrl          = sendctrl_6120_mod;
3471 	dd->f_set_armlaunch     = qib_set_6120_armlaunch;
3472 	dd->f_set_cntr_sample   = qib_set_cntr_6120_sample;
3473 	dd->f_iblink_state      = qib_6120_iblink_state;
3474 	dd->f_ibphys_portstate  = qib_6120_phys_portstate;
3475 	dd->f_get_ib_cfg        = qib_6120_get_ib_cfg;
3476 	dd->f_set_ib_cfg        = qib_6120_set_ib_cfg;
3477 	dd->f_set_ib_loopback   = qib_6120_set_loopback;
3478 	dd->f_set_intr_state    = qib_6120_set_intr_state;
3479 	dd->f_setextled         = qib_6120_setup_setextled;
3480 	dd->f_txchk_change      = qib_6120_txchk_change;
3481 	dd->f_update_usrhead    = qib_update_6120_usrhead;
3482 	dd->f_wantpiobuf_intr   = qib_wantpiobuf_6120_intr;
3483 	dd->f_xgxs_reset        = qib_6120_xgxs_reset;
3484 	dd->f_writescratch      = writescratch;
3485 	dd->f_tempsense_rd	= qib_6120_tempsense_rd;
3486 #ifdef CONFIG_INFINIBAND_QIB_DCA
3487 	dd->f_notify_dca = qib_6120_notify_dca;
3488 #endif
3489 	/*
3490 	 * Do remaining pcie setup and save pcie values in dd.
3491 	 * Any error printing is already done by the init code.
3492 	 * On return, we have the chip mapped and accessible,
3493 	 * but chip registers are not set up until start of
3494 	 * init_6120_variables.
3495 	 */
3496 	ret = qib_pcie_ddinit(dd, pdev, ent);
3497 	if (ret < 0)
3498 		goto bail_free;
3499 
3500 	/* initialize chip-specific variables */
3501 	ret = init_6120_variables(dd);
3502 	if (ret)
3503 		goto bail_cleanup;
3504 
3505 	if (qib_mini_init)
3506 		goto bail;
3507 
3508 	if (qib_pcie_params(dd, 8, NULL))
3509 		qib_dev_err(dd,
3510 			"Failed to setup PCIe or interrupts; continuing anyway\n");
3511 	/* clear diagctrl register, in case diags were running and crashed */
3512 	qib_write_kreg(dd, kr_hwdiagctrl, 0);
3513 
3514 	if (qib_read_kreg64(dd, kr_hwerrstatus) &
3515 	    QLOGIC_IB_HWE_SERDESPLLFAILED)
3516 		qib_write_kreg(dd, kr_hwerrclear,
3517 			       QLOGIC_IB_HWE_SERDESPLLFAILED);
3518 
3519 	/* setup interrupt handler (interrupt type handled above) */
3520 	qib_setup_6120_interrupt(dd);
3521 	/* Note that qpn_mask is set by qib_6120_config_ctxts() first */
3522 	qib_6120_init_hwerrors(dd);
3523 
3524 	goto bail;
3525 
3526 bail_cleanup:
3527 	qib_pcie_ddcleanup(dd);
3528 bail_free:
3529 	qib_free_devdata(dd);
3530 	dd = ERR_PTR(ret);
3531 bail:
3532 	return dd;
3533 }
3534