1 /*
2  * Copyright (c) 2013 - 2017 Intel Corporation. All rights reserved.
3  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
4  * All rights reserved.
5  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 /*
36  * This file contains all of the code that is specific to the
37  * QLogic_IB 6120 PCIe chip.
38  */
39 
40 #include <linux/interrupt.h>
41 #include <linux/pci.h>
42 #include <linux/delay.h>
43 #include <rdma/ib_verbs.h>
44 
45 #include "qib.h"
46 #include "qib_6120_regs.h"
47 
48 static void qib_6120_setup_setextled(struct qib_pportdata *, u32);
49 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op);
50 static u8 qib_6120_phys_portstate(u64);
51 static u32 qib_6120_iblink_state(u64);
52 
53 /*
54  * This file contains all the chip-specific register information and
55  * access functions for the Intel Intel_IB PCI-Express chip.
56  *
57  */
58 
59 /* KREG_IDX uses machine-generated #defines */
60 #define KREG_IDX(regname) (QIB_6120_##regname##_OFFS / sizeof(u64))
61 
62 /* Use defines to tie machine-generated names to lower-case names */
63 #define kr_extctrl KREG_IDX(EXTCtrl)
64 #define kr_extstatus KREG_IDX(EXTStatus)
65 #define kr_gpio_clear KREG_IDX(GPIOClear)
66 #define kr_gpio_mask KREG_IDX(GPIOMask)
67 #define kr_gpio_out KREG_IDX(GPIOOut)
68 #define kr_gpio_status KREG_IDX(GPIOStatus)
69 #define kr_rcvctrl KREG_IDX(RcvCtrl)
70 #define kr_sendctrl KREG_IDX(SendCtrl)
71 #define kr_partitionkey KREG_IDX(RcvPartitionKey)
72 #define kr_hwdiagctrl KREG_IDX(HwDiagCtrl)
73 #define kr_ibcstatus KREG_IDX(IBCStatus)
74 #define kr_ibcctrl KREG_IDX(IBCCtrl)
75 #define kr_sendbuffererror KREG_IDX(SendBufErr0)
76 #define kr_rcvbthqp KREG_IDX(RcvBTHQP)
77 #define kr_counterregbase KREG_IDX(CntrRegBase)
78 #define kr_palign KREG_IDX(PageAlign)
79 #define kr_rcvegrbase KREG_IDX(RcvEgrBase)
80 #define kr_rcvegrcnt KREG_IDX(RcvEgrCnt)
81 #define kr_rcvhdrcnt KREG_IDX(RcvHdrCnt)
82 #define kr_rcvhdrentsize KREG_IDX(RcvHdrEntSize)
83 #define kr_rcvhdrsize KREG_IDX(RcvHdrSize)
84 #define kr_rcvtidbase KREG_IDX(RcvTIDBase)
85 #define kr_rcvtidcnt KREG_IDX(RcvTIDCnt)
86 #define kr_scratch KREG_IDX(Scratch)
87 #define kr_sendctrl KREG_IDX(SendCtrl)
88 #define kr_sendpioavailaddr KREG_IDX(SendPIOAvailAddr)
89 #define kr_sendpiobufbase KREG_IDX(SendPIOBufBase)
90 #define kr_sendpiobufcnt KREG_IDX(SendPIOBufCnt)
91 #define kr_sendpiosize KREG_IDX(SendPIOSize)
92 #define kr_sendregbase KREG_IDX(SendRegBase)
93 #define kr_userregbase KREG_IDX(UserRegBase)
94 #define kr_control KREG_IDX(Control)
95 #define kr_intclear KREG_IDX(IntClear)
96 #define kr_intmask KREG_IDX(IntMask)
97 #define kr_intstatus KREG_IDX(IntStatus)
98 #define kr_errclear KREG_IDX(ErrClear)
99 #define kr_errmask KREG_IDX(ErrMask)
100 #define kr_errstatus KREG_IDX(ErrStatus)
101 #define kr_hwerrclear KREG_IDX(HwErrClear)
102 #define kr_hwerrmask KREG_IDX(HwErrMask)
103 #define kr_hwerrstatus KREG_IDX(HwErrStatus)
104 #define kr_revision KREG_IDX(Revision)
105 #define kr_portcnt KREG_IDX(PortCnt)
106 #define kr_serdes_cfg0 KREG_IDX(SerdesCfg0)
107 #define kr_serdes_cfg1 (kr_serdes_cfg0 + 1)
108 #define kr_serdes_stat KREG_IDX(SerdesStat)
109 #define kr_xgxs_cfg KREG_IDX(XGXSCfg)
110 
111 /* These must only be written via qib_write_kreg_ctxt() */
112 #define kr_rcvhdraddr KREG_IDX(RcvHdrAddr0)
113 #define kr_rcvhdrtailaddr KREG_IDX(RcvHdrTailAddr0)
114 
115 #define CREG_IDX(regname) ((QIB_6120_##regname##_OFFS - \
116 			QIB_6120_LBIntCnt_OFFS) / sizeof(u64))
117 
118 #define cr_badformat CREG_IDX(RxBadFormatCnt)
119 #define cr_erricrc CREG_IDX(RxICRCErrCnt)
120 #define cr_errlink CREG_IDX(RxLinkProblemCnt)
121 #define cr_errlpcrc CREG_IDX(RxLPCRCErrCnt)
122 #define cr_errpkey CREG_IDX(RxPKeyMismatchCnt)
123 #define cr_rcvflowctrl_err CREG_IDX(RxFlowCtrlErrCnt)
124 #define cr_err_rlen CREG_IDX(RxLenErrCnt)
125 #define cr_errslen CREG_IDX(TxLenErrCnt)
126 #define cr_errtidfull CREG_IDX(RxTIDFullErrCnt)
127 #define cr_errtidvalid CREG_IDX(RxTIDValidErrCnt)
128 #define cr_errvcrc CREG_IDX(RxVCRCErrCnt)
129 #define cr_ibstatuschange CREG_IDX(IBStatusChangeCnt)
130 #define cr_lbint CREG_IDX(LBIntCnt)
131 #define cr_invalidrlen CREG_IDX(RxMaxMinLenErrCnt)
132 #define cr_invalidslen CREG_IDX(TxMaxMinLenErrCnt)
133 #define cr_lbflowstall CREG_IDX(LBFlowStallCnt)
134 #define cr_pktrcv CREG_IDX(RxDataPktCnt)
135 #define cr_pktrcvflowctrl CREG_IDX(RxFlowPktCnt)
136 #define cr_pktsend CREG_IDX(TxDataPktCnt)
137 #define cr_pktsendflow CREG_IDX(TxFlowPktCnt)
138 #define cr_portovfl CREG_IDX(RxP0HdrEgrOvflCnt)
139 #define cr_rcvebp CREG_IDX(RxEBPCnt)
140 #define cr_rcvovfl CREG_IDX(RxBufOvflCnt)
141 #define cr_senddropped CREG_IDX(TxDroppedPktCnt)
142 #define cr_sendstall CREG_IDX(TxFlowStallCnt)
143 #define cr_sendunderrun CREG_IDX(TxUnderrunCnt)
144 #define cr_wordrcv CREG_IDX(RxDwordCnt)
145 #define cr_wordsend CREG_IDX(TxDwordCnt)
146 #define cr_txunsupvl CREG_IDX(TxUnsupVLErrCnt)
147 #define cr_rxdroppkt CREG_IDX(RxDroppedPktCnt)
148 #define cr_iblinkerrrecov CREG_IDX(IBLinkErrRecoveryCnt)
149 #define cr_iblinkdown CREG_IDX(IBLinkDownedCnt)
150 #define cr_ibsymbolerr CREG_IDX(IBSymbolErrCnt)
151 
152 #define SYM_RMASK(regname, fldname) ((u64)              \
153 	QIB_6120_##regname##_##fldname##_RMASK)
154 #define SYM_MASK(regname, fldname) ((u64)               \
155 	QIB_6120_##regname##_##fldname##_RMASK <<       \
156 	 QIB_6120_##regname##_##fldname##_LSB)
157 #define SYM_LSB(regname, fldname) (QIB_6120_##regname##_##fldname##_LSB)
158 
159 #define SYM_FIELD(value, regname, fldname) ((u64) \
160 	(((value) >> SYM_LSB(regname, fldname)) & \
161 	 SYM_RMASK(regname, fldname)))
162 #define ERR_MASK(fldname) SYM_MASK(ErrMask, fldname##Mask)
163 #define HWE_MASK(fldname) SYM_MASK(HwErrMask, fldname##Mask)
164 
165 /* link training states, from IBC */
166 #define IB_6120_LT_STATE_DISABLED        0x00
167 #define IB_6120_LT_STATE_LINKUP          0x01
168 #define IB_6120_LT_STATE_POLLACTIVE      0x02
169 #define IB_6120_LT_STATE_POLLQUIET       0x03
170 #define IB_6120_LT_STATE_SLEEPDELAY      0x04
171 #define IB_6120_LT_STATE_SLEEPQUIET      0x05
172 #define IB_6120_LT_STATE_CFGDEBOUNCE     0x08
173 #define IB_6120_LT_STATE_CFGRCVFCFG      0x09
174 #define IB_6120_LT_STATE_CFGWAITRMT      0x0a
175 #define IB_6120_LT_STATE_CFGIDLE 0x0b
176 #define IB_6120_LT_STATE_RECOVERRETRAIN  0x0c
177 #define IB_6120_LT_STATE_RECOVERWAITRMT  0x0e
178 #define IB_6120_LT_STATE_RECOVERIDLE     0x0f
179 
180 /* link state machine states from IBC */
181 #define IB_6120_L_STATE_DOWN             0x0
182 #define IB_6120_L_STATE_INIT             0x1
183 #define IB_6120_L_STATE_ARM              0x2
184 #define IB_6120_L_STATE_ACTIVE           0x3
185 #define IB_6120_L_STATE_ACT_DEFER        0x4
186 
187 static const u8 qib_6120_physportstate[0x20] = {
188 	[IB_6120_LT_STATE_DISABLED] = IB_PHYSPORTSTATE_DISABLED,
189 	[IB_6120_LT_STATE_LINKUP] = IB_PHYSPORTSTATE_LINKUP,
190 	[IB_6120_LT_STATE_POLLACTIVE] = IB_PHYSPORTSTATE_POLL,
191 	[IB_6120_LT_STATE_POLLQUIET] = IB_PHYSPORTSTATE_POLL,
192 	[IB_6120_LT_STATE_SLEEPDELAY] = IB_PHYSPORTSTATE_SLEEP,
193 	[IB_6120_LT_STATE_SLEEPQUIET] = IB_PHYSPORTSTATE_SLEEP,
194 	[IB_6120_LT_STATE_CFGDEBOUNCE] =
195 		IB_PHYSPORTSTATE_CFG_TRAIN,
196 	[IB_6120_LT_STATE_CFGRCVFCFG] =
197 		IB_PHYSPORTSTATE_CFG_TRAIN,
198 	[IB_6120_LT_STATE_CFGWAITRMT] =
199 		IB_PHYSPORTSTATE_CFG_TRAIN,
200 	[IB_6120_LT_STATE_CFGIDLE] = IB_PHYSPORTSTATE_CFG_TRAIN,
201 	[IB_6120_LT_STATE_RECOVERRETRAIN] =
202 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
203 	[IB_6120_LT_STATE_RECOVERWAITRMT] =
204 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
205 	[IB_6120_LT_STATE_RECOVERIDLE] =
206 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
207 	[0x10] = IB_PHYSPORTSTATE_CFG_TRAIN,
208 	[0x11] = IB_PHYSPORTSTATE_CFG_TRAIN,
209 	[0x12] = IB_PHYSPORTSTATE_CFG_TRAIN,
210 	[0x13] = IB_PHYSPORTSTATE_CFG_TRAIN,
211 	[0x14] = IB_PHYSPORTSTATE_CFG_TRAIN,
212 	[0x15] = IB_PHYSPORTSTATE_CFG_TRAIN,
213 	[0x16] = IB_PHYSPORTSTATE_CFG_TRAIN,
214 	[0x17] = IB_PHYSPORTSTATE_CFG_TRAIN
215 };
216 
217 
218 struct qib_chip_specific {
219 	u64 __iomem *cregbase;
220 	u64 *cntrs;
221 	u64 *portcntrs;
222 	void *dummy_hdrq;   /* used after ctxt close */
223 	dma_addr_t dummy_hdrq_phys;
224 	spinlock_t kernel_tid_lock; /* no back to back kernel TID writes */
225 	spinlock_t user_tid_lock; /* no back to back user TID writes */
226 	spinlock_t rcvmod_lock; /* protect rcvctrl shadow changes */
227 	spinlock_t gpio_lock; /* RMW of shadows/regs for ExtCtrl and GPIO */
228 	u64 hwerrmask;
229 	u64 errormask;
230 	u64 gpio_out; /* shadow of kr_gpio_out, for rmw ops */
231 	u64 gpio_mask; /* shadow the gpio mask register */
232 	u64 extctrl; /* shadow the gpio output enable, etc... */
233 	/*
234 	 * these 5 fields are used to establish deltas for IB symbol
235 	 * errors and linkrecovery errors.  They can be reported on
236 	 * some chips during link negotiation prior to INIT, and with
237 	 * DDR when faking DDR negotiations with non-IBTA switches.
238 	 * The chip counters are adjusted at driver unload if there is
239 	 * a non-zero delta.
240 	 */
241 	u64 ibdeltainprog;
242 	u64 ibsymdelta;
243 	u64 ibsymsnap;
244 	u64 iblnkerrdelta;
245 	u64 iblnkerrsnap;
246 	u64 ibcctrl; /* shadow for kr_ibcctrl */
247 	u32 lastlinkrecov; /* link recovery issue */
248 	u32 cntrnamelen;
249 	u32 portcntrnamelen;
250 	u32 ncntrs;
251 	u32 nportcntrs;
252 	/* used with gpio interrupts to implement IB counters */
253 	u32 rxfc_unsupvl_errs;
254 	u32 overrun_thresh_errs;
255 	/*
256 	 * these count only cases where _successive_ LocalLinkIntegrity
257 	 * errors were seen in the receive headers of IB standard packets
258 	 */
259 	u32 lli_errs;
260 	u32 lli_counter;
261 	u64 lli_thresh;
262 	u64 sword; /* total dwords sent (sample result) */
263 	u64 rword; /* total dwords received (sample result) */
264 	u64 spkts; /* total packets sent (sample result) */
265 	u64 rpkts; /* total packets received (sample result) */
266 	u64 xmit_wait; /* # of ticks no data sent (sample result) */
267 	struct timer_list pma_timer;
268 	struct qib_pportdata *ppd;
269 	char emsgbuf[128];
270 	char bitsmsgbuf[64];
271 	u8 pma_sample_status;
272 };
273 
274 /* ibcctrl bits */
275 #define QLOGIC_IB_IBCC_LINKINITCMD_DISABLE 1
276 /* cycle through TS1/TS2 till OK */
277 #define QLOGIC_IB_IBCC_LINKINITCMD_POLL 2
278 /* wait for TS1, then go on */
279 #define QLOGIC_IB_IBCC_LINKINITCMD_SLEEP 3
280 #define QLOGIC_IB_IBCC_LINKINITCMD_SHIFT 16
281 
282 #define QLOGIC_IB_IBCC_LINKCMD_DOWN 1           /* move to 0x11 */
283 #define QLOGIC_IB_IBCC_LINKCMD_ARMED 2          /* move to 0x21 */
284 #define QLOGIC_IB_IBCC_LINKCMD_ACTIVE 3 /* move to 0x31 */
285 #define QLOGIC_IB_IBCC_LINKCMD_SHIFT 18
286 
287 /*
288  * We could have a single register get/put routine, that takes a group type,
289  * but this is somewhat clearer and cleaner.  It also gives us some error
290  * checking.  64 bit register reads should always work, but are inefficient
291  * on opteron (the northbridge always generates 2 separate HT 32 bit reads),
292  * so we use kreg32 wherever possible.  User register and counter register
293  * reads are always 32 bit reads, so only one form of those routines.
294  */
295 
296 /**
297  * qib_read_ureg32 - read 32-bit virtualized per-context register
298  * @dd: device
299  * @regno: register number
300  * @ctxt: context number
301  *
302  * Return the contents of a register that is virtualized to be per context.
303  * Returns -1 on errors (not distinguishable from valid contents at
304  * runtime; we may add a separate error variable at some point).
305  */
306 static inline u32 qib_read_ureg32(const struct qib_devdata *dd,
307 				  enum qib_ureg regno, int ctxt)
308 {
309 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
310 		return 0;
311 
312 	if (dd->userbase)
313 		return readl(regno + (u64 __iomem *)
314 			     ((char __iomem *)dd->userbase +
315 			      dd->ureg_align * ctxt));
316 	else
317 		return readl(regno + (u64 __iomem *)
318 			     (dd->uregbase +
319 			      (char __iomem *)dd->kregbase +
320 			      dd->ureg_align * ctxt));
321 }
322 
323 /**
324  * qib_write_ureg - write 32-bit virtualized per-context register
325  * @dd: device
326  * @regno: register number
327  * @value: value
328  * @ctxt: context
329  *
330  * Write the contents of a register that is virtualized to be per context.
331  */
332 static inline void qib_write_ureg(const struct qib_devdata *dd,
333 				  enum qib_ureg regno, u64 value, int ctxt)
334 {
335 	u64 __iomem *ubase;
336 
337 	if (dd->userbase)
338 		ubase = (u64 __iomem *)
339 			((char __iomem *) dd->userbase +
340 			 dd->ureg_align * ctxt);
341 	else
342 		ubase = (u64 __iomem *)
343 			(dd->uregbase +
344 			 (char __iomem *) dd->kregbase +
345 			 dd->ureg_align * ctxt);
346 
347 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
348 		writeq(value, &ubase[regno]);
349 }
350 
351 static inline u32 qib_read_kreg32(const struct qib_devdata *dd,
352 				  const u16 regno)
353 {
354 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
355 		return -1;
356 	return readl((u32 __iomem *)&dd->kregbase[regno]);
357 }
358 
359 static inline u64 qib_read_kreg64(const struct qib_devdata *dd,
360 				  const u16 regno)
361 {
362 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
363 		return -1;
364 
365 	return readq(&dd->kregbase[regno]);
366 }
367 
368 static inline void qib_write_kreg(const struct qib_devdata *dd,
369 				  const u16 regno, u64 value)
370 {
371 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
372 		writeq(value, &dd->kregbase[regno]);
373 }
374 
375 /**
376  * qib_write_kreg_ctxt - write a device's per-ctxt 64-bit kernel register
377  * @dd: the qlogic_ib device
378  * @regno: the register number to write
379  * @ctxt: the context containing the register
380  * @value: the value to write
381  */
382 static inline void qib_write_kreg_ctxt(const struct qib_devdata *dd,
383 				       const u16 regno, unsigned ctxt,
384 				       u64 value)
385 {
386 	qib_write_kreg(dd, regno + ctxt, value);
387 }
388 
389 static inline void write_6120_creg(const struct qib_devdata *dd,
390 				   u16 regno, u64 value)
391 {
392 	if (dd->cspec->cregbase && (dd->flags & QIB_PRESENT))
393 		writeq(value, &dd->cspec->cregbase[regno]);
394 }
395 
396 static inline u64 read_6120_creg(const struct qib_devdata *dd, u16 regno)
397 {
398 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
399 		return 0;
400 	return readq(&dd->cspec->cregbase[regno]);
401 }
402 
403 static inline u32 read_6120_creg32(const struct qib_devdata *dd, u16 regno)
404 {
405 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
406 		return 0;
407 	return readl(&dd->cspec->cregbase[regno]);
408 }
409 
410 /* kr_control bits */
411 #define QLOGIC_IB_C_RESET 1U
412 
413 /* kr_intstatus, kr_intclear, kr_intmask bits */
414 #define QLOGIC_IB_I_RCVURG_MASK ((1U << 5) - 1)
415 #define QLOGIC_IB_I_RCVURG_SHIFT 0
416 #define QLOGIC_IB_I_RCVAVAIL_MASK ((1U << 5) - 1)
417 #define QLOGIC_IB_I_RCVAVAIL_SHIFT 12
418 
419 #define QLOGIC_IB_C_FREEZEMODE 0x00000002
420 #define QLOGIC_IB_C_LINKENABLE 0x00000004
421 #define QLOGIC_IB_I_ERROR               0x0000000080000000ULL
422 #define QLOGIC_IB_I_SPIOSENT            0x0000000040000000ULL
423 #define QLOGIC_IB_I_SPIOBUFAVAIL        0x0000000020000000ULL
424 #define QLOGIC_IB_I_GPIO                0x0000000010000000ULL
425 #define QLOGIC_IB_I_BITSEXTANT \
426 		((QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT) | \
427 		(QLOGIC_IB_I_RCVAVAIL_MASK << \
428 		 QLOGIC_IB_I_RCVAVAIL_SHIFT) | \
429 		QLOGIC_IB_I_ERROR | QLOGIC_IB_I_SPIOSENT | \
430 		QLOGIC_IB_I_SPIOBUFAVAIL | QLOGIC_IB_I_GPIO)
431 
432 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
433 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK  0x000000000000003fULL
434 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT 0
435 #define QLOGIC_IB_HWE_PCIEPOISONEDTLP      0x0000000010000000ULL
436 #define QLOGIC_IB_HWE_PCIECPLTIMEOUT       0x0000000020000000ULL
437 #define QLOGIC_IB_HWE_PCIEBUSPARITYXTLH    0x0000000040000000ULL
438 #define QLOGIC_IB_HWE_PCIEBUSPARITYXADM    0x0000000080000000ULL
439 #define QLOGIC_IB_HWE_PCIEBUSPARITYRADM    0x0000000100000000ULL
440 #define QLOGIC_IB_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
441 #define QLOGIC_IB_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
442 #define QLOGIC_IB_HWE_PCIE1PLLFAILED       0x0400000000000000ULL
443 #define QLOGIC_IB_HWE_PCIE0PLLFAILED       0x0800000000000000ULL
444 #define QLOGIC_IB_HWE_SERDESPLLFAILED      0x1000000000000000ULL
445 
446 
447 /* kr_extstatus bits */
448 #define QLOGIC_IB_EXTS_FREQSEL 0x2
449 #define QLOGIC_IB_EXTS_SERDESSEL 0x4
450 #define QLOGIC_IB_EXTS_MEMBIST_ENDTEST     0x0000000000004000
451 #define QLOGIC_IB_EXTS_MEMBIST_FOUND       0x0000000000008000
452 
453 /* kr_xgxsconfig bits */
454 #define QLOGIC_IB_XGXS_RESET          0x5ULL
455 
456 #define _QIB_GPIO_SDA_NUM 1
457 #define _QIB_GPIO_SCL_NUM 0
458 
459 /* Bits in GPIO for the added IB link interrupts */
460 #define GPIO_RXUVL_BIT 3
461 #define GPIO_OVRUN_BIT 4
462 #define GPIO_LLI_BIT 5
463 #define GPIO_ERRINTR_MASK 0x38
464 
465 
466 #define QLOGIC_IB_RT_BUFSIZE_MASK 0xe0000000ULL
467 #define QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid) \
468 	((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) >> 29) + 11 - 1)
469 #define QLOGIC_IB_RT_BUFSIZE(tid) (1 << QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid))
470 #define QLOGIC_IB_RT_IS_VALID(tid) \
471 	(((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) && \
472 	 ((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) != QLOGIC_IB_RT_BUFSIZE_MASK)))
473 #define QLOGIC_IB_RT_ADDR_MASK 0x1FFFFFFFULL /* 29 bits valid */
474 #define QLOGIC_IB_RT_ADDR_SHIFT 10
475 
476 #define QLOGIC_IB_R_INTRAVAIL_SHIFT 16
477 #define QLOGIC_IB_R_TAILUPD_SHIFT 31
478 #define IBA6120_R_PKEY_DIS_SHIFT 30
479 
480 #define PBC_6120_VL15_SEND_CTRL (1ULL << 31) /* pbc; VL15; link_buf only */
481 
482 #define IBCBUSFRSPCPARITYERR HWE_MASK(IBCBusFromSPCParityErr)
483 #define IBCBUSTOSPCPARITYERR HWE_MASK(IBCBusToSPCParityErr)
484 
485 #define SYM_MASK_BIT(regname, fldname, bit) ((u64) \
486 	((1ULL << (SYM_LSB(regname, fldname) + (bit)))))
487 
488 #define TXEMEMPARITYERR_PIOBUF \
489 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 0)
490 #define TXEMEMPARITYERR_PIOPBC \
491 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 1)
492 #define TXEMEMPARITYERR_PIOLAUNCHFIFO \
493 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 2)
494 
495 #define RXEMEMPARITYERR_RCVBUF \
496 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 0)
497 #define RXEMEMPARITYERR_LOOKUPQ \
498 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 1)
499 #define RXEMEMPARITYERR_EXPTID \
500 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 2)
501 #define RXEMEMPARITYERR_EAGERTID \
502 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 3)
503 #define RXEMEMPARITYERR_FLAGBUF \
504 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 4)
505 #define RXEMEMPARITYERR_DATAINFO \
506 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 5)
507 #define RXEMEMPARITYERR_HDRINFO \
508 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 6)
509 
510 /* 6120 specific hardware errors... */
511 static const struct qib_hwerror_msgs qib_6120_hwerror_msgs[] = {
512 	/* generic hardware errors */
513 	QLOGIC_IB_HWE_MSG(IBCBUSFRSPCPARITYERR, "QIB2IB Parity"),
514 	QLOGIC_IB_HWE_MSG(IBCBUSTOSPCPARITYERR, "IB2QIB Parity"),
515 
516 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOBUF,
517 			  "TXE PIOBUF Memory Parity"),
518 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOPBC,
519 			  "TXE PIOPBC Memory Parity"),
520 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOLAUNCHFIFO,
521 			  "TXE PIOLAUNCHFIFO Memory Parity"),
522 
523 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_RCVBUF,
524 			  "RXE RCVBUF Memory Parity"),
525 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_LOOKUPQ,
526 			  "RXE LOOKUPQ Memory Parity"),
527 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EAGERTID,
528 			  "RXE EAGERTID Memory Parity"),
529 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EXPTID,
530 			  "RXE EXPTID Memory Parity"),
531 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_FLAGBUF,
532 			  "RXE FLAGBUF Memory Parity"),
533 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_DATAINFO,
534 			  "RXE DATAINFO Memory Parity"),
535 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_HDRINFO,
536 			  "RXE HDRINFO Memory Parity"),
537 
538 	/* chip-specific hardware errors */
539 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEPOISONEDTLP,
540 			  "PCIe Poisoned TLP"),
541 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLTIMEOUT,
542 			  "PCIe completion timeout"),
543 	/*
544 	 * In practice, it's unlikely wthat we'll see PCIe PLL, or bus
545 	 * parity or memory parity error failures, because most likely we
546 	 * won't be able to talk to the core of the chip.  Nonetheless, we
547 	 * might see them, if they are in parts of the PCIe core that aren't
548 	 * essential.
549 	 */
550 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE1PLLFAILED,
551 			  "PCIePLL1"),
552 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE0PLLFAILED,
553 			  "PCIePLL0"),
554 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXTLH,
555 			  "PCIe XTLH core parity"),
556 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXADM,
557 			  "PCIe ADM TX core parity"),
558 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYRADM,
559 			  "PCIe ADM RX core parity"),
560 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_SERDESPLLFAILED,
561 			  "SerDes PLL"),
562 };
563 
564 #define TXE_PIO_PARITY (TXEMEMPARITYERR_PIOBUF | TXEMEMPARITYERR_PIOPBC)
565 #define _QIB_PLL_FAIL (QLOGIC_IB_HWE_COREPLL_FBSLIP |   \
566 		QLOGIC_IB_HWE_COREPLL_RFSLIP)
567 
568 	/* variables for sanity checking interrupt and errors */
569 #define IB_HWE_BITSEXTANT \
570 	(HWE_MASK(RXEMemParityErr) |					\
571 	 HWE_MASK(TXEMemParityErr) |					\
572 	 (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<			\
573 	  QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) |			\
574 	 QLOGIC_IB_HWE_PCIE1PLLFAILED |					\
575 	 QLOGIC_IB_HWE_PCIE0PLLFAILED |					\
576 	 QLOGIC_IB_HWE_PCIEPOISONEDTLP |				\
577 	 QLOGIC_IB_HWE_PCIECPLTIMEOUT |					\
578 	 QLOGIC_IB_HWE_PCIEBUSPARITYXTLH |				\
579 	 QLOGIC_IB_HWE_PCIEBUSPARITYXADM |				\
580 	 QLOGIC_IB_HWE_PCIEBUSPARITYRADM |				\
581 	 HWE_MASK(PowerOnBISTFailed) |					\
582 	 QLOGIC_IB_HWE_COREPLL_FBSLIP |					\
583 	 QLOGIC_IB_HWE_COREPLL_RFSLIP |					\
584 	 QLOGIC_IB_HWE_SERDESPLLFAILED |				\
585 	 HWE_MASK(IBCBusToSPCParityErr) |				\
586 	 HWE_MASK(IBCBusFromSPCParityErr))
587 
588 #define IB_E_BITSEXTANT \
589 	(ERR_MASK(RcvFormatErr) | ERR_MASK(RcvVCRCErr) |		\
590 	 ERR_MASK(RcvICRCErr) | ERR_MASK(RcvMinPktLenErr) |		\
591 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvLongPktLenErr) |	\
592 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvUnexpectedCharErr) | \
593 	 ERR_MASK(RcvUnsupportedVLErr) | ERR_MASK(RcvEBPErr) |		\
594 	 ERR_MASK(RcvIBFlowErr) | ERR_MASK(RcvBadVersionErr) |		\
595 	 ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) |		\
596 	 ERR_MASK(RcvBadTidErr) | ERR_MASK(RcvHdrLenErr) |		\
597 	 ERR_MASK(RcvHdrErr) | ERR_MASK(RcvIBLostLinkErr) |		\
598 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendMaxPktLenErr) |	\
599 	 ERR_MASK(SendUnderRunErr) | ERR_MASK(SendPktLenErr) |		\
600 	 ERR_MASK(SendDroppedSmpPktErr) |				\
601 	 ERR_MASK(SendDroppedDataPktErr) |				\
602 	 ERR_MASK(SendPioArmLaunchErr) |				\
603 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
604 	 ERR_MASK(SendUnsupportedVLErr) | ERR_MASK(IBStatusChanged) |	\
605 	 ERR_MASK(InvalidAddrErr) | ERR_MASK(ResetNegated) |		\
606 	 ERR_MASK(HardwareErr))
607 
608 #define QLOGIC_IB_E_PKTERRS ( \
609 		ERR_MASK(SendPktLenErr) |				\
610 		ERR_MASK(SendDroppedDataPktErr) |			\
611 		ERR_MASK(RcvVCRCErr) |					\
612 		ERR_MASK(RcvICRCErr) |					\
613 		ERR_MASK(RcvShortPktLenErr) |				\
614 		ERR_MASK(RcvEBPErr))
615 
616 /* These are all rcv-related errors which we want to count for stats */
617 #define E_SUM_PKTERRS						\
618 	(ERR_MASK(RcvHdrLenErr) | ERR_MASK(RcvBadTidErr) |		\
619 	 ERR_MASK(RcvBadVersionErr) | ERR_MASK(RcvHdrErr) |		\
620 	 ERR_MASK(RcvLongPktLenErr) | ERR_MASK(RcvShortPktLenErr) |	\
621 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
622 	 ERR_MASK(RcvFormatErr) | ERR_MASK(RcvUnsupportedVLErr) |	\
623 	 ERR_MASK(RcvUnexpectedCharErr) | ERR_MASK(RcvEBPErr))
624 
625 /* These are all send-related errors which we want to count for stats */
626 #define E_SUM_ERRS							\
627 	(ERR_MASK(SendPioArmLaunchErr) |				\
628 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
629 	 ERR_MASK(SendDroppedDataPktErr) |				\
630 	 ERR_MASK(SendDroppedSmpPktErr) |				\
631 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnsupportedVLErr) |	\
632 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
633 	 ERR_MASK(InvalidAddrErr))
634 
635 /*
636  * this is similar to E_SUM_ERRS, but can't ignore armlaunch, don't ignore
637  * errors not related to freeze and cancelling buffers.  Can't ignore
638  * armlaunch because could get more while still cleaning up, and need
639  * to cancel those as they happen.
640  */
641 #define E_SPKT_ERRS_IGNORE \
642 	(ERR_MASK(SendDroppedDataPktErr) |				\
643 	 ERR_MASK(SendDroppedSmpPktErr) |				\
644 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendMinPktLenErr) |	\
645 	 ERR_MASK(SendPktLenErr))
646 
647 /*
648  * these are errors that can occur when the link changes state while
649  * a packet is being sent or received.  This doesn't cover things
650  * like EBP or VCRC that can be the result of a sending having the
651  * link change state, so we receive a "known bad" packet.
652  */
653 #define E_SUM_LINK_PKTERRS		\
654 	(ERR_MASK(SendDroppedDataPktErr) |				\
655 	 ERR_MASK(SendDroppedSmpPktErr) |				\
656 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
657 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
658 	 ERR_MASK(RcvUnexpectedCharErr))
659 
660 static void qib_6120_put_tid_2(struct qib_devdata *, u64 __iomem *,
661 			       u32, unsigned long);
662 
663 /*
664  * On platforms using this chip, and not having ordered WC stores, we
665  * can get TXE parity errors due to speculative reads to the PIO buffers,
666  * and this, due to a chip issue can result in (many) false parity error
667  * reports.  So it's a debug print on those, and an info print on systems
668  * where the speculative reads don't occur.
669  */
670 static void qib_6120_txe_recover(struct qib_devdata *dd)
671 {
672 	if (!qib_unordered_wc())
673 		qib_devinfo(dd->pcidev,
674 			    "Recovering from TXE PIO parity error\n");
675 }
676 
677 /* enable/disable chip from delivering interrupts */
678 static void qib_6120_set_intr_state(struct qib_devdata *dd, u32 enable)
679 {
680 	if (enable) {
681 		if (dd->flags & QIB_BADINTR)
682 			return;
683 		qib_write_kreg(dd, kr_intmask, ~0ULL);
684 		/* force re-interrupt of any pending interrupts. */
685 		qib_write_kreg(dd, kr_intclear, 0ULL);
686 	} else
687 		qib_write_kreg(dd, kr_intmask, 0ULL);
688 }
689 
690 /*
691  * Try to cleanup as much as possible for anything that might have gone
692  * wrong while in freeze mode, such as pio buffers being written by user
693  * processes (causing armlaunch), send errors due to going into freeze mode,
694  * etc., and try to avoid causing extra interrupts while doing so.
695  * Forcibly update the in-memory pioavail register copies after cleanup
696  * because the chip won't do it while in freeze mode (the register values
697  * themselves are kept correct).
698  * Make sure that we don't lose any important interrupts by using the chip
699  * feature that says that writing 0 to a bit in *clear that is set in
700  * *status will cause an interrupt to be generated again (if allowed by
701  * the *mask value).
702  * This is in chip-specific code because of all of the register accesses,
703  * even though the details are similar on most chips
704  */
705 static void qib_6120_clear_freeze(struct qib_devdata *dd)
706 {
707 	/* disable error interrupts, to avoid confusion */
708 	qib_write_kreg(dd, kr_errmask, 0ULL);
709 
710 	/* also disable interrupts; errormask is sometimes overwritten */
711 	qib_6120_set_intr_state(dd, 0);
712 
713 	qib_cancel_sends(dd->pport);
714 
715 	/* clear the freeze, and be sure chip saw it */
716 	qib_write_kreg(dd, kr_control, dd->control);
717 	qib_read_kreg32(dd, kr_scratch);
718 
719 	/* force in-memory update now we are out of freeze */
720 	qib_force_pio_avail_update(dd);
721 
722 	/*
723 	 * force new interrupt if any hwerr, error or interrupt bits are
724 	 * still set, and clear "safe" send packet errors related to freeze
725 	 * and cancelling sends.  Re-enable error interrupts before possible
726 	 * force of re-interrupt on pending interrupts.
727 	 */
728 	qib_write_kreg(dd, kr_hwerrclear, 0ULL);
729 	qib_write_kreg(dd, kr_errclear, E_SPKT_ERRS_IGNORE);
730 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
731 	qib_6120_set_intr_state(dd, 1);
732 }
733 
734 /**
735  * qib_handle_6120_hwerrors - display hardware errors.
736  * @dd: the qlogic_ib device
737  * @msg: the output buffer
738  * @msgl: the size of the output buffer
739  *
740  * Use same msg buffer as regular errors to avoid excessive stack
741  * use.  Most hardware errors are catastrophic, but for right now,
742  * we'll print them and continue.  Reuse the same message buffer as
743  * handle_6120_errors() to avoid excessive stack usage.
744  */
745 static void qib_handle_6120_hwerrors(struct qib_devdata *dd, char *msg,
746 				     size_t msgl)
747 {
748 	u64 hwerrs;
749 	u32 bits, ctrl;
750 	int isfatal = 0;
751 	char *bitsmsg;
752 
753 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
754 	if (!hwerrs)
755 		return;
756 	if (hwerrs == ~0ULL) {
757 		qib_dev_err(dd,
758 			"Read of hardware error status failed (all bits set); ignoring\n");
759 		return;
760 	}
761 	qib_stats.sps_hwerrs++;
762 
763 	/* Always clear the error status register, except MEMBISTFAIL,
764 	 * regardless of whether we continue or stop using the chip.
765 	 * We want that set so we know it failed, even across driver reload.
766 	 * We'll still ignore it in the hwerrmask.  We do this partly for
767 	 * diagnostics, but also for support */
768 	qib_write_kreg(dd, kr_hwerrclear,
769 		       hwerrs & ~HWE_MASK(PowerOnBISTFailed));
770 
771 	hwerrs &= dd->cspec->hwerrmask;
772 
773 	/*
774 	 * Make sure we get this much out, unless told to be quiet,
775 	 * or it's occurred within the last 5 seconds.
776 	 */
777 	if (hwerrs & ~(TXE_PIO_PARITY | RXEMEMPARITYERR_EAGERTID))
778 		qib_devinfo(dd->pcidev,
779 			"Hardware error: hwerr=0x%llx (cleared)\n",
780 			(unsigned long long) hwerrs);
781 
782 	if (hwerrs & ~IB_HWE_BITSEXTANT)
783 		qib_dev_err(dd,
784 			"hwerror interrupt with unknown errors %llx set\n",
785 			(unsigned long long)(hwerrs & ~IB_HWE_BITSEXTANT));
786 
787 	ctrl = qib_read_kreg32(dd, kr_control);
788 	if ((ctrl & QLOGIC_IB_C_FREEZEMODE) && !dd->diag_client) {
789 		/*
790 		 * Parity errors in send memory are recoverable,
791 		 * just cancel the send (if indicated in * sendbuffererror),
792 		 * count the occurrence, unfreeze (if no other handled
793 		 * hardware error bits are set), and continue. They can
794 		 * occur if a processor speculative read is done to the PIO
795 		 * buffer while we are sending a packet, for example.
796 		 */
797 		if (hwerrs & TXE_PIO_PARITY) {
798 			qib_6120_txe_recover(dd);
799 			hwerrs &= ~TXE_PIO_PARITY;
800 		}
801 
802 		if (!hwerrs) {
803 			static u32 freeze_cnt;
804 
805 			freeze_cnt++;
806 			qib_6120_clear_freeze(dd);
807 		} else
808 			isfatal = 1;
809 	}
810 
811 	*msg = '\0';
812 
813 	if (hwerrs & HWE_MASK(PowerOnBISTFailed)) {
814 		isfatal = 1;
815 		strlcat(msg,
816 			"[Memory BIST test failed, InfiniPath hardware unusable]",
817 			msgl);
818 		/* ignore from now on, so disable until driver reloaded */
819 		dd->cspec->hwerrmask &= ~HWE_MASK(PowerOnBISTFailed);
820 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
821 	}
822 
823 	qib_format_hwerrors(hwerrs, qib_6120_hwerror_msgs,
824 			    ARRAY_SIZE(qib_6120_hwerror_msgs), msg, msgl);
825 
826 	bitsmsg = dd->cspec->bitsmsgbuf;
827 	if (hwerrs & (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<
828 		      QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT)) {
829 		bits = (u32) ((hwerrs >>
830 			       QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) &
831 			      QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK);
832 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
833 			 "[PCIe Mem Parity Errs %x] ", bits);
834 		strlcat(msg, bitsmsg, msgl);
835 	}
836 
837 	if (hwerrs & _QIB_PLL_FAIL) {
838 		isfatal = 1;
839 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
840 			 "[PLL failed (%llx), InfiniPath hardware unusable]",
841 			 (unsigned long long) hwerrs & _QIB_PLL_FAIL);
842 		strlcat(msg, bitsmsg, msgl);
843 		/* ignore from now on, so disable until driver reloaded */
844 		dd->cspec->hwerrmask &= ~(hwerrs & _QIB_PLL_FAIL);
845 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
846 	}
847 
848 	if (hwerrs & QLOGIC_IB_HWE_SERDESPLLFAILED) {
849 		/*
850 		 * If it occurs, it is left masked since the external
851 		 * interface is unused
852 		 */
853 		dd->cspec->hwerrmask &= ~QLOGIC_IB_HWE_SERDESPLLFAILED;
854 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
855 	}
856 
857 	if (hwerrs)
858 		/*
859 		 * if any set that we aren't ignoring; only
860 		 * make the complaint once, in case it's stuck
861 		 * or recurring, and we get here multiple
862 		 * times.
863 		 */
864 		qib_dev_err(dd, "%s hardware error\n", msg);
865 	else
866 		*msg = 0; /* recovered from all of them */
867 
868 	if (isfatal && !dd->diag_client) {
869 		qib_dev_err(dd,
870 			"Fatal Hardware Error, no longer usable, SN %.16s\n",
871 			dd->serial);
872 		/*
873 		 * for /sys status file and user programs to print; if no
874 		 * trailing brace is copied, we'll know it was truncated.
875 		 */
876 		if (dd->freezemsg)
877 			snprintf(dd->freezemsg, dd->freezelen,
878 				 "{%s}", msg);
879 		qib_disable_after_error(dd);
880 	}
881 }
882 
883 /*
884  * Decode the error status into strings, deciding whether to always
885  * print * it or not depending on "normal packet errors" vs everything
886  * else.   Return 1 if "real" errors, otherwise 0 if only packet
887  * errors, so caller can decide what to print with the string.
888  */
889 static int qib_decode_6120_err(struct qib_devdata *dd, char *buf, size_t blen,
890 			       u64 err)
891 {
892 	int iserr = 1;
893 
894 	*buf = '\0';
895 	if (err & QLOGIC_IB_E_PKTERRS) {
896 		if (!(err & ~QLOGIC_IB_E_PKTERRS))
897 			iserr = 0;
898 		if ((err & ERR_MASK(RcvICRCErr)) &&
899 		    !(err&(ERR_MASK(RcvVCRCErr)|ERR_MASK(RcvEBPErr))))
900 			strlcat(buf, "CRC ", blen);
901 		if (!iserr)
902 			goto done;
903 	}
904 	if (err & ERR_MASK(RcvHdrLenErr))
905 		strlcat(buf, "rhdrlen ", blen);
906 	if (err & ERR_MASK(RcvBadTidErr))
907 		strlcat(buf, "rbadtid ", blen);
908 	if (err & ERR_MASK(RcvBadVersionErr))
909 		strlcat(buf, "rbadversion ", blen);
910 	if (err & ERR_MASK(RcvHdrErr))
911 		strlcat(buf, "rhdr ", blen);
912 	if (err & ERR_MASK(RcvLongPktLenErr))
913 		strlcat(buf, "rlongpktlen ", blen);
914 	if (err & ERR_MASK(RcvMaxPktLenErr))
915 		strlcat(buf, "rmaxpktlen ", blen);
916 	if (err & ERR_MASK(RcvMinPktLenErr))
917 		strlcat(buf, "rminpktlen ", blen);
918 	if (err & ERR_MASK(SendMinPktLenErr))
919 		strlcat(buf, "sminpktlen ", blen);
920 	if (err & ERR_MASK(RcvFormatErr))
921 		strlcat(buf, "rformaterr ", blen);
922 	if (err & ERR_MASK(RcvUnsupportedVLErr))
923 		strlcat(buf, "runsupvl ", blen);
924 	if (err & ERR_MASK(RcvUnexpectedCharErr))
925 		strlcat(buf, "runexpchar ", blen);
926 	if (err & ERR_MASK(RcvIBFlowErr))
927 		strlcat(buf, "ribflow ", blen);
928 	if (err & ERR_MASK(SendUnderRunErr))
929 		strlcat(buf, "sunderrun ", blen);
930 	if (err & ERR_MASK(SendPioArmLaunchErr))
931 		strlcat(buf, "spioarmlaunch ", blen);
932 	if (err & ERR_MASK(SendUnexpectedPktNumErr))
933 		strlcat(buf, "sunexperrpktnum ", blen);
934 	if (err & ERR_MASK(SendDroppedSmpPktErr))
935 		strlcat(buf, "sdroppedsmppkt ", blen);
936 	if (err & ERR_MASK(SendMaxPktLenErr))
937 		strlcat(buf, "smaxpktlen ", blen);
938 	if (err & ERR_MASK(SendUnsupportedVLErr))
939 		strlcat(buf, "sunsupVL ", blen);
940 	if (err & ERR_MASK(InvalidAddrErr))
941 		strlcat(buf, "invalidaddr ", blen);
942 	if (err & ERR_MASK(RcvEgrFullErr))
943 		strlcat(buf, "rcvegrfull ", blen);
944 	if (err & ERR_MASK(RcvHdrFullErr))
945 		strlcat(buf, "rcvhdrfull ", blen);
946 	if (err & ERR_MASK(IBStatusChanged))
947 		strlcat(buf, "ibcstatuschg ", blen);
948 	if (err & ERR_MASK(RcvIBLostLinkErr))
949 		strlcat(buf, "riblostlink ", blen);
950 	if (err & ERR_MASK(HardwareErr))
951 		strlcat(buf, "hardware ", blen);
952 	if (err & ERR_MASK(ResetNegated))
953 		strlcat(buf, "reset ", blen);
954 done:
955 	return iserr;
956 }
957 
958 /*
959  * Called when we might have an error that is specific to a particular
960  * PIO buffer, and may need to cancel that buffer, so it can be re-used.
961  */
962 static void qib_disarm_6120_senderrbufs(struct qib_pportdata *ppd)
963 {
964 	unsigned long sbuf[2];
965 	struct qib_devdata *dd = ppd->dd;
966 
967 	/*
968 	 * It's possible that sendbuffererror could have bits set; might
969 	 * have already done this as a result of hardware error handling.
970 	 */
971 	sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
972 	sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
973 
974 	if (sbuf[0] || sbuf[1])
975 		qib_disarm_piobufs_set(dd, sbuf,
976 				       dd->piobcnt2k + dd->piobcnt4k);
977 }
978 
979 static int chk_6120_linkrecovery(struct qib_devdata *dd, u64 ibcs)
980 {
981 	int ret = 1;
982 	u32 ibstate = qib_6120_iblink_state(ibcs);
983 	u32 linkrecov = read_6120_creg32(dd, cr_iblinkerrrecov);
984 
985 	if (linkrecov != dd->cspec->lastlinkrecov) {
986 		/* and no more until active again */
987 		dd->cspec->lastlinkrecov = 0;
988 		qib_set_linkstate(dd->pport, QIB_IB_LINKDOWN);
989 		ret = 0;
990 	}
991 	if (ibstate == IB_PORT_ACTIVE)
992 		dd->cspec->lastlinkrecov =
993 			read_6120_creg32(dd, cr_iblinkerrrecov);
994 	return ret;
995 }
996 
997 static void handle_6120_errors(struct qib_devdata *dd, u64 errs)
998 {
999 	char *msg;
1000 	u64 ignore_this_time = 0;
1001 	u64 iserr = 0;
1002 	struct qib_pportdata *ppd = dd->pport;
1003 	u64 mask;
1004 
1005 	/* don't report errors that are masked */
1006 	errs &= dd->cspec->errormask;
1007 	msg = dd->cspec->emsgbuf;
1008 
1009 	/* do these first, they are most important */
1010 	if (errs & ERR_MASK(HardwareErr))
1011 		qib_handle_6120_hwerrors(dd, msg, sizeof(dd->cspec->emsgbuf));
1012 
1013 	if (errs & ~IB_E_BITSEXTANT)
1014 		qib_dev_err(dd,
1015 			"error interrupt with unknown errors %llx set\n",
1016 			(unsigned long long) (errs & ~IB_E_BITSEXTANT));
1017 
1018 	if (errs & E_SUM_ERRS) {
1019 		qib_disarm_6120_senderrbufs(ppd);
1020 		if ((errs & E_SUM_LINK_PKTERRS) &&
1021 		    !(ppd->lflags & QIBL_LINKACTIVE)) {
1022 			/*
1023 			 * This can happen when trying to bring the link
1024 			 * up, but the IB link changes state at the "wrong"
1025 			 * time. The IB logic then complains that the packet
1026 			 * isn't valid.  We don't want to confuse people, so
1027 			 * we just don't print them, except at debug
1028 			 */
1029 			ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1030 		}
1031 	} else if ((errs & E_SUM_LINK_PKTERRS) &&
1032 		   !(ppd->lflags & QIBL_LINKACTIVE)) {
1033 		/*
1034 		 * This can happen when SMA is trying to bring the link
1035 		 * up, but the IB link changes state at the "wrong" time.
1036 		 * The IB logic then complains that the packet isn't
1037 		 * valid.  We don't want to confuse people, so we just
1038 		 * don't print them, except at debug
1039 		 */
1040 		ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1041 	}
1042 
1043 	qib_write_kreg(dd, kr_errclear, errs);
1044 
1045 	errs &= ~ignore_this_time;
1046 	if (!errs)
1047 		goto done;
1048 
1049 	/*
1050 	 * The ones we mask off are handled specially below
1051 	 * or above.
1052 	 */
1053 	mask = ERR_MASK(IBStatusChanged) | ERR_MASK(RcvEgrFullErr) |
1054 		ERR_MASK(RcvHdrFullErr) | ERR_MASK(HardwareErr);
1055 	qib_decode_6120_err(dd, msg, sizeof(dd->cspec->emsgbuf), errs & ~mask);
1056 
1057 	if (errs & E_SUM_PKTERRS)
1058 		qib_stats.sps_rcverrs++;
1059 	if (errs & E_SUM_ERRS)
1060 		qib_stats.sps_txerrs++;
1061 
1062 	iserr = errs & ~(E_SUM_PKTERRS | QLOGIC_IB_E_PKTERRS);
1063 
1064 	if (errs & ERR_MASK(IBStatusChanged)) {
1065 		u64 ibcs = qib_read_kreg64(dd, kr_ibcstatus);
1066 		u32 ibstate = qib_6120_iblink_state(ibcs);
1067 		int handle = 1;
1068 
1069 		if (ibstate != IB_PORT_INIT && dd->cspec->lastlinkrecov)
1070 			handle = chk_6120_linkrecovery(dd, ibcs);
1071 		/*
1072 		 * Since going into a recovery state causes the link state
1073 		 * to go down and since recovery is transitory, it is better
1074 		 * if we "miss" ever seeing the link training state go into
1075 		 * recovery (i.e., ignore this transition for link state
1076 		 * special handling purposes) without updating lastibcstat.
1077 		 */
1078 		if (handle && qib_6120_phys_portstate(ibcs) ==
1079 					    IB_PHYSPORTSTATE_LINK_ERR_RECOVER)
1080 			handle = 0;
1081 		if (handle)
1082 			qib_handle_e_ibstatuschanged(ppd, ibcs);
1083 	}
1084 
1085 	if (errs & ERR_MASK(ResetNegated)) {
1086 		qib_dev_err(dd,
1087 			"Got reset, requires re-init (unload and reload driver)\n");
1088 		dd->flags &= ~QIB_INITTED;  /* needs re-init */
1089 		/* mark as having had error */
1090 		*dd->devstatusp |= QIB_STATUS_HWERROR;
1091 		*dd->pport->statusp &= ~QIB_STATUS_IB_CONF;
1092 	}
1093 
1094 	if (*msg && iserr)
1095 		qib_dev_porterr(dd, ppd->port, "%s error\n", msg);
1096 
1097 	if (ppd->state_wanted & ppd->lflags)
1098 		wake_up_interruptible(&ppd->state_wait);
1099 
1100 	/*
1101 	 * If there were hdrq or egrfull errors, wake up any processes
1102 	 * waiting in poll.  We used to try to check which contexts had
1103 	 * the overflow, but given the cost of that and the chip reads
1104 	 * to support it, it's better to just wake everybody up if we
1105 	 * get an overflow; waiters can poll again if it's not them.
1106 	 */
1107 	if (errs & (ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr))) {
1108 		qib_handle_urcv(dd, ~0U);
1109 		if (errs & ERR_MASK(RcvEgrFullErr))
1110 			qib_stats.sps_buffull++;
1111 		else
1112 			qib_stats.sps_hdrfull++;
1113 	}
1114 done:
1115 	return;
1116 }
1117 
1118 /**
1119  * qib_6120_init_hwerrors - enable hardware errors
1120  * @dd: the qlogic_ib device
1121  *
1122  * now that we have finished initializing everything that might reasonably
1123  * cause a hardware error, and cleared those errors bits as they occur,
1124  * we can enable hardware errors in the mask (potentially enabling
1125  * freeze mode), and enable hardware errors as errors (along with
1126  * everything else) in errormask
1127  */
1128 static void qib_6120_init_hwerrors(struct qib_devdata *dd)
1129 {
1130 	u64 val;
1131 	u64 extsval;
1132 
1133 	extsval = qib_read_kreg64(dd, kr_extstatus);
1134 
1135 	if (!(extsval & QLOGIC_IB_EXTS_MEMBIST_ENDTEST))
1136 		qib_dev_err(dd, "MemBIST did not complete!\n");
1137 
1138 	/* init so all hwerrors interrupt, and enter freeze, ajdust below */
1139 	val = ~0ULL;
1140 	if (dd->minrev < 2) {
1141 		/*
1142 		 * Avoid problem with internal interface bus parity
1143 		 * checking. Fixed in Rev2.
1144 		 */
1145 		val &= ~QLOGIC_IB_HWE_PCIEBUSPARITYRADM;
1146 	}
1147 	/* avoid some intel cpu's speculative read freeze mode issue */
1148 	val &= ~TXEMEMPARITYERR_PIOBUF;
1149 
1150 	dd->cspec->hwerrmask = val;
1151 
1152 	qib_write_kreg(dd, kr_hwerrclear, ~HWE_MASK(PowerOnBISTFailed));
1153 	qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1154 
1155 	/* clear all */
1156 	qib_write_kreg(dd, kr_errclear, ~0ULL);
1157 	/* enable errors that are masked, at least this first time. */
1158 	qib_write_kreg(dd, kr_errmask, ~0ULL);
1159 	dd->cspec->errormask = qib_read_kreg64(dd, kr_errmask);
1160 	/* clear any interrupts up to this point (ints still not enabled) */
1161 	qib_write_kreg(dd, kr_intclear, ~0ULL);
1162 
1163 	qib_write_kreg(dd, kr_rcvbthqp,
1164 		       dd->qpn_mask << (QIB_6120_RcvBTHQP_BTHQP_Mask_LSB - 1) |
1165 		       QIB_KD_QP);
1166 }
1167 
1168 /*
1169  * Disable and enable the armlaunch error.  Used for PIO bandwidth testing
1170  * on chips that are count-based, rather than trigger-based.  There is no
1171  * reference counting, but that's also fine, given the intended use.
1172  * Only chip-specific because it's all register accesses
1173  */
1174 static void qib_set_6120_armlaunch(struct qib_devdata *dd, u32 enable)
1175 {
1176 	if (enable) {
1177 		qib_write_kreg(dd, kr_errclear,
1178 			       ERR_MASK(SendPioArmLaunchErr));
1179 		dd->cspec->errormask |= ERR_MASK(SendPioArmLaunchErr);
1180 	} else
1181 		dd->cspec->errormask &= ~ERR_MASK(SendPioArmLaunchErr);
1182 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1183 }
1184 
1185 /*
1186  * Formerly took parameter <which> in pre-shifted,
1187  * pre-merged form with LinkCmd and LinkInitCmd
1188  * together, and assuming the zero was NOP.
1189  */
1190 static void qib_set_ib_6120_lstate(struct qib_pportdata *ppd, u16 linkcmd,
1191 				   u16 linitcmd)
1192 {
1193 	u64 mod_wd;
1194 	struct qib_devdata *dd = ppd->dd;
1195 	unsigned long flags;
1196 
1197 	if (linitcmd == QLOGIC_IB_IBCC_LINKINITCMD_DISABLE) {
1198 		/*
1199 		 * If we are told to disable, note that so link-recovery
1200 		 * code does not attempt to bring us back up.
1201 		 */
1202 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1203 		ppd->lflags |= QIBL_IB_LINK_DISABLED;
1204 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1205 	} else if (linitcmd || linkcmd == QLOGIC_IB_IBCC_LINKCMD_DOWN) {
1206 		/*
1207 		 * Any other linkinitcmd will lead to LINKDOWN and then
1208 		 * to INIT (if all is well), so clear flag to let
1209 		 * link-recovery code attempt to bring us back up.
1210 		 */
1211 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1212 		ppd->lflags &= ~QIBL_IB_LINK_DISABLED;
1213 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1214 	}
1215 
1216 	mod_wd = (linkcmd << QLOGIC_IB_IBCC_LINKCMD_SHIFT) |
1217 		(linitcmd << QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1218 
1219 	qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl | mod_wd);
1220 	/* write to chip to prevent back-to-back writes of control reg */
1221 	qib_write_kreg(dd, kr_scratch, 0);
1222 }
1223 
1224 /**
1225  * qib_6120_bringup_serdes - bring up the serdes
1226  * @ppd: the qlogic_ib device
1227  */
1228 static int qib_6120_bringup_serdes(struct qib_pportdata *ppd)
1229 {
1230 	struct qib_devdata *dd = ppd->dd;
1231 	u64 val, config1, prev_val, hwstat, ibc;
1232 
1233 	/* Put IBC in reset, sends disabled */
1234 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1235 	qib_write_kreg(dd, kr_control, 0ULL);
1236 
1237 	dd->cspec->ibdeltainprog = 1;
1238 	dd->cspec->ibsymsnap = read_6120_creg32(dd, cr_ibsymbolerr);
1239 	dd->cspec->iblnkerrsnap = read_6120_creg32(dd, cr_iblinkerrrecov);
1240 
1241 	/* flowcontrolwatermark is in units of KBytes */
1242 	ibc = 0x5ULL << SYM_LSB(IBCCtrl, FlowCtrlWaterMark);
1243 	/*
1244 	 * How often flowctrl sent.  More or less in usecs; balance against
1245 	 * watermark value, so that in theory senders always get a flow
1246 	 * control update in time to not let the IB link go idle.
1247 	 */
1248 	ibc |= 0x3ULL << SYM_LSB(IBCCtrl, FlowCtrlPeriod);
1249 	/* max error tolerance */
1250 	dd->cspec->lli_thresh = 0xf;
1251 	ibc |= (u64) dd->cspec->lli_thresh << SYM_LSB(IBCCtrl, PhyerrThreshold);
1252 	/* use "real" buffer space for */
1253 	ibc |= 4ULL << SYM_LSB(IBCCtrl, CreditScale);
1254 	/* IB credit flow control. */
1255 	ibc |= 0xfULL << SYM_LSB(IBCCtrl, OverrunThreshold);
1256 	/*
1257 	 * set initial max size pkt IBC will send, including ICRC; it's the
1258 	 * PIO buffer size in dwords, less 1; also see qib_set_mtu()
1259 	 */
1260 	ibc |= ((u64)(ppd->ibmaxlen >> 2) + 1) << SYM_LSB(IBCCtrl, MaxPktLen);
1261 	dd->cspec->ibcctrl = ibc; /* without linkcmd or linkinitcmd! */
1262 
1263 	/* initially come up waiting for TS1, without sending anything. */
1264 	val = dd->cspec->ibcctrl | (QLOGIC_IB_IBCC_LINKINITCMD_DISABLE <<
1265 		QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1266 	qib_write_kreg(dd, kr_ibcctrl, val);
1267 
1268 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1269 	config1 = qib_read_kreg64(dd, kr_serdes_cfg1);
1270 
1271 	/*
1272 	 * Force reset on, also set rxdetect enable.  Must do before reading
1273 	 * serdesstatus at least for simulation, or some of the bits in
1274 	 * serdes status will come back as undefined and cause simulation
1275 	 * failures
1276 	 */
1277 	val |= SYM_MASK(SerdesCfg0, ResetPLL) |
1278 		SYM_MASK(SerdesCfg0, RxDetEnX) |
1279 		(SYM_MASK(SerdesCfg0, L1PwrDnA) |
1280 		 SYM_MASK(SerdesCfg0, L1PwrDnB) |
1281 		 SYM_MASK(SerdesCfg0, L1PwrDnC) |
1282 		 SYM_MASK(SerdesCfg0, L1PwrDnD));
1283 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1284 	/* be sure chip saw it */
1285 	qib_read_kreg64(dd, kr_scratch);
1286 	udelay(5);              /* need pll reset set at least for a bit */
1287 	/*
1288 	 * after PLL is reset, set the per-lane Resets and TxIdle and
1289 	 * clear the PLL reset and rxdetect (to get falling edge).
1290 	 * Leave L1PWR bits set (permanently)
1291 	 */
1292 	val &= ~(SYM_MASK(SerdesCfg0, RxDetEnX) |
1293 		 SYM_MASK(SerdesCfg0, ResetPLL) |
1294 		 (SYM_MASK(SerdesCfg0, L1PwrDnA) |
1295 		  SYM_MASK(SerdesCfg0, L1PwrDnB) |
1296 		  SYM_MASK(SerdesCfg0, L1PwrDnC) |
1297 		  SYM_MASK(SerdesCfg0, L1PwrDnD)));
1298 	val |= (SYM_MASK(SerdesCfg0, ResetA) |
1299 		SYM_MASK(SerdesCfg0, ResetB) |
1300 		SYM_MASK(SerdesCfg0, ResetC) |
1301 		SYM_MASK(SerdesCfg0, ResetD)) |
1302 		SYM_MASK(SerdesCfg0, TxIdeEnX);
1303 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1304 	/* be sure chip saw it */
1305 	(void) qib_read_kreg64(dd, kr_scratch);
1306 	/* need PLL reset clear for at least 11 usec before lane
1307 	 * resets cleared; give it a few more to be sure */
1308 	udelay(15);
1309 	val &= ~((SYM_MASK(SerdesCfg0, ResetA) |
1310 		  SYM_MASK(SerdesCfg0, ResetB) |
1311 		  SYM_MASK(SerdesCfg0, ResetC) |
1312 		  SYM_MASK(SerdesCfg0, ResetD)) |
1313 		 SYM_MASK(SerdesCfg0, TxIdeEnX));
1314 
1315 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1316 	/* be sure chip saw it */
1317 	(void) qib_read_kreg64(dd, kr_scratch);
1318 
1319 	val = qib_read_kreg64(dd, kr_xgxs_cfg);
1320 	prev_val = val;
1321 	if (val & QLOGIC_IB_XGXS_RESET)
1322 		val &= ~QLOGIC_IB_XGXS_RESET;
1323 	if (SYM_FIELD(val, XGXSCfg, polarity_inv) != ppd->rx_pol_inv) {
1324 		/* need to compensate for Tx inversion in partner */
1325 		val &= ~SYM_MASK(XGXSCfg, polarity_inv);
1326 		val |= (u64)ppd->rx_pol_inv << SYM_LSB(XGXSCfg, polarity_inv);
1327 	}
1328 	if (val != prev_val)
1329 		qib_write_kreg(dd, kr_xgxs_cfg, val);
1330 
1331 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1332 
1333 	/* clear current and de-emphasis bits */
1334 	config1 &= ~0x0ffffffff00ULL;
1335 	/* set current to 20ma */
1336 	config1 |= 0x00000000000ULL;
1337 	/* set de-emphasis to -5.68dB */
1338 	config1 |= 0x0cccc000000ULL;
1339 	qib_write_kreg(dd, kr_serdes_cfg1, config1);
1340 
1341 	/* base and port guid same for single port */
1342 	ppd->guid = dd->base_guid;
1343 
1344 	/*
1345 	 * the process of setting and un-resetting the serdes normally
1346 	 * causes a serdes PLL error, so check for that and clear it
1347 	 * here.  Also clearr hwerr bit in errstatus, but not others.
1348 	 */
1349 	hwstat = qib_read_kreg64(dd, kr_hwerrstatus);
1350 	if (hwstat) {
1351 		/* should just have PLL, clear all set, in an case */
1352 		qib_write_kreg(dd, kr_hwerrclear, hwstat);
1353 		qib_write_kreg(dd, kr_errclear, ERR_MASK(HardwareErr));
1354 	}
1355 
1356 	dd->control |= QLOGIC_IB_C_LINKENABLE;
1357 	dd->control &= ~QLOGIC_IB_C_FREEZEMODE;
1358 	qib_write_kreg(dd, kr_control, dd->control);
1359 
1360 	return 0;
1361 }
1362 
1363 /**
1364  * qib_6120_quiet_serdes - set serdes to txidle
1365  * @ppd: physical port of the qlogic_ib device
1366  * Called when driver is being unloaded
1367  */
1368 static void qib_6120_quiet_serdes(struct qib_pportdata *ppd)
1369 {
1370 	struct qib_devdata *dd = ppd->dd;
1371 	u64 val;
1372 
1373 	qib_set_ib_6120_lstate(ppd, 0, QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1374 
1375 	/* disable IBC */
1376 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1377 	qib_write_kreg(dd, kr_control,
1378 		       dd->control | QLOGIC_IB_C_FREEZEMODE);
1379 
1380 	if (dd->cspec->ibsymdelta || dd->cspec->iblnkerrdelta ||
1381 	    dd->cspec->ibdeltainprog) {
1382 		u64 diagc;
1383 
1384 		/* enable counter writes */
1385 		diagc = qib_read_kreg64(dd, kr_hwdiagctrl);
1386 		qib_write_kreg(dd, kr_hwdiagctrl,
1387 			       diagc | SYM_MASK(HwDiagCtrl, CounterWrEnable));
1388 
1389 		if (dd->cspec->ibsymdelta || dd->cspec->ibdeltainprog) {
1390 			val = read_6120_creg32(dd, cr_ibsymbolerr);
1391 			if (dd->cspec->ibdeltainprog)
1392 				val -= val - dd->cspec->ibsymsnap;
1393 			val -= dd->cspec->ibsymdelta;
1394 			write_6120_creg(dd, cr_ibsymbolerr, val);
1395 		}
1396 		if (dd->cspec->iblnkerrdelta || dd->cspec->ibdeltainprog) {
1397 			val = read_6120_creg32(dd, cr_iblinkerrrecov);
1398 			if (dd->cspec->ibdeltainprog)
1399 				val -= val - dd->cspec->iblnkerrsnap;
1400 			val -= dd->cspec->iblnkerrdelta;
1401 			write_6120_creg(dd, cr_iblinkerrrecov, val);
1402 		}
1403 
1404 		/* and disable counter writes */
1405 		qib_write_kreg(dd, kr_hwdiagctrl, diagc);
1406 	}
1407 
1408 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1409 	val |= SYM_MASK(SerdesCfg0, TxIdeEnX);
1410 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1411 }
1412 
1413 /**
1414  * qib_6120_setup_setextled - set the state of the two external LEDs
1415  * @ppd: the qlogic_ib device
1416  * @on: whether the link is up or not
1417  *
1418  * The exact combo of LEDs if on is true is determined by looking
1419  * at the ibcstatus.
1420  * These LEDs indicate the physical and logical state of IB link.
1421  * For this chip (at least with recommended board pinouts), LED1
1422  * is Yellow (logical state) and LED2 is Green (physical state),
1423  *
1424  * Note:  We try to match the Mellanox HCA LED behavior as best
1425  * we can.  Green indicates physical link state is OK (something is
1426  * plugged in, and we can train).
1427  * Amber indicates the link is logically up (ACTIVE).
1428  * Mellanox further blinks the amber LED to indicate data packet
1429  * activity, but we have no hardware support for that, so it would
1430  * require waking up every 10-20 msecs and checking the counters
1431  * on the chip, and then turning the LED off if appropriate.  That's
1432  * visible overhead, so not something we will do.
1433  *
1434  */
1435 static void qib_6120_setup_setextled(struct qib_pportdata *ppd, u32 on)
1436 {
1437 	u64 extctl, val, lst, ltst;
1438 	unsigned long flags;
1439 	struct qib_devdata *dd = ppd->dd;
1440 
1441 	/*
1442 	 * The diags use the LED to indicate diag info, so we leave
1443 	 * the external LED alone when the diags are running.
1444 	 */
1445 	if (dd->diag_client)
1446 		return;
1447 
1448 	/* Allow override of LED display for, e.g. Locating system in rack */
1449 	if (ppd->led_override) {
1450 		ltst = (ppd->led_override & QIB_LED_PHYS) ?
1451 			IB_PHYSPORTSTATE_LINKUP : IB_PHYSPORTSTATE_DISABLED,
1452 		lst = (ppd->led_override & QIB_LED_LOG) ?
1453 			IB_PORT_ACTIVE : IB_PORT_DOWN;
1454 	} else if (on) {
1455 		val = qib_read_kreg64(dd, kr_ibcstatus);
1456 		ltst = qib_6120_phys_portstate(val);
1457 		lst = qib_6120_iblink_state(val);
1458 	} else {
1459 		ltst = 0;
1460 		lst = 0;
1461 	}
1462 
1463 	spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
1464 	extctl = dd->cspec->extctrl & ~(SYM_MASK(EXTCtrl, LEDPriPortGreenOn) |
1465 				 SYM_MASK(EXTCtrl, LEDPriPortYellowOn));
1466 
1467 	if (ltst == IB_PHYSPORTSTATE_LINKUP)
1468 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortYellowOn);
1469 	if (lst == IB_PORT_ACTIVE)
1470 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortGreenOn);
1471 	dd->cspec->extctrl = extctl;
1472 	qib_write_kreg(dd, kr_extctrl, extctl);
1473 	spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
1474 }
1475 
1476 /**
1477  * qib_6120_setup_cleanup - clean up any per-chip chip-specific stuff
1478  * @dd: the qlogic_ib device
1479  *
1480  * This is called during driver unload.
1481 */
1482 static void qib_6120_setup_cleanup(struct qib_devdata *dd)
1483 {
1484 	qib_free_irq(dd);
1485 	kfree(dd->cspec->cntrs);
1486 	kfree(dd->cspec->portcntrs);
1487 	if (dd->cspec->dummy_hdrq) {
1488 		dma_free_coherent(&dd->pcidev->dev,
1489 				  ALIGN(dd->rcvhdrcnt *
1490 					dd->rcvhdrentsize *
1491 					sizeof(u32), PAGE_SIZE),
1492 				  dd->cspec->dummy_hdrq,
1493 				  dd->cspec->dummy_hdrq_phys);
1494 		dd->cspec->dummy_hdrq = NULL;
1495 	}
1496 }
1497 
1498 static void qib_wantpiobuf_6120_intr(struct qib_devdata *dd, u32 needint)
1499 {
1500 	unsigned long flags;
1501 
1502 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
1503 	if (needint)
1504 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOIntBufAvail);
1505 	else
1506 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOIntBufAvail);
1507 	qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
1508 	qib_write_kreg(dd, kr_scratch, 0ULL);
1509 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
1510 }
1511 
1512 /*
1513  * handle errors and unusual events first, separate function
1514  * to improve cache hits for fast path interrupt handling
1515  */
1516 static noinline void unlikely_6120_intr(struct qib_devdata *dd, u64 istat)
1517 {
1518 	if (unlikely(istat & ~QLOGIC_IB_I_BITSEXTANT))
1519 		qib_dev_err(dd, "interrupt with unknown interrupts %Lx set\n",
1520 			    istat & ~QLOGIC_IB_I_BITSEXTANT);
1521 
1522 	if (istat & QLOGIC_IB_I_ERROR) {
1523 		u64 estat = 0;
1524 
1525 		qib_stats.sps_errints++;
1526 		estat = qib_read_kreg64(dd, kr_errstatus);
1527 		if (!estat)
1528 			qib_devinfo(dd->pcidev,
1529 				"error interrupt (%Lx), but no error bits set!\n",
1530 				istat);
1531 		handle_6120_errors(dd, estat);
1532 	}
1533 
1534 	if (istat & QLOGIC_IB_I_GPIO) {
1535 		u32 gpiostatus;
1536 		u32 to_clear = 0;
1537 
1538 		/*
1539 		 * GPIO_3..5 on IBA6120 Rev2 chips indicate
1540 		 * errors that we need to count.
1541 		 */
1542 		gpiostatus = qib_read_kreg32(dd, kr_gpio_status);
1543 		/* First the error-counter case. */
1544 		if (gpiostatus & GPIO_ERRINTR_MASK) {
1545 			/* want to clear the bits we see asserted. */
1546 			to_clear |= (gpiostatus & GPIO_ERRINTR_MASK);
1547 
1548 			/*
1549 			 * Count appropriately, clear bits out of our copy,
1550 			 * as they have been "handled".
1551 			 */
1552 			if (gpiostatus & (1 << GPIO_RXUVL_BIT))
1553 				dd->cspec->rxfc_unsupvl_errs++;
1554 			if (gpiostatus & (1 << GPIO_OVRUN_BIT))
1555 				dd->cspec->overrun_thresh_errs++;
1556 			if (gpiostatus & (1 << GPIO_LLI_BIT))
1557 				dd->cspec->lli_errs++;
1558 			gpiostatus &= ~GPIO_ERRINTR_MASK;
1559 		}
1560 		if (gpiostatus) {
1561 			/*
1562 			 * Some unexpected bits remain. If they could have
1563 			 * caused the interrupt, complain and clear.
1564 			 * To avoid repetition of this condition, also clear
1565 			 * the mask. It is almost certainly due to error.
1566 			 */
1567 			const u32 mask = qib_read_kreg32(dd, kr_gpio_mask);
1568 
1569 			/*
1570 			 * Also check that the chip reflects our shadow,
1571 			 * and report issues, If they caused the interrupt.
1572 			 * we will suppress by refreshing from the shadow.
1573 			 */
1574 			if (mask & gpiostatus) {
1575 				to_clear |= (gpiostatus & mask);
1576 				dd->cspec->gpio_mask &= ~(gpiostatus & mask);
1577 				qib_write_kreg(dd, kr_gpio_mask,
1578 					       dd->cspec->gpio_mask);
1579 			}
1580 		}
1581 		if (to_clear)
1582 			qib_write_kreg(dd, kr_gpio_clear, (u64) to_clear);
1583 	}
1584 }
1585 
1586 static irqreturn_t qib_6120intr(int irq, void *data)
1587 {
1588 	struct qib_devdata *dd = data;
1589 	irqreturn_t ret;
1590 	u32 istat, ctxtrbits, rmask, crcs = 0;
1591 	unsigned i;
1592 
1593 	if ((dd->flags & (QIB_PRESENT | QIB_BADINTR)) != QIB_PRESENT) {
1594 		/*
1595 		 * This return value is not great, but we do not want the
1596 		 * interrupt core code to remove our interrupt handler
1597 		 * because we don't appear to be handling an interrupt
1598 		 * during a chip reset.
1599 		 */
1600 		ret = IRQ_HANDLED;
1601 		goto bail;
1602 	}
1603 
1604 	istat = qib_read_kreg32(dd, kr_intstatus);
1605 
1606 	if (unlikely(!istat)) {
1607 		ret = IRQ_NONE; /* not our interrupt, or already handled */
1608 		goto bail;
1609 	}
1610 	if (unlikely(istat == -1)) {
1611 		qib_bad_intrstatus(dd);
1612 		/* don't know if it was our interrupt or not */
1613 		ret = IRQ_NONE;
1614 		goto bail;
1615 	}
1616 
1617 	this_cpu_inc(*dd->int_counter);
1618 
1619 	if (unlikely(istat & (~QLOGIC_IB_I_BITSEXTANT |
1620 			      QLOGIC_IB_I_GPIO | QLOGIC_IB_I_ERROR)))
1621 		unlikely_6120_intr(dd, istat);
1622 
1623 	/*
1624 	 * Clear the interrupt bits we found set, relatively early, so we
1625 	 * "know" know the chip will have seen this by the time we process
1626 	 * the queue, and will re-interrupt if necessary.  The processor
1627 	 * itself won't take the interrupt again until we return.
1628 	 */
1629 	qib_write_kreg(dd, kr_intclear, istat);
1630 
1631 	/*
1632 	 * Handle kernel receive queues before checking for pio buffers
1633 	 * available since receives can overflow; piobuf waiters can afford
1634 	 * a few extra cycles, since they were waiting anyway.
1635 	 */
1636 	ctxtrbits = istat &
1637 		((QLOGIC_IB_I_RCVAVAIL_MASK << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1638 		 (QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT));
1639 	if (ctxtrbits) {
1640 		rmask = (1U << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1641 			(1U << QLOGIC_IB_I_RCVURG_SHIFT);
1642 		for (i = 0; i < dd->first_user_ctxt; i++) {
1643 			if (ctxtrbits & rmask) {
1644 				ctxtrbits &= ~rmask;
1645 				crcs += qib_kreceive(dd->rcd[i],
1646 						     &dd->cspec->lli_counter,
1647 						     NULL);
1648 			}
1649 			rmask <<= 1;
1650 		}
1651 		if (crcs) {
1652 			u32 cntr = dd->cspec->lli_counter;
1653 
1654 			cntr += crcs;
1655 			if (cntr) {
1656 				if (cntr > dd->cspec->lli_thresh) {
1657 					dd->cspec->lli_counter = 0;
1658 					dd->cspec->lli_errs++;
1659 				} else
1660 					dd->cspec->lli_counter += cntr;
1661 			}
1662 		}
1663 
1664 
1665 		if (ctxtrbits) {
1666 			ctxtrbits =
1667 				(ctxtrbits >> QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1668 				(ctxtrbits >> QLOGIC_IB_I_RCVURG_SHIFT);
1669 			qib_handle_urcv(dd, ctxtrbits);
1670 		}
1671 	}
1672 
1673 	if ((istat & QLOGIC_IB_I_SPIOBUFAVAIL) && (dd->flags & QIB_INITTED))
1674 		qib_ib_piobufavail(dd);
1675 
1676 	ret = IRQ_HANDLED;
1677 bail:
1678 	return ret;
1679 }
1680 
1681 /*
1682  * Set up our chip-specific interrupt handler
1683  * The interrupt type has already been setup, so
1684  * we just need to do the registration and error checking.
1685  */
1686 static void qib_setup_6120_interrupt(struct qib_devdata *dd)
1687 {
1688 	int ret;
1689 
1690 	/*
1691 	 * If the chip supports added error indication via GPIO pins,
1692 	 * enable interrupts on those bits so the interrupt routine
1693 	 * can count the events. Also set flag so interrupt routine
1694 	 * can know they are expected.
1695 	 */
1696 	if (SYM_FIELD(dd->revision, Revision_R,
1697 		      ChipRevMinor) > 1) {
1698 		/* Rev2+ reports extra errors via internal GPIO pins */
1699 		dd->cspec->gpio_mask |= GPIO_ERRINTR_MASK;
1700 		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1701 	}
1702 
1703 	ret = pci_request_irq(dd->pcidev, 0, qib_6120intr, NULL, dd,
1704 			      QIB_DRV_NAME);
1705 	if (ret)
1706 		qib_dev_err(dd,
1707 			    "Couldn't setup interrupt (irq=%d): %d\n",
1708 			    pci_irq_vector(dd->pcidev, 0), ret);
1709 }
1710 
1711 /**
1712  * pe_boardname - fill in the board name
1713  * @dd: the qlogic_ib device
1714  *
1715  * info is based on the board revision register
1716  */
1717 static void pe_boardname(struct qib_devdata *dd)
1718 {
1719 	u32 boardid;
1720 
1721 	boardid = SYM_FIELD(dd->revision, Revision,
1722 			    BoardID);
1723 
1724 	switch (boardid) {
1725 	case 2:
1726 		dd->boardname = "InfiniPath_QLE7140";
1727 		break;
1728 	default:
1729 		qib_dev_err(dd, "Unknown 6120 board with ID %u\n", boardid);
1730 		dd->boardname = "Unknown_InfiniPath_6120";
1731 		break;
1732 	}
1733 
1734 	if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
1735 		qib_dev_err(dd,
1736 			    "Unsupported InfiniPath hardware revision %u.%u!\n",
1737 			    dd->majrev, dd->minrev);
1738 
1739 	snprintf(dd->boardversion, sizeof(dd->boardversion),
1740 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
1741 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
1742 		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, Arch),
1743 		 dd->majrev, dd->minrev,
1744 		 (unsigned int)SYM_FIELD(dd->revision, Revision_R, SW));
1745 }
1746 
1747 /*
1748  * This routine sleeps, so it can only be called from user context, not
1749  * from interrupt context.  If we need interrupt context, we can split
1750  * it into two routines.
1751  */
1752 static int qib_6120_setup_reset(struct qib_devdata *dd)
1753 {
1754 	u64 val;
1755 	int i;
1756 	int ret;
1757 	u16 cmdval;
1758 	u8 int_line, clinesz;
1759 
1760 	qib_pcie_getcmd(dd, &cmdval, &int_line, &clinesz);
1761 
1762 	/* Use ERROR so it shows up in logs, etc. */
1763 	qib_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->unit);
1764 
1765 	/* no interrupts till re-initted */
1766 	qib_6120_set_intr_state(dd, 0);
1767 
1768 	dd->cspec->ibdeltainprog = 0;
1769 	dd->cspec->ibsymdelta = 0;
1770 	dd->cspec->iblnkerrdelta = 0;
1771 
1772 	/*
1773 	 * Keep chip from being accessed until we are ready.  Use
1774 	 * writeq() directly, to allow the write even though QIB_PRESENT
1775 	 * isn't set.
1776 	 */
1777 	dd->flags &= ~(QIB_INITTED | QIB_PRESENT);
1778 	/* so we check interrupts work again */
1779 	dd->z_int_counter = qib_int_counter(dd);
1780 	val = dd->control | QLOGIC_IB_C_RESET;
1781 	writeq(val, &dd->kregbase[kr_control]);
1782 	mb(); /* prevent compiler re-ordering around actual reset */
1783 
1784 	for (i = 1; i <= 5; i++) {
1785 		/*
1786 		 * Allow MBIST, etc. to complete; longer on each retry.
1787 		 * We sometimes get machine checks from bus timeout if no
1788 		 * response, so for now, make it *really* long.
1789 		 */
1790 		msleep(1000 + (1 + i) * 2000);
1791 
1792 		qib_pcie_reenable(dd, cmdval, int_line, clinesz);
1793 
1794 		/*
1795 		 * Use readq directly, so we don't need to mark it as PRESENT
1796 		 * until we get a successful indication that all is well.
1797 		 */
1798 		val = readq(&dd->kregbase[kr_revision]);
1799 		if (val == dd->revision) {
1800 			dd->flags |= QIB_PRESENT; /* it's back */
1801 			ret = qib_reinit_intr(dd);
1802 			goto bail;
1803 		}
1804 	}
1805 	ret = 0; /* failed */
1806 
1807 bail:
1808 	if (ret) {
1809 		if (qib_pcie_params(dd, dd->lbus_width, NULL))
1810 			qib_dev_err(dd,
1811 				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
1812 		/* clear the reset error, init error/hwerror mask */
1813 		qib_6120_init_hwerrors(dd);
1814 		/* for Rev2 error interrupts; nop for rev 1 */
1815 		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1816 		/* clear the reset error, init error/hwerror mask */
1817 		qib_6120_init_hwerrors(dd);
1818 	}
1819 	return ret;
1820 }
1821 
1822 /**
1823  * qib_6120_put_tid - write a TID in chip
1824  * @dd: the qlogic_ib device
1825  * @tidptr: pointer to the expected TID (in chip) to update
1826  * @type: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1827  * for expected
1828  * @pa: physical address of in memory buffer; tidinvalid if freeing
1829  *
1830  * This exists as a separate routine to allow for special locking etc.
1831  * It's used for both the full cleanup on exit, as well as the normal
1832  * setup and teardown.
1833  */
1834 static void qib_6120_put_tid(struct qib_devdata *dd, u64 __iomem *tidptr,
1835 			     u32 type, unsigned long pa)
1836 {
1837 	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1838 	unsigned long flags;
1839 	int tidx;
1840 	spinlock_t *tidlockp; /* select appropriate spinlock */
1841 
1842 	if (!dd->kregbase)
1843 		return;
1844 
1845 	if (pa != dd->tidinvalid) {
1846 		if (pa & ((1U << 11) - 1)) {
1847 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1848 				    pa);
1849 			return;
1850 		}
1851 		pa >>= 11;
1852 		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1853 			qib_dev_err(dd,
1854 				"Physical page address 0x%lx larger than supported\n",
1855 				pa);
1856 			return;
1857 		}
1858 
1859 		if (type == RCVHQ_RCV_TYPE_EAGER)
1860 			pa |= dd->tidtemplate;
1861 		else /* for now, always full 4KB page */
1862 			pa |= 2 << 29;
1863 	}
1864 
1865 	/*
1866 	 * Avoid chip issue by writing the scratch register
1867 	 * before and after the TID, and with an io write barrier.
1868 	 * We use a spinlock around the writes, so they can't intermix
1869 	 * with other TID (eager or expected) writes (the chip problem
1870 	 * is triggered by back to back TID writes). Unfortunately, this
1871 	 * call can be done from interrupt level for the ctxt 0 eager TIDs,
1872 	 * so we have to use irqsave locks.
1873 	 */
1874 	/*
1875 	 * Assumes tidptr always > egrtidbase
1876 	 * if type == RCVHQ_RCV_TYPE_EAGER.
1877 	 */
1878 	tidx = tidptr - dd->egrtidbase;
1879 
1880 	tidlockp = (type == RCVHQ_RCV_TYPE_EAGER && tidx < dd->rcvhdrcnt)
1881 		? &dd->cspec->kernel_tid_lock : &dd->cspec->user_tid_lock;
1882 	spin_lock_irqsave(tidlockp, flags);
1883 	qib_write_kreg(dd, kr_scratch, 0xfeeddeaf);
1884 	writel(pa, tidp32);
1885 	qib_write_kreg(dd, kr_scratch, 0xdeadbeef);
1886 	spin_unlock_irqrestore(tidlockp, flags);
1887 }
1888 
1889 /**
1890  * qib_6120_put_tid_2 - write a TID in chip, Revision 2 or higher
1891  * @dd: the qlogic_ib device
1892  * @tidptr: pointer to the expected TID (in chip) to update
1893  * @type: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1894  * for expected
1895  * @pa: physical address of in memory buffer; tidinvalid if freeing
1896  *
1897  * This exists as a separate routine to allow for selection of the
1898  * appropriate "flavor". The static calls in cleanup just use the
1899  * revision-agnostic form, as they are not performance critical.
1900  */
1901 static void qib_6120_put_tid_2(struct qib_devdata *dd, u64 __iomem *tidptr,
1902 			       u32 type, unsigned long pa)
1903 {
1904 	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1905 
1906 	if (!dd->kregbase)
1907 		return;
1908 
1909 	if (pa != dd->tidinvalid) {
1910 		if (pa & ((1U << 11) - 1)) {
1911 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1912 				    pa);
1913 			return;
1914 		}
1915 		pa >>= 11;
1916 		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1917 			qib_dev_err(dd,
1918 				"Physical page address 0x%lx larger than supported\n",
1919 				pa);
1920 			return;
1921 		}
1922 
1923 		if (type == RCVHQ_RCV_TYPE_EAGER)
1924 			pa |= dd->tidtemplate;
1925 		else /* for now, always full 4KB page */
1926 			pa |= 2 << 29;
1927 	}
1928 	writel(pa, tidp32);
1929 }
1930 
1931 
1932 /**
1933  * qib_6120_clear_tids - clear all TID entries for a context, expected and eager
1934  * @dd: the qlogic_ib device
1935  * @rcd: the context
1936  *
1937  * clear all TID entries for a context, expected and eager.
1938  * Used from qib_close().  On this chip, TIDs are only 32 bits,
1939  * not 64, but they are still on 64 bit boundaries, so tidbase
1940  * is declared as u64 * for the pointer math, even though we write 32 bits
1941  */
1942 static void qib_6120_clear_tids(struct qib_devdata *dd,
1943 				struct qib_ctxtdata *rcd)
1944 {
1945 	u64 __iomem *tidbase;
1946 	unsigned long tidinv;
1947 	u32 ctxt;
1948 	int i;
1949 
1950 	if (!dd->kregbase || !rcd)
1951 		return;
1952 
1953 	ctxt = rcd->ctxt;
1954 
1955 	tidinv = dd->tidinvalid;
1956 	tidbase = (u64 __iomem *)
1957 		((char __iomem *)(dd->kregbase) +
1958 		 dd->rcvtidbase +
1959 		 ctxt * dd->rcvtidcnt * sizeof(*tidbase));
1960 
1961 	for (i = 0; i < dd->rcvtidcnt; i++)
1962 		/* use func pointer because could be one of two funcs */
1963 		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
1964 				  tidinv);
1965 
1966 	tidbase = (u64 __iomem *)
1967 		((char __iomem *)(dd->kregbase) +
1968 		 dd->rcvegrbase +
1969 		 rcd->rcvegr_tid_base * sizeof(*tidbase));
1970 
1971 	for (i = 0; i < rcd->rcvegrcnt; i++)
1972 		/* use func pointer because could be one of two funcs */
1973 		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EAGER,
1974 				  tidinv);
1975 }
1976 
1977 /**
1978  * qib_6120_tidtemplate - setup constants for TID updates
1979  * @dd: the qlogic_ib device
1980  *
1981  * We setup stuff that we use a lot, to avoid calculating each time
1982  */
1983 static void qib_6120_tidtemplate(struct qib_devdata *dd)
1984 {
1985 	u32 egrsize = dd->rcvegrbufsize;
1986 
1987 	/*
1988 	 * For now, we always allocate 4KB buffers (at init) so we can
1989 	 * receive max size packets.  We may want a module parameter to
1990 	 * specify 2KB or 4KB and/or make be per ctxt instead of per device
1991 	 * for those who want to reduce memory footprint.  Note that the
1992 	 * rcvhdrentsize size must be large enough to hold the largest
1993 	 * IB header (currently 96 bytes) that we expect to handle (plus of
1994 	 * course the 2 dwords of RHF).
1995 	 */
1996 	if (egrsize == 2048)
1997 		dd->tidtemplate = 1U << 29;
1998 	else if (egrsize == 4096)
1999 		dd->tidtemplate = 2U << 29;
2000 	dd->tidinvalid = 0;
2001 }
2002 
2003 int __attribute__((weak)) qib_unordered_wc(void)
2004 {
2005 	return 0;
2006 }
2007 
2008 /**
2009  * qib_6120_get_base_info - set chip-specific flags for user code
2010  * @rcd: the qlogic_ib ctxt
2011  * @kinfo: qib_base_info pointer
2012  *
2013  * We set the PCIE flag because the lower bandwidth on PCIe vs
2014  * HyperTransport can affect some user packet algorithms.
2015  */
2016 static int qib_6120_get_base_info(struct qib_ctxtdata *rcd,
2017 				  struct qib_base_info *kinfo)
2018 {
2019 	if (qib_unordered_wc())
2020 		kinfo->spi_runtime_flags |= QIB_RUNTIME_FORCE_WC_ORDER;
2021 
2022 	kinfo->spi_runtime_flags |= QIB_RUNTIME_PCIE |
2023 		QIB_RUNTIME_FORCE_PIOAVAIL | QIB_RUNTIME_PIO_REGSWAPPED;
2024 	return 0;
2025 }
2026 
2027 
2028 static struct qib_message_header *
2029 qib_6120_get_msgheader(struct qib_devdata *dd, __le32 *rhf_addr)
2030 {
2031 	return (struct qib_message_header *)
2032 		&rhf_addr[sizeof(u64) / sizeof(u32)];
2033 }
2034 
2035 static void qib_6120_config_ctxts(struct qib_devdata *dd)
2036 {
2037 	dd->ctxtcnt = qib_read_kreg32(dd, kr_portcnt);
2038 	if (qib_n_krcv_queues > 1) {
2039 		dd->first_user_ctxt = qib_n_krcv_queues * dd->num_pports;
2040 		if (dd->first_user_ctxt > dd->ctxtcnt)
2041 			dd->first_user_ctxt = dd->ctxtcnt;
2042 		dd->qpn_mask = dd->first_user_ctxt <= 2 ? 2 : 6;
2043 	} else
2044 		dd->first_user_ctxt = dd->num_pports;
2045 	dd->n_krcv_queues = dd->first_user_ctxt;
2046 }
2047 
2048 static void qib_update_6120_usrhead(struct qib_ctxtdata *rcd, u64 hd,
2049 				    u32 updegr, u32 egrhd, u32 npkts)
2050 {
2051 	if (updegr)
2052 		qib_write_ureg(rcd->dd, ur_rcvegrindexhead, egrhd, rcd->ctxt);
2053 	qib_write_ureg(rcd->dd, ur_rcvhdrhead, hd, rcd->ctxt);
2054 }
2055 
2056 static u32 qib_6120_hdrqempty(struct qib_ctxtdata *rcd)
2057 {
2058 	u32 head, tail;
2059 
2060 	head = qib_read_ureg32(rcd->dd, ur_rcvhdrhead, rcd->ctxt);
2061 	if (rcd->rcvhdrtail_kvaddr)
2062 		tail = qib_get_rcvhdrtail(rcd);
2063 	else
2064 		tail = qib_read_ureg32(rcd->dd, ur_rcvhdrtail, rcd->ctxt);
2065 	return head == tail;
2066 }
2067 
2068 /*
2069  * Used when we close any ctxt, for DMA already in flight
2070  * at close.  Can't be done until we know hdrq size, so not
2071  * early in chip init.
2072  */
2073 static void alloc_dummy_hdrq(struct qib_devdata *dd)
2074 {
2075 	dd->cspec->dummy_hdrq = dma_alloc_coherent(&dd->pcidev->dev,
2076 					dd->rcd[0]->rcvhdrq_size,
2077 					&dd->cspec->dummy_hdrq_phys,
2078 					GFP_ATOMIC | __GFP_COMP);
2079 	if (!dd->cspec->dummy_hdrq) {
2080 		qib_devinfo(dd->pcidev, "Couldn't allocate dummy hdrq\n");
2081 		/* fallback to just 0'ing */
2082 		dd->cspec->dummy_hdrq_phys = 0UL;
2083 	}
2084 }
2085 
2086 /*
2087  * Modify the RCVCTRL register in chip-specific way. This
2088  * is a function because bit positions and (future) register
2089  * location is chip-specific, but the needed operations are
2090  * generic. <op> is a bit-mask because we often want to
2091  * do multiple modifications.
2092  */
2093 static void rcvctrl_6120_mod(struct qib_pportdata *ppd, unsigned int op,
2094 			     int ctxt)
2095 {
2096 	struct qib_devdata *dd = ppd->dd;
2097 	u64 mask, val;
2098 	unsigned long flags;
2099 
2100 	spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2101 
2102 	if (op & QIB_RCVCTRL_TAILUPD_ENB)
2103 		dd->rcvctrl |= (1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2104 	if (op & QIB_RCVCTRL_TAILUPD_DIS)
2105 		dd->rcvctrl &= ~(1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2106 	if (op & QIB_RCVCTRL_PKEY_ENB)
2107 		dd->rcvctrl &= ~(1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2108 	if (op & QIB_RCVCTRL_PKEY_DIS)
2109 		dd->rcvctrl |= (1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2110 	if (ctxt < 0)
2111 		mask = (1ULL << dd->ctxtcnt) - 1;
2112 	else
2113 		mask = (1ULL << ctxt);
2114 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2115 		/* always done for specific ctxt */
2116 		dd->rcvctrl |= (mask << SYM_LSB(RcvCtrl, PortEnable));
2117 		if (!(dd->flags & QIB_NODMA_RTAIL))
2118 			dd->rcvctrl |= 1ULL << QLOGIC_IB_R_TAILUPD_SHIFT;
2119 		/* Write these registers before the context is enabled. */
2120 		qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2121 			dd->rcd[ctxt]->rcvhdrqtailaddr_phys);
2122 		qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2123 			dd->rcd[ctxt]->rcvhdrq_phys);
2124 
2125 		if (ctxt == 0 && !dd->cspec->dummy_hdrq)
2126 			alloc_dummy_hdrq(dd);
2127 	}
2128 	if (op & QIB_RCVCTRL_CTXT_DIS)
2129 		dd->rcvctrl &= ~(mask << SYM_LSB(RcvCtrl, PortEnable));
2130 	if (op & QIB_RCVCTRL_INTRAVAIL_ENB)
2131 		dd->rcvctrl |= (mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2132 	if (op & QIB_RCVCTRL_INTRAVAIL_DIS)
2133 		dd->rcvctrl &= ~(mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2134 	qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2135 	if ((op & QIB_RCVCTRL_INTRAVAIL_ENB) && dd->rhdrhead_intr_off) {
2136 		/* arm rcv interrupt */
2137 		val = qib_read_ureg32(dd, ur_rcvhdrhead, ctxt) |
2138 			dd->rhdrhead_intr_off;
2139 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2140 	}
2141 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2142 		/*
2143 		 * Init the context registers also; if we were
2144 		 * disabled, tail and head should both be zero
2145 		 * already from the enable, but since we don't
2146 		 * know, we have to do it explicitly.
2147 		 */
2148 		val = qib_read_ureg32(dd, ur_rcvegrindextail, ctxt);
2149 		qib_write_ureg(dd, ur_rcvegrindexhead, val, ctxt);
2150 
2151 		val = qib_read_ureg32(dd, ur_rcvhdrtail, ctxt);
2152 		dd->rcd[ctxt]->head = val;
2153 		/* If kctxt, interrupt on next receive. */
2154 		if (ctxt < dd->first_user_ctxt)
2155 			val |= dd->rhdrhead_intr_off;
2156 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2157 	}
2158 	if (op & QIB_RCVCTRL_CTXT_DIS) {
2159 		/*
2160 		 * Be paranoid, and never write 0's to these, just use an
2161 		 * unused page.  Of course,
2162 		 * rcvhdraddr points to a large chunk of memory, so this
2163 		 * could still trash things, but at least it won't trash
2164 		 * page 0, and by disabling the ctxt, it should stop "soon",
2165 		 * even if a packet or two is in already in flight after we
2166 		 * disabled the ctxt.  Only 6120 has this issue.
2167 		 */
2168 		if (ctxt >= 0) {
2169 			qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2170 					    dd->cspec->dummy_hdrq_phys);
2171 			qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2172 					    dd->cspec->dummy_hdrq_phys);
2173 		} else {
2174 			unsigned i;
2175 
2176 			for (i = 0; i < dd->cfgctxts; i++) {
2177 				qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr,
2178 					    i, dd->cspec->dummy_hdrq_phys);
2179 				qib_write_kreg_ctxt(dd, kr_rcvhdraddr,
2180 					    i, dd->cspec->dummy_hdrq_phys);
2181 			}
2182 		}
2183 	}
2184 	spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2185 }
2186 
2187 /*
2188  * Modify the SENDCTRL register in chip-specific way. This
2189  * is a function there may be multiple such registers with
2190  * slightly different layouts. Only operations actually used
2191  * are implemented yet.
2192  * Chip requires no back-back sendctrl writes, so write
2193  * scratch register after writing sendctrl
2194  */
2195 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op)
2196 {
2197 	struct qib_devdata *dd = ppd->dd;
2198 	u64 tmp_dd_sendctrl;
2199 	unsigned long flags;
2200 
2201 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
2202 
2203 	/* First the ones that are "sticky", saved in shadow */
2204 	if (op & QIB_SENDCTRL_CLEAR)
2205 		dd->sendctrl = 0;
2206 	if (op & QIB_SENDCTRL_SEND_DIS)
2207 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOEnable);
2208 	else if (op & QIB_SENDCTRL_SEND_ENB)
2209 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOEnable);
2210 	if (op & QIB_SENDCTRL_AVAIL_DIS)
2211 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2212 	else if (op & QIB_SENDCTRL_AVAIL_ENB)
2213 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOBufAvailUpd);
2214 
2215 	if (op & QIB_SENDCTRL_DISARM_ALL) {
2216 		u32 i, last;
2217 
2218 		tmp_dd_sendctrl = dd->sendctrl;
2219 		/*
2220 		 * disarm any that are not yet launched, disabling sends
2221 		 * and updates until done.
2222 		 */
2223 		last = dd->piobcnt2k + dd->piobcnt4k;
2224 		tmp_dd_sendctrl &=
2225 			~(SYM_MASK(SendCtrl, PIOEnable) |
2226 			  SYM_MASK(SendCtrl, PIOBufAvailUpd));
2227 		for (i = 0; i < last; i++) {
2228 			qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl |
2229 				       SYM_MASK(SendCtrl, Disarm) | i);
2230 			qib_write_kreg(dd, kr_scratch, 0);
2231 		}
2232 	}
2233 
2234 	tmp_dd_sendctrl = dd->sendctrl;
2235 
2236 	if (op & QIB_SENDCTRL_FLUSH)
2237 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Abort);
2238 	if (op & QIB_SENDCTRL_DISARM)
2239 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Disarm) |
2240 			((op & QIB_6120_SendCtrl_DisarmPIOBuf_RMASK) <<
2241 			 SYM_LSB(SendCtrl, DisarmPIOBuf));
2242 	if (op & QIB_SENDCTRL_AVAIL_BLIP)
2243 		tmp_dd_sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2244 
2245 	qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl);
2246 	qib_write_kreg(dd, kr_scratch, 0);
2247 
2248 	if (op & QIB_SENDCTRL_AVAIL_BLIP) {
2249 		qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
2250 		qib_write_kreg(dd, kr_scratch, 0);
2251 	}
2252 
2253 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
2254 
2255 	if (op & QIB_SENDCTRL_FLUSH) {
2256 		u32 v;
2257 		/*
2258 		 * ensure writes have hit chip, then do a few
2259 		 * more reads, to allow DMA of pioavail registers
2260 		 * to occur, so in-memory copy is in sync with
2261 		 * the chip.  Not always safe to sleep.
2262 		 */
2263 		v = qib_read_kreg32(dd, kr_scratch);
2264 		qib_write_kreg(dd, kr_scratch, v);
2265 		v = qib_read_kreg32(dd, kr_scratch);
2266 		qib_write_kreg(dd, kr_scratch, v);
2267 		qib_read_kreg32(dd, kr_scratch);
2268 	}
2269 }
2270 
2271 /**
2272  * qib_portcntr_6120 - read a per-port counter
2273  * @ppd: the qlogic_ib device
2274  * @reg: the counter to snapshot
2275  */
2276 static u64 qib_portcntr_6120(struct qib_pportdata *ppd, u32 reg)
2277 {
2278 	u64 ret = 0ULL;
2279 	struct qib_devdata *dd = ppd->dd;
2280 	u16 creg;
2281 	/* 0xffff for unimplemented or synthesized counters */
2282 	static const u16 xlator[] = {
2283 		[QIBPORTCNTR_PKTSEND] = cr_pktsend,
2284 		[QIBPORTCNTR_WORDSEND] = cr_wordsend,
2285 		[QIBPORTCNTR_PSXMITDATA] = 0xffff,
2286 		[QIBPORTCNTR_PSXMITPKTS] = 0xffff,
2287 		[QIBPORTCNTR_PSXMITWAIT] = 0xffff,
2288 		[QIBPORTCNTR_SENDSTALL] = cr_sendstall,
2289 		[QIBPORTCNTR_PKTRCV] = cr_pktrcv,
2290 		[QIBPORTCNTR_PSRCVDATA] = 0xffff,
2291 		[QIBPORTCNTR_PSRCVPKTS] = 0xffff,
2292 		[QIBPORTCNTR_RCVEBP] = cr_rcvebp,
2293 		[QIBPORTCNTR_RCVOVFL] = cr_rcvovfl,
2294 		[QIBPORTCNTR_WORDRCV] = cr_wordrcv,
2295 		[QIBPORTCNTR_RXDROPPKT] = cr_rxdroppkt,
2296 		[QIBPORTCNTR_RXLOCALPHYERR] = 0xffff,
2297 		[QIBPORTCNTR_RXVLERR] = 0xffff,
2298 		[QIBPORTCNTR_ERRICRC] = cr_erricrc,
2299 		[QIBPORTCNTR_ERRVCRC] = cr_errvcrc,
2300 		[QIBPORTCNTR_ERRLPCRC] = cr_errlpcrc,
2301 		[QIBPORTCNTR_BADFORMAT] = cr_badformat,
2302 		[QIBPORTCNTR_ERR_RLEN] = cr_err_rlen,
2303 		[QIBPORTCNTR_IBSYMBOLERR] = cr_ibsymbolerr,
2304 		[QIBPORTCNTR_INVALIDRLEN] = cr_invalidrlen,
2305 		[QIBPORTCNTR_UNSUPVL] = cr_txunsupvl,
2306 		[QIBPORTCNTR_EXCESSBUFOVFL] = 0xffff,
2307 		[QIBPORTCNTR_ERRLINK] = cr_errlink,
2308 		[QIBPORTCNTR_IBLINKDOWN] = cr_iblinkdown,
2309 		[QIBPORTCNTR_IBLINKERRRECOV] = cr_iblinkerrrecov,
2310 		[QIBPORTCNTR_LLI] = 0xffff,
2311 		[QIBPORTCNTR_PSINTERVAL] = 0xffff,
2312 		[QIBPORTCNTR_PSSTART] = 0xffff,
2313 		[QIBPORTCNTR_PSSTAT] = 0xffff,
2314 		[QIBPORTCNTR_VL15PKTDROP] = 0xffff,
2315 		[QIBPORTCNTR_ERRPKEY] = cr_errpkey,
2316 		[QIBPORTCNTR_KHDROVFL] = 0xffff,
2317 	};
2318 
2319 	if (reg >= ARRAY_SIZE(xlator)) {
2320 		qib_devinfo(ppd->dd->pcidev,
2321 			 "Unimplemented portcounter %u\n", reg);
2322 		goto done;
2323 	}
2324 	creg = xlator[reg];
2325 
2326 	/* handle counters requests not implemented as chip counters */
2327 	if (reg == QIBPORTCNTR_LLI)
2328 		ret = dd->cspec->lli_errs;
2329 	else if (reg == QIBPORTCNTR_EXCESSBUFOVFL)
2330 		ret = dd->cspec->overrun_thresh_errs;
2331 	else if (reg == QIBPORTCNTR_KHDROVFL) {
2332 		int i;
2333 
2334 		/* sum over all kernel contexts */
2335 		for (i = 0; i < dd->first_user_ctxt; i++)
2336 			ret += read_6120_creg32(dd, cr_portovfl + i);
2337 	} else if (reg == QIBPORTCNTR_PSSTAT)
2338 		ret = dd->cspec->pma_sample_status;
2339 	if (creg == 0xffff)
2340 		goto done;
2341 
2342 	/*
2343 	 * only fast incrementing counters are 64bit; use 32 bit reads to
2344 	 * avoid two independent reads when on opteron
2345 	 */
2346 	if (creg == cr_wordsend || creg == cr_wordrcv ||
2347 	    creg == cr_pktsend || creg == cr_pktrcv)
2348 		ret = read_6120_creg(dd, creg);
2349 	else
2350 		ret = read_6120_creg32(dd, creg);
2351 	if (creg == cr_ibsymbolerr) {
2352 		if (dd->cspec->ibdeltainprog)
2353 			ret -= ret - dd->cspec->ibsymsnap;
2354 		ret -= dd->cspec->ibsymdelta;
2355 	} else if (creg == cr_iblinkerrrecov) {
2356 		if (dd->cspec->ibdeltainprog)
2357 			ret -= ret - dd->cspec->iblnkerrsnap;
2358 		ret -= dd->cspec->iblnkerrdelta;
2359 	}
2360 	if (reg == QIBPORTCNTR_RXDROPPKT) /* add special cased count */
2361 		ret += dd->cspec->rxfc_unsupvl_errs;
2362 
2363 done:
2364 	return ret;
2365 }
2366 
2367 /*
2368  * Device counter names (not port-specific), one line per stat,
2369  * single string.  Used by utilities like ipathstats to print the stats
2370  * in a way which works for different versions of drivers, without changing
2371  * the utility.  Names need to be 12 chars or less (w/o newline), for proper
2372  * display by utility.
2373  * Non-error counters are first.
2374  * Start of "error" conters is indicated by a leading "E " on the first
2375  * "error" counter, and doesn't count in label length.
2376  * The EgrOvfl list needs to be last so we truncate them at the configured
2377  * context count for the device.
2378  * cntr6120indices contains the corresponding register indices.
2379  */
2380 static const char cntr6120names[] =
2381 	"Interrupts\n"
2382 	"HostBusStall\n"
2383 	"E RxTIDFull\n"
2384 	"RxTIDInvalid\n"
2385 	"Ctxt0EgrOvfl\n"
2386 	"Ctxt1EgrOvfl\n"
2387 	"Ctxt2EgrOvfl\n"
2388 	"Ctxt3EgrOvfl\n"
2389 	"Ctxt4EgrOvfl\n";
2390 
2391 static const size_t cntr6120indices[] = {
2392 	cr_lbint,
2393 	cr_lbflowstall,
2394 	cr_errtidfull,
2395 	cr_errtidvalid,
2396 	cr_portovfl + 0,
2397 	cr_portovfl + 1,
2398 	cr_portovfl + 2,
2399 	cr_portovfl + 3,
2400 	cr_portovfl + 4,
2401 };
2402 
2403 /*
2404  * same as cntr6120names and cntr6120indices, but for port-specific counters.
2405  * portcntr6120indices is somewhat complicated by some registers needing
2406  * adjustments of various kinds, and those are ORed with _PORT_VIRT_FLAG
2407  */
2408 static const char portcntr6120names[] =
2409 	"TxPkt\n"
2410 	"TxFlowPkt\n"
2411 	"TxWords\n"
2412 	"RxPkt\n"
2413 	"RxFlowPkt\n"
2414 	"RxWords\n"
2415 	"TxFlowStall\n"
2416 	"E IBStatusChng\n"
2417 	"IBLinkDown\n"
2418 	"IBLnkRecov\n"
2419 	"IBRxLinkErr\n"
2420 	"IBSymbolErr\n"
2421 	"RxLLIErr\n"
2422 	"RxBadFormat\n"
2423 	"RxBadLen\n"
2424 	"RxBufOvrfl\n"
2425 	"RxEBP\n"
2426 	"RxFlowCtlErr\n"
2427 	"RxICRCerr\n"
2428 	"RxLPCRCerr\n"
2429 	"RxVCRCerr\n"
2430 	"RxInvalLen\n"
2431 	"RxInvalPKey\n"
2432 	"RxPktDropped\n"
2433 	"TxBadLength\n"
2434 	"TxDropped\n"
2435 	"TxInvalLen\n"
2436 	"TxUnderrun\n"
2437 	"TxUnsupVL\n"
2438 	;
2439 
2440 #define _PORT_VIRT_FLAG 0x8000 /* "virtual", need adjustments */
2441 static const size_t portcntr6120indices[] = {
2442 	QIBPORTCNTR_PKTSEND | _PORT_VIRT_FLAG,
2443 	cr_pktsendflow,
2444 	QIBPORTCNTR_WORDSEND | _PORT_VIRT_FLAG,
2445 	QIBPORTCNTR_PKTRCV | _PORT_VIRT_FLAG,
2446 	cr_pktrcvflowctrl,
2447 	QIBPORTCNTR_WORDRCV | _PORT_VIRT_FLAG,
2448 	QIBPORTCNTR_SENDSTALL | _PORT_VIRT_FLAG,
2449 	cr_ibstatuschange,
2450 	QIBPORTCNTR_IBLINKDOWN | _PORT_VIRT_FLAG,
2451 	QIBPORTCNTR_IBLINKERRRECOV | _PORT_VIRT_FLAG,
2452 	QIBPORTCNTR_ERRLINK | _PORT_VIRT_FLAG,
2453 	QIBPORTCNTR_IBSYMBOLERR | _PORT_VIRT_FLAG,
2454 	QIBPORTCNTR_LLI | _PORT_VIRT_FLAG,
2455 	QIBPORTCNTR_BADFORMAT | _PORT_VIRT_FLAG,
2456 	QIBPORTCNTR_ERR_RLEN | _PORT_VIRT_FLAG,
2457 	QIBPORTCNTR_RCVOVFL | _PORT_VIRT_FLAG,
2458 	QIBPORTCNTR_RCVEBP | _PORT_VIRT_FLAG,
2459 	cr_rcvflowctrl_err,
2460 	QIBPORTCNTR_ERRICRC | _PORT_VIRT_FLAG,
2461 	QIBPORTCNTR_ERRLPCRC | _PORT_VIRT_FLAG,
2462 	QIBPORTCNTR_ERRVCRC | _PORT_VIRT_FLAG,
2463 	QIBPORTCNTR_INVALIDRLEN | _PORT_VIRT_FLAG,
2464 	QIBPORTCNTR_ERRPKEY | _PORT_VIRT_FLAG,
2465 	QIBPORTCNTR_RXDROPPKT | _PORT_VIRT_FLAG,
2466 	cr_invalidslen,
2467 	cr_senddropped,
2468 	cr_errslen,
2469 	cr_sendunderrun,
2470 	cr_txunsupvl,
2471 };
2472 
2473 /* do all the setup to make the counter reads efficient later */
2474 static void init_6120_cntrnames(struct qib_devdata *dd)
2475 {
2476 	int i, j = 0;
2477 	char *s;
2478 
2479 	for (i = 0, s = (char *)cntr6120names; s && j <= dd->cfgctxts;
2480 	     i++) {
2481 		/* we always have at least one counter before the egrovfl */
2482 		if (!j && !strncmp("Ctxt0EgrOvfl", s + 1, 12))
2483 			j = 1;
2484 		s = strchr(s + 1, '\n');
2485 		if (s && j)
2486 			j++;
2487 	}
2488 	dd->cspec->ncntrs = i;
2489 	if (!s)
2490 		/* full list; size is without terminating null */
2491 		dd->cspec->cntrnamelen = sizeof(cntr6120names) - 1;
2492 	else
2493 		dd->cspec->cntrnamelen = 1 + s - cntr6120names;
2494 	dd->cspec->cntrs = kmalloc_array(dd->cspec->ncntrs, sizeof(u64),
2495 					 GFP_KERNEL);
2496 
2497 	for (i = 0, s = (char *)portcntr6120names; s; i++)
2498 		s = strchr(s + 1, '\n');
2499 	dd->cspec->nportcntrs = i - 1;
2500 	dd->cspec->portcntrnamelen = sizeof(portcntr6120names) - 1;
2501 	dd->cspec->portcntrs = kmalloc_array(dd->cspec->nportcntrs,
2502 					     sizeof(u64),
2503 					     GFP_KERNEL);
2504 }
2505 
2506 static u32 qib_read_6120cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
2507 			      u64 **cntrp)
2508 {
2509 	u32 ret;
2510 
2511 	if (namep) {
2512 		ret = dd->cspec->cntrnamelen;
2513 		if (pos >= ret)
2514 			ret = 0; /* final read after getting everything */
2515 		else
2516 			*namep = (char *)cntr6120names;
2517 	} else {
2518 		u64 *cntr = dd->cspec->cntrs;
2519 		int i;
2520 
2521 		ret = dd->cspec->ncntrs * sizeof(u64);
2522 		if (!cntr || pos >= ret) {
2523 			/* everything read, or couldn't get memory */
2524 			ret = 0;
2525 			goto done;
2526 		}
2527 		if (pos >= ret) {
2528 			ret = 0; /* final read after getting everything */
2529 			goto done;
2530 		}
2531 		*cntrp = cntr;
2532 		for (i = 0; i < dd->cspec->ncntrs; i++)
2533 			*cntr++ = read_6120_creg32(dd, cntr6120indices[i]);
2534 	}
2535 done:
2536 	return ret;
2537 }
2538 
2539 static u32 qib_read_6120portcntrs(struct qib_devdata *dd, loff_t pos, u32 port,
2540 				  char **namep, u64 **cntrp)
2541 {
2542 	u32 ret;
2543 
2544 	if (namep) {
2545 		ret = dd->cspec->portcntrnamelen;
2546 		if (pos >= ret)
2547 			ret = 0; /* final read after getting everything */
2548 		else
2549 			*namep = (char *)portcntr6120names;
2550 	} else {
2551 		u64 *cntr = dd->cspec->portcntrs;
2552 		struct qib_pportdata *ppd = &dd->pport[port];
2553 		int i;
2554 
2555 		ret = dd->cspec->nportcntrs * sizeof(u64);
2556 		if (!cntr || pos >= ret) {
2557 			/* everything read, or couldn't get memory */
2558 			ret = 0;
2559 			goto done;
2560 		}
2561 		*cntrp = cntr;
2562 		for (i = 0; i < dd->cspec->nportcntrs; i++) {
2563 			if (portcntr6120indices[i] & _PORT_VIRT_FLAG)
2564 				*cntr++ = qib_portcntr_6120(ppd,
2565 					portcntr6120indices[i] &
2566 					~_PORT_VIRT_FLAG);
2567 			else
2568 				*cntr++ = read_6120_creg32(dd,
2569 					   portcntr6120indices[i]);
2570 		}
2571 	}
2572 done:
2573 	return ret;
2574 }
2575 
2576 static void qib_chk_6120_errormask(struct qib_devdata *dd)
2577 {
2578 	static u32 fixed;
2579 	u32 ctrl;
2580 	unsigned long errormask;
2581 	unsigned long hwerrs;
2582 
2583 	if (!dd->cspec->errormask || !(dd->flags & QIB_INITTED))
2584 		return;
2585 
2586 	errormask = qib_read_kreg64(dd, kr_errmask);
2587 
2588 	if (errormask == dd->cspec->errormask)
2589 		return;
2590 	fixed++;
2591 
2592 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
2593 	ctrl = qib_read_kreg32(dd, kr_control);
2594 
2595 	qib_write_kreg(dd, kr_errmask,
2596 		dd->cspec->errormask);
2597 
2598 	if ((hwerrs & dd->cspec->hwerrmask) ||
2599 	    (ctrl & QLOGIC_IB_C_FREEZEMODE)) {
2600 		qib_write_kreg(dd, kr_hwerrclear, 0ULL);
2601 		qib_write_kreg(dd, kr_errclear, 0ULL);
2602 		/* force re-interrupt of pending events, just in case */
2603 		qib_write_kreg(dd, kr_intclear, 0ULL);
2604 		qib_devinfo(dd->pcidev,
2605 			 "errormask fixed(%u) %lx->%lx, ctrl %x hwerr %lx\n",
2606 			 fixed, errormask, (unsigned long)dd->cspec->errormask,
2607 			 ctrl, hwerrs);
2608 	}
2609 }
2610 
2611 /**
2612  * qib_get_6120_faststats - get word counters from chip before they overflow
2613  * @t: contains a pointer to the qlogic_ib device qib_devdata
2614  *
2615  * This needs more work; in particular, decision on whether we really
2616  * need traffic_wds done the way it is
2617  * called from add_timer
2618  */
2619 static void qib_get_6120_faststats(struct timer_list *t)
2620 {
2621 	struct qib_devdata *dd = from_timer(dd, t, stats_timer);
2622 	struct qib_pportdata *ppd = dd->pport;
2623 	unsigned long flags;
2624 	u64 traffic_wds;
2625 
2626 	/*
2627 	 * don't access the chip while running diags, or memory diags can
2628 	 * fail
2629 	 */
2630 	if (!(dd->flags & QIB_INITTED) || dd->diag_client)
2631 		/* but re-arm the timer, for diags case; won't hurt other */
2632 		goto done;
2633 
2634 	/*
2635 	 * We now try to maintain an activity timer, based on traffic
2636 	 * exceeding a threshold, so we need to check the word-counts
2637 	 * even if they are 64-bit.
2638 	 */
2639 	traffic_wds = qib_portcntr_6120(ppd, cr_wordsend) +
2640 		qib_portcntr_6120(ppd, cr_wordrcv);
2641 	spin_lock_irqsave(&dd->eep_st_lock, flags);
2642 	traffic_wds -= dd->traffic_wds;
2643 	dd->traffic_wds += traffic_wds;
2644 	spin_unlock_irqrestore(&dd->eep_st_lock, flags);
2645 
2646 	qib_chk_6120_errormask(dd);
2647 done:
2648 	mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
2649 }
2650 
2651 /* no interrupt fallback for these chips */
2652 static int qib_6120_nointr_fallback(struct qib_devdata *dd)
2653 {
2654 	return 0;
2655 }
2656 
2657 /*
2658  * reset the XGXS (between serdes and IBC).  Slightly less intrusive
2659  * than resetting the IBC or external link state, and useful in some
2660  * cases to cause some retraining.  To do this right, we reset IBC
2661  * as well.
2662  */
2663 static void qib_6120_xgxs_reset(struct qib_pportdata *ppd)
2664 {
2665 	u64 val, prev_val;
2666 	struct qib_devdata *dd = ppd->dd;
2667 
2668 	prev_val = qib_read_kreg64(dd, kr_xgxs_cfg);
2669 	val = prev_val | QLOGIC_IB_XGXS_RESET;
2670 	prev_val &= ~QLOGIC_IB_XGXS_RESET; /* be sure */
2671 	qib_write_kreg(dd, kr_control,
2672 		       dd->control & ~QLOGIC_IB_C_LINKENABLE);
2673 	qib_write_kreg(dd, kr_xgxs_cfg, val);
2674 	qib_read_kreg32(dd, kr_scratch);
2675 	qib_write_kreg(dd, kr_xgxs_cfg, prev_val);
2676 	qib_write_kreg(dd, kr_control, dd->control);
2677 }
2678 
2679 static int qib_6120_get_ib_cfg(struct qib_pportdata *ppd, int which)
2680 {
2681 	int ret;
2682 
2683 	switch (which) {
2684 	case QIB_IB_CFG_LWID:
2685 		ret = ppd->link_width_active;
2686 		break;
2687 
2688 	case QIB_IB_CFG_SPD:
2689 		ret = ppd->link_speed_active;
2690 		break;
2691 
2692 	case QIB_IB_CFG_LWID_ENB:
2693 		ret = ppd->link_width_enabled;
2694 		break;
2695 
2696 	case QIB_IB_CFG_SPD_ENB:
2697 		ret = ppd->link_speed_enabled;
2698 		break;
2699 
2700 	case QIB_IB_CFG_OP_VLS:
2701 		ret = ppd->vls_operational;
2702 		break;
2703 
2704 	case QIB_IB_CFG_VL_HIGH_CAP:
2705 		ret = 0;
2706 		break;
2707 
2708 	case QIB_IB_CFG_VL_LOW_CAP:
2709 		ret = 0;
2710 		break;
2711 
2712 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2713 		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2714 				OverrunThreshold);
2715 		break;
2716 
2717 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2718 		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2719 				PhyerrThreshold);
2720 		break;
2721 
2722 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2723 		/* will only take effect when the link state changes */
2724 		ret = (ppd->dd->cspec->ibcctrl &
2725 		       SYM_MASK(IBCCtrl, LinkDownDefaultState)) ?
2726 			IB_LINKINITCMD_SLEEP : IB_LINKINITCMD_POLL;
2727 		break;
2728 
2729 	case QIB_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2730 		ret = 0; /* no heartbeat on this chip */
2731 		break;
2732 
2733 	case QIB_IB_CFG_PMA_TICKS:
2734 		ret = 250; /* 1 usec. */
2735 		break;
2736 
2737 	default:
2738 		ret =  -EINVAL;
2739 		break;
2740 	}
2741 	return ret;
2742 }
2743 
2744 /*
2745  * We assume range checking is already done, if needed.
2746  */
2747 static int qib_6120_set_ib_cfg(struct qib_pportdata *ppd, int which, u32 val)
2748 {
2749 	struct qib_devdata *dd = ppd->dd;
2750 	int ret = 0;
2751 	u64 val64;
2752 	u16 lcmd, licmd;
2753 
2754 	switch (which) {
2755 	case QIB_IB_CFG_LWID_ENB:
2756 		ppd->link_width_enabled = val;
2757 		break;
2758 
2759 	case QIB_IB_CFG_SPD_ENB:
2760 		ppd->link_speed_enabled = val;
2761 		break;
2762 
2763 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2764 		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2765 				  OverrunThreshold);
2766 		if (val64 != val) {
2767 			dd->cspec->ibcctrl &=
2768 				~SYM_MASK(IBCCtrl, OverrunThreshold);
2769 			dd->cspec->ibcctrl |= (u64) val <<
2770 				SYM_LSB(IBCCtrl, OverrunThreshold);
2771 			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2772 			qib_write_kreg(dd, kr_scratch, 0);
2773 		}
2774 		break;
2775 
2776 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2777 		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2778 				  PhyerrThreshold);
2779 		if (val64 != val) {
2780 			dd->cspec->ibcctrl &=
2781 				~SYM_MASK(IBCCtrl, PhyerrThreshold);
2782 			dd->cspec->ibcctrl |= (u64) val <<
2783 				SYM_LSB(IBCCtrl, PhyerrThreshold);
2784 			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2785 			qib_write_kreg(dd, kr_scratch, 0);
2786 		}
2787 		break;
2788 
2789 	case QIB_IB_CFG_PKEYS: /* update pkeys */
2790 		val64 = (u64) ppd->pkeys[0] | ((u64) ppd->pkeys[1] << 16) |
2791 			((u64) ppd->pkeys[2] << 32) |
2792 			((u64) ppd->pkeys[3] << 48);
2793 		qib_write_kreg(dd, kr_partitionkey, val64);
2794 		break;
2795 
2796 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2797 		/* will only take effect when the link state changes */
2798 		if (val == IB_LINKINITCMD_POLL)
2799 			dd->cspec->ibcctrl &=
2800 				~SYM_MASK(IBCCtrl, LinkDownDefaultState);
2801 		else /* SLEEP */
2802 			dd->cspec->ibcctrl |=
2803 				SYM_MASK(IBCCtrl, LinkDownDefaultState);
2804 		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2805 		qib_write_kreg(dd, kr_scratch, 0);
2806 		break;
2807 
2808 	case QIB_IB_CFG_MTU: /* update the MTU in IBC */
2809 		/*
2810 		 * Update our housekeeping variables, and set IBC max
2811 		 * size, same as init code; max IBC is max we allow in
2812 		 * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2813 		 * Set even if it's unchanged, print debug message only
2814 		 * on changes.
2815 		 */
2816 		val = (ppd->ibmaxlen >> 2) + 1;
2817 		dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, MaxPktLen);
2818 		dd->cspec->ibcctrl |= (u64)val <<
2819 			SYM_LSB(IBCCtrl, MaxPktLen);
2820 		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2821 		qib_write_kreg(dd, kr_scratch, 0);
2822 		break;
2823 
2824 	case QIB_IB_CFG_LSTATE: /* set the IB link state */
2825 		switch (val & 0xffff0000) {
2826 		case IB_LINKCMD_DOWN:
2827 			lcmd = QLOGIC_IB_IBCC_LINKCMD_DOWN;
2828 			if (!dd->cspec->ibdeltainprog) {
2829 				dd->cspec->ibdeltainprog = 1;
2830 				dd->cspec->ibsymsnap =
2831 					read_6120_creg32(dd, cr_ibsymbolerr);
2832 				dd->cspec->iblnkerrsnap =
2833 					read_6120_creg32(dd, cr_iblinkerrrecov);
2834 			}
2835 			break;
2836 
2837 		case IB_LINKCMD_ARMED:
2838 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ARMED;
2839 			break;
2840 
2841 		case IB_LINKCMD_ACTIVE:
2842 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ACTIVE;
2843 			break;
2844 
2845 		default:
2846 			ret = -EINVAL;
2847 			qib_dev_err(dd, "bad linkcmd req 0x%x\n", val >> 16);
2848 			goto bail;
2849 		}
2850 		switch (val & 0xffff) {
2851 		case IB_LINKINITCMD_NOP:
2852 			licmd = 0;
2853 			break;
2854 
2855 		case IB_LINKINITCMD_POLL:
2856 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_POLL;
2857 			break;
2858 
2859 		case IB_LINKINITCMD_SLEEP:
2860 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_SLEEP;
2861 			break;
2862 
2863 		case IB_LINKINITCMD_DISABLE:
2864 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_DISABLE;
2865 			break;
2866 
2867 		default:
2868 			ret = -EINVAL;
2869 			qib_dev_err(dd, "bad linkinitcmd req 0x%x\n",
2870 				    val & 0xffff);
2871 			goto bail;
2872 		}
2873 		qib_set_ib_6120_lstate(ppd, lcmd, licmd);
2874 		goto bail;
2875 
2876 	case QIB_IB_CFG_HRTBT:
2877 		ret = -EINVAL;
2878 		break;
2879 
2880 	default:
2881 		ret = -EINVAL;
2882 	}
2883 bail:
2884 	return ret;
2885 }
2886 
2887 static int qib_6120_set_loopback(struct qib_pportdata *ppd, const char *what)
2888 {
2889 	int ret = 0;
2890 
2891 	if (!strncmp(what, "ibc", 3)) {
2892 		ppd->dd->cspec->ibcctrl |= SYM_MASK(IBCCtrl, Loopback);
2893 		qib_devinfo(ppd->dd->pcidev, "Enabling IB%u:%u IBC loopback\n",
2894 			 ppd->dd->unit, ppd->port);
2895 	} else if (!strncmp(what, "off", 3)) {
2896 		ppd->dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, Loopback);
2897 		qib_devinfo(ppd->dd->pcidev,
2898 			"Disabling IB%u:%u IBC loopback (normal)\n",
2899 			ppd->dd->unit, ppd->port);
2900 	} else
2901 		ret = -EINVAL;
2902 	if (!ret) {
2903 		qib_write_kreg(ppd->dd, kr_ibcctrl, ppd->dd->cspec->ibcctrl);
2904 		qib_write_kreg(ppd->dd, kr_scratch, 0);
2905 	}
2906 	return ret;
2907 }
2908 
2909 static void pma_6120_timer(struct timer_list *t)
2910 {
2911 	struct qib_chip_specific *cs = from_timer(cs, t, pma_timer);
2912 	struct qib_pportdata *ppd = cs->ppd;
2913 	struct qib_ibport *ibp = &ppd->ibport_data;
2914 	unsigned long flags;
2915 
2916 	spin_lock_irqsave(&ibp->rvp.lock, flags);
2917 	if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED) {
2918 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2919 		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2920 				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2921 		mod_timer(&cs->pma_timer,
2922 		      jiffies + usecs_to_jiffies(ibp->rvp.pma_sample_interval));
2923 	} else if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
2924 		u64 ta, tb, tc, td, te;
2925 
2926 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2927 		qib_snapshot_counters(ppd, &ta, &tb, &tc, &td, &te);
2928 
2929 		cs->sword = ta - cs->sword;
2930 		cs->rword = tb - cs->rword;
2931 		cs->spkts = tc - cs->spkts;
2932 		cs->rpkts = td - cs->rpkts;
2933 		cs->xmit_wait = te - cs->xmit_wait;
2934 	}
2935 	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
2936 }
2937 
2938 /*
2939  * Note that the caller has the ibp->rvp.lock held.
2940  */
2941 static void qib_set_cntr_6120_sample(struct qib_pportdata *ppd, u32 intv,
2942 				     u32 start)
2943 {
2944 	struct qib_chip_specific *cs = ppd->dd->cspec;
2945 
2946 	if (start && intv) {
2947 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_STARTED;
2948 		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(start));
2949 	} else if (intv) {
2950 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2951 		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2952 				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2953 		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(intv));
2954 	} else {
2955 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2956 		cs->sword = 0;
2957 		cs->rword = 0;
2958 		cs->spkts = 0;
2959 		cs->rpkts = 0;
2960 		cs->xmit_wait = 0;
2961 	}
2962 }
2963 
2964 static u32 qib_6120_iblink_state(u64 ibcs)
2965 {
2966 	u32 state = (u32)SYM_FIELD(ibcs, IBCStatus, LinkState);
2967 
2968 	switch (state) {
2969 	case IB_6120_L_STATE_INIT:
2970 		state = IB_PORT_INIT;
2971 		break;
2972 	case IB_6120_L_STATE_ARM:
2973 		state = IB_PORT_ARMED;
2974 		break;
2975 	case IB_6120_L_STATE_ACTIVE:
2976 	case IB_6120_L_STATE_ACT_DEFER:
2977 		state = IB_PORT_ACTIVE;
2978 		break;
2979 	default:
2980 		fallthrough;
2981 	case IB_6120_L_STATE_DOWN:
2982 		state = IB_PORT_DOWN;
2983 		break;
2984 	}
2985 	return state;
2986 }
2987 
2988 /* returns the IBTA port state, rather than the IBC link training state */
2989 static u8 qib_6120_phys_portstate(u64 ibcs)
2990 {
2991 	u8 state = (u8)SYM_FIELD(ibcs, IBCStatus, LinkTrainingState);
2992 	return qib_6120_physportstate[state];
2993 }
2994 
2995 static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs)
2996 {
2997 	unsigned long flags;
2998 
2999 	spin_lock_irqsave(&ppd->lflags_lock, flags);
3000 	ppd->lflags &= ~QIBL_IB_FORCE_NOTIFY;
3001 	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3002 
3003 	if (ibup) {
3004 		if (ppd->dd->cspec->ibdeltainprog) {
3005 			ppd->dd->cspec->ibdeltainprog = 0;
3006 			ppd->dd->cspec->ibsymdelta +=
3007 				read_6120_creg32(ppd->dd, cr_ibsymbolerr) -
3008 					ppd->dd->cspec->ibsymsnap;
3009 			ppd->dd->cspec->iblnkerrdelta +=
3010 				read_6120_creg32(ppd->dd, cr_iblinkerrrecov) -
3011 					ppd->dd->cspec->iblnkerrsnap;
3012 		}
3013 		qib_hol_init(ppd);
3014 	} else {
3015 		ppd->dd->cspec->lli_counter = 0;
3016 		if (!ppd->dd->cspec->ibdeltainprog) {
3017 			ppd->dd->cspec->ibdeltainprog = 1;
3018 			ppd->dd->cspec->ibsymsnap =
3019 				read_6120_creg32(ppd->dd, cr_ibsymbolerr);
3020 			ppd->dd->cspec->iblnkerrsnap =
3021 				read_6120_creg32(ppd->dd, cr_iblinkerrrecov);
3022 		}
3023 		qib_hol_down(ppd);
3024 	}
3025 
3026 	qib_6120_setup_setextled(ppd, ibup);
3027 
3028 	return 0;
3029 }
3030 
3031 /* Does read/modify/write to appropriate registers to
3032  * set output and direction bits selected by mask.
3033  * these are in their canonical postions (e.g. lsb of
3034  * dir will end up in D48 of extctrl on existing chips).
3035  * returns contents of GP Inputs.
3036  */
3037 static int gpio_6120_mod(struct qib_devdata *dd, u32 out, u32 dir, u32 mask)
3038 {
3039 	u64 read_val, new_out;
3040 	unsigned long flags;
3041 
3042 	if (mask) {
3043 		/* some bits being written, lock access to GPIO */
3044 		dir &= mask;
3045 		out &= mask;
3046 		spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
3047 		dd->cspec->extctrl &= ~((u64)mask << SYM_LSB(EXTCtrl, GPIOOe));
3048 		dd->cspec->extctrl |= ((u64) dir << SYM_LSB(EXTCtrl, GPIOOe));
3049 		new_out = (dd->cspec->gpio_out & ~mask) | out;
3050 
3051 		qib_write_kreg(dd, kr_extctrl, dd->cspec->extctrl);
3052 		qib_write_kreg(dd, kr_gpio_out, new_out);
3053 		dd->cspec->gpio_out = new_out;
3054 		spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
3055 	}
3056 	/*
3057 	 * It is unlikely that a read at this time would get valid
3058 	 * data on a pin whose direction line was set in the same
3059 	 * call to this function. We include the read here because
3060 	 * that allows us to potentially combine a change on one pin with
3061 	 * a read on another, and because the old code did something like
3062 	 * this.
3063 	 */
3064 	read_val = qib_read_kreg64(dd, kr_extstatus);
3065 	return SYM_FIELD(read_val, EXTStatus, GPIOIn);
3066 }
3067 
3068 /*
3069  * Read fundamental info we need to use the chip.  These are
3070  * the registers that describe chip capabilities, and are
3071  * saved in shadow registers.
3072  */
3073 static void get_6120_chip_params(struct qib_devdata *dd)
3074 {
3075 	u64 val;
3076 	u32 piobufs;
3077 	int mtu;
3078 
3079 	dd->uregbase = qib_read_kreg32(dd, kr_userregbase);
3080 
3081 	dd->rcvtidcnt = qib_read_kreg32(dd, kr_rcvtidcnt);
3082 	dd->rcvtidbase = qib_read_kreg32(dd, kr_rcvtidbase);
3083 	dd->rcvegrbase = qib_read_kreg32(dd, kr_rcvegrbase);
3084 	dd->palign = qib_read_kreg32(dd, kr_palign);
3085 	dd->piobufbase = qib_read_kreg64(dd, kr_sendpiobufbase);
3086 	dd->pio2k_bufbase = dd->piobufbase & 0xffffffff;
3087 
3088 	dd->rcvhdrcnt = qib_read_kreg32(dd, kr_rcvegrcnt);
3089 
3090 	val = qib_read_kreg64(dd, kr_sendpiosize);
3091 	dd->piosize2k = val & ~0U;
3092 	dd->piosize4k = val >> 32;
3093 
3094 	mtu = ib_mtu_enum_to_int(qib_ibmtu);
3095 	if (mtu == -1)
3096 		mtu = QIB_DEFAULT_MTU;
3097 	dd->pport->ibmtu = (u32)mtu;
3098 
3099 	val = qib_read_kreg64(dd, kr_sendpiobufcnt);
3100 	dd->piobcnt2k = val & ~0U;
3101 	dd->piobcnt4k = val >> 32;
3102 	dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
3103 	/* these may be adjusted in init_chip_wc_pat() */
3104 	dd->pio2kbase = (u32 __iomem *)
3105 		(((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
3106 	if (dd->piobcnt4k) {
3107 		dd->pio4kbase = (u32 __iomem *)
3108 			(((char __iomem *) dd->kregbase) +
3109 			 (dd->piobufbase >> 32));
3110 		/*
3111 		 * 4K buffers take 2 pages; we use roundup just to be
3112 		 * paranoid; we calculate it once here, rather than on
3113 		 * ever buf allocate
3114 		 */
3115 		dd->align4k = ALIGN(dd->piosize4k, dd->palign);
3116 	}
3117 
3118 	piobufs = dd->piobcnt4k + dd->piobcnt2k;
3119 
3120 	dd->pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) /
3121 		(sizeof(u64) * BITS_PER_BYTE / 2);
3122 }
3123 
3124 /*
3125  * The chip base addresses in cspec and cpspec have to be set
3126  * after possible init_chip_wc_pat(), rather than in
3127  * get_6120_chip_params(), so split out as separate function
3128  */
3129 static void set_6120_baseaddrs(struct qib_devdata *dd)
3130 {
3131 	u32 cregbase;
3132 
3133 	cregbase = qib_read_kreg32(dd, kr_counterregbase);
3134 	dd->cspec->cregbase = (u64 __iomem *)
3135 		((char __iomem *) dd->kregbase + cregbase);
3136 
3137 	dd->egrtidbase = (u64 __iomem *)
3138 		((char __iomem *) dd->kregbase + dd->rcvegrbase);
3139 }
3140 
3141 /*
3142  * Write the final few registers that depend on some of the
3143  * init setup.  Done late in init, just before bringing up
3144  * the serdes.
3145  */
3146 static int qib_late_6120_initreg(struct qib_devdata *dd)
3147 {
3148 	int ret = 0;
3149 	u64 val;
3150 
3151 	qib_write_kreg(dd, kr_rcvhdrentsize, dd->rcvhdrentsize);
3152 	qib_write_kreg(dd, kr_rcvhdrsize, dd->rcvhdrsize);
3153 	qib_write_kreg(dd, kr_rcvhdrcnt, dd->rcvhdrcnt);
3154 	qib_write_kreg(dd, kr_sendpioavailaddr, dd->pioavailregs_phys);
3155 	val = qib_read_kreg64(dd, kr_sendpioavailaddr);
3156 	if (val != dd->pioavailregs_phys) {
3157 		qib_dev_err(dd,
3158 			"Catastrophic software error, SendPIOAvailAddr written as %lx, read back as %llx\n",
3159 			(unsigned long) dd->pioavailregs_phys,
3160 			(unsigned long long) val);
3161 		ret = -EINVAL;
3162 	}
3163 	return ret;
3164 }
3165 
3166 static int init_6120_variables(struct qib_devdata *dd)
3167 {
3168 	int ret = 0;
3169 	struct qib_pportdata *ppd;
3170 	u32 sbufs;
3171 
3172 	ppd = (struct qib_pportdata *)(dd + 1);
3173 	dd->pport = ppd;
3174 	dd->num_pports = 1;
3175 
3176 	dd->cspec = (struct qib_chip_specific *)(ppd + dd->num_pports);
3177 	dd->cspec->ppd = ppd;
3178 	ppd->cpspec = NULL; /* not used in this chip */
3179 
3180 	spin_lock_init(&dd->cspec->kernel_tid_lock);
3181 	spin_lock_init(&dd->cspec->user_tid_lock);
3182 	spin_lock_init(&dd->cspec->rcvmod_lock);
3183 	spin_lock_init(&dd->cspec->gpio_lock);
3184 
3185 	/* we haven't yet set QIB_PRESENT, so use read directly */
3186 	dd->revision = readq(&dd->kregbase[kr_revision]);
3187 
3188 	if ((dd->revision & 0xffffffffU) == 0xffffffffU) {
3189 		qib_dev_err(dd,
3190 			"Revision register read failure, giving up initialization\n");
3191 		ret = -ENODEV;
3192 		goto bail;
3193 	}
3194 	dd->flags |= QIB_PRESENT;  /* now register routines work */
3195 
3196 	dd->majrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3197 				    ChipRevMajor);
3198 	dd->minrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3199 				    ChipRevMinor);
3200 
3201 	get_6120_chip_params(dd);
3202 	pe_boardname(dd); /* fill in boardname */
3203 
3204 	/*
3205 	 * GPIO bits for TWSI data and clock,
3206 	 * used for serial EEPROM.
3207 	 */
3208 	dd->gpio_sda_num = _QIB_GPIO_SDA_NUM;
3209 	dd->gpio_scl_num = _QIB_GPIO_SCL_NUM;
3210 	dd->twsi_eeprom_dev = QIB_TWSI_NO_DEV;
3211 
3212 	if (qib_unordered_wc())
3213 		dd->flags |= QIB_PIO_FLUSH_WC;
3214 
3215 	ret = qib_init_pportdata(ppd, dd, 0, 1);
3216 	if (ret)
3217 		goto bail;
3218 	ppd->link_width_supported = IB_WIDTH_1X | IB_WIDTH_4X;
3219 	ppd->link_speed_supported = QIB_IB_SDR;
3220 	ppd->link_width_enabled = IB_WIDTH_4X;
3221 	ppd->link_speed_enabled = ppd->link_speed_supported;
3222 	/* these can't change for this chip, so set once */
3223 	ppd->link_width_active = ppd->link_width_enabled;
3224 	ppd->link_speed_active = ppd->link_speed_enabled;
3225 	ppd->vls_supported = IB_VL_VL0;
3226 	ppd->vls_operational = ppd->vls_supported;
3227 
3228 	dd->rcvhdrentsize = QIB_RCVHDR_ENTSIZE;
3229 	dd->rcvhdrsize = QIB_DFLT_RCVHDRSIZE;
3230 	dd->rhf_offset = 0;
3231 
3232 	/* we always allocate at least 2048 bytes for eager buffers */
3233 	ret = ib_mtu_enum_to_int(qib_ibmtu);
3234 	dd->rcvegrbufsize = ret != -1 ? max(ret, 2048) : QIB_DEFAULT_MTU;
3235 	dd->rcvegrbufsize_shift = ilog2(dd->rcvegrbufsize);
3236 
3237 	qib_6120_tidtemplate(dd);
3238 
3239 	/*
3240 	 * We can request a receive interrupt for 1 or
3241 	 * more packets from current offset.  For now, we set this
3242 	 * up for a single packet.
3243 	 */
3244 	dd->rhdrhead_intr_off = 1ULL << 32;
3245 
3246 	/* setup the stats timer; the add_timer is done at end of init */
3247 	timer_setup(&dd->stats_timer, qib_get_6120_faststats, 0);
3248 	timer_setup(&dd->cspec->pma_timer, pma_6120_timer, 0);
3249 
3250 	dd->ureg_align = qib_read_kreg32(dd, kr_palign);
3251 
3252 	dd->piosize2kmax_dwords = dd->piosize2k >> 2;
3253 	qib_6120_config_ctxts(dd);
3254 	qib_set_ctxtcnt(dd);
3255 
3256 	ret = init_chip_wc_pat(dd, 0);
3257 	if (ret)
3258 		goto bail;
3259 	set_6120_baseaddrs(dd); /* set chip access pointers now */
3260 
3261 	ret = 0;
3262 	if (qib_mini_init)
3263 		goto bail;
3264 
3265 	qib_num_cfg_vls = 1; /* if any 6120's, only one VL */
3266 
3267 	ret = qib_create_ctxts(dd);
3268 	init_6120_cntrnames(dd);
3269 
3270 	/* use all of 4KB buffers for the kernel, otherwise 16 */
3271 	sbufs = dd->piobcnt4k ?  dd->piobcnt4k : 16;
3272 
3273 	dd->lastctxt_piobuf = dd->piobcnt2k + dd->piobcnt4k - sbufs;
3274 	dd->pbufsctxt = dd->lastctxt_piobuf /
3275 		(dd->cfgctxts - dd->first_user_ctxt);
3276 
3277 	if (ret)
3278 		goto bail;
3279 bail:
3280 	return ret;
3281 }
3282 
3283 /*
3284  * For this chip, we want to use the same buffer every time
3285  * when we are trying to bring the link up (they are always VL15
3286  * packets).  At that link state the packet should always go out immediately
3287  * (or at least be discarded at the tx interface if the link is down).
3288  * If it doesn't, and the buffer isn't available, that means some other
3289  * sender has gotten ahead of us, and is preventing our packet from going
3290  * out.  In that case, we flush all packets, and try again.  If that still
3291  * fails, we fail the request, and hope things work the next time around.
3292  *
3293  * We don't need very complicated heuristics on whether the packet had
3294  * time to go out or not, since even at SDR 1X, it goes out in very short
3295  * time periods, covered by the chip reads done here and as part of the
3296  * flush.
3297  */
3298 static u32 __iomem *get_6120_link_buf(struct qib_pportdata *ppd, u32 *bnum)
3299 {
3300 	u32 __iomem *buf;
3301 	u32 lbuf = ppd->dd->piobcnt2k + ppd->dd->piobcnt4k - 1;
3302 
3303 	/*
3304 	 * always blip to get avail list updated, since it's almost
3305 	 * always needed, and is fairly cheap.
3306 	 */
3307 	sendctrl_6120_mod(ppd->dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
3308 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3309 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3310 	if (buf)
3311 		goto done;
3312 
3313 	sendctrl_6120_mod(ppd, QIB_SENDCTRL_DISARM_ALL | QIB_SENDCTRL_FLUSH |
3314 			  QIB_SENDCTRL_AVAIL_BLIP);
3315 	ppd->dd->upd_pio_shadow  = 1; /* update our idea of what's busy */
3316 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3317 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3318 done:
3319 	return buf;
3320 }
3321 
3322 static u32 __iomem *qib_6120_getsendbuf(struct qib_pportdata *ppd, u64 pbc,
3323 					u32 *pbufnum)
3324 {
3325 	u32 first, last, plen = pbc & QIB_PBC_LENGTH_MASK;
3326 	struct qib_devdata *dd = ppd->dd;
3327 	u32 __iomem *buf;
3328 
3329 	if (((pbc >> 32) & PBC_6120_VL15_SEND_CTRL) &&
3330 		!(ppd->lflags & (QIBL_IB_AUTONEG_INPROG | QIBL_LINKACTIVE)))
3331 		buf = get_6120_link_buf(ppd, pbufnum);
3332 	else {
3333 
3334 		if ((plen + 1) > dd->piosize2kmax_dwords)
3335 			first = dd->piobcnt2k;
3336 		else
3337 			first = 0;
3338 		/* try 4k if all 2k busy, so same last for both sizes */
3339 		last = dd->piobcnt2k + dd->piobcnt4k - 1;
3340 		buf = qib_getsendbuf_range(dd, pbufnum, first, last);
3341 	}
3342 	return buf;
3343 }
3344 
3345 static int init_sdma_6120_regs(struct qib_pportdata *ppd)
3346 {
3347 	return -ENODEV;
3348 }
3349 
3350 static u16 qib_sdma_6120_gethead(struct qib_pportdata *ppd)
3351 {
3352 	return 0;
3353 }
3354 
3355 static int qib_sdma_6120_busy(struct qib_pportdata *ppd)
3356 {
3357 	return 0;
3358 }
3359 
3360 static void qib_sdma_update_6120_tail(struct qib_pportdata *ppd, u16 tail)
3361 {
3362 }
3363 
3364 static void qib_6120_sdma_sendctrl(struct qib_pportdata *ppd, unsigned op)
3365 {
3366 }
3367 
3368 static void qib_sdma_set_6120_desc_cnt(struct qib_pportdata *ppd, unsigned cnt)
3369 {
3370 }
3371 
3372 /*
3373  * the pbc doesn't need a VL15 indicator, but we need it for link_buf.
3374  * The chip ignores the bit if set.
3375  */
3376 static u32 qib_6120_setpbc_control(struct qib_pportdata *ppd, u32 plen,
3377 				   u8 srate, u8 vl)
3378 {
3379 	return vl == 15 ? PBC_6120_VL15_SEND_CTRL : 0;
3380 }
3381 
3382 static void qib_6120_initvl15_bufs(struct qib_devdata *dd)
3383 {
3384 }
3385 
3386 static void qib_6120_init_ctxt(struct qib_ctxtdata *rcd)
3387 {
3388 	rcd->rcvegrcnt = rcd->dd->rcvhdrcnt;
3389 	rcd->rcvegr_tid_base = rcd->ctxt * rcd->rcvegrcnt;
3390 }
3391 
3392 static void qib_6120_txchk_change(struct qib_devdata *dd, u32 start,
3393 	u32 len, u32 avail, struct qib_ctxtdata *rcd)
3394 {
3395 }
3396 
3397 static void writescratch(struct qib_devdata *dd, u32 val)
3398 {
3399 	(void) qib_write_kreg(dd, kr_scratch, val);
3400 }
3401 
3402 static int qib_6120_tempsense_rd(struct qib_devdata *dd, int regnum)
3403 {
3404 	return -ENXIO;
3405 }
3406 
3407 #ifdef CONFIG_INFINIBAND_QIB_DCA
3408 static int qib_6120_notify_dca(struct qib_devdata *dd, unsigned long event)
3409 {
3410 	return 0;
3411 }
3412 #endif
3413 
3414 /* Dummy function, as 6120 boards never disable EEPROM Write */
3415 static int qib_6120_eeprom_wen(struct qib_devdata *dd, int wen)
3416 {
3417 	return 1;
3418 }
3419 
3420 /**
3421  * qib_init_iba6120_funcs - set up the chip-specific function pointers
3422  * @pdev: pci_dev of the qlogic_ib device
3423  * @ent: pci_device_id matching this chip
3424  *
3425  * This is global, and is called directly at init to set up the
3426  * chip-specific function pointers for later use.
3427  *
3428  * It also allocates/partially-inits the qib_devdata struct for
3429  * this device.
3430  */
3431 struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *pdev,
3432 					   const struct pci_device_id *ent)
3433 {
3434 	struct qib_devdata *dd;
3435 	int ret;
3436 
3437 	dd = qib_alloc_devdata(pdev, sizeof(struct qib_pportdata) +
3438 			       sizeof(struct qib_chip_specific));
3439 	if (IS_ERR(dd))
3440 		goto bail;
3441 
3442 	dd->f_bringup_serdes    = qib_6120_bringup_serdes;
3443 	dd->f_cleanup           = qib_6120_setup_cleanup;
3444 	dd->f_clear_tids        = qib_6120_clear_tids;
3445 	dd->f_free_irq          = qib_free_irq;
3446 	dd->f_get_base_info     = qib_6120_get_base_info;
3447 	dd->f_get_msgheader     = qib_6120_get_msgheader;
3448 	dd->f_getsendbuf        = qib_6120_getsendbuf;
3449 	dd->f_gpio_mod          = gpio_6120_mod;
3450 	dd->f_eeprom_wen	= qib_6120_eeprom_wen;
3451 	dd->f_hdrqempty         = qib_6120_hdrqempty;
3452 	dd->f_ib_updown         = qib_6120_ib_updown;
3453 	dd->f_init_ctxt         = qib_6120_init_ctxt;
3454 	dd->f_initvl15_bufs     = qib_6120_initvl15_bufs;
3455 	dd->f_intr_fallback     = qib_6120_nointr_fallback;
3456 	dd->f_late_initreg      = qib_late_6120_initreg;
3457 	dd->f_setpbc_control    = qib_6120_setpbc_control;
3458 	dd->f_portcntr          = qib_portcntr_6120;
3459 	dd->f_put_tid           = (dd->minrev >= 2) ?
3460 				      qib_6120_put_tid_2 :
3461 				      qib_6120_put_tid;
3462 	dd->f_quiet_serdes      = qib_6120_quiet_serdes;
3463 	dd->f_rcvctrl           = rcvctrl_6120_mod;
3464 	dd->f_read_cntrs        = qib_read_6120cntrs;
3465 	dd->f_read_portcntrs    = qib_read_6120portcntrs;
3466 	dd->f_reset             = qib_6120_setup_reset;
3467 	dd->f_init_sdma_regs    = init_sdma_6120_regs;
3468 	dd->f_sdma_busy         = qib_sdma_6120_busy;
3469 	dd->f_sdma_gethead      = qib_sdma_6120_gethead;
3470 	dd->f_sdma_sendctrl     = qib_6120_sdma_sendctrl;
3471 	dd->f_sdma_set_desc_cnt = qib_sdma_set_6120_desc_cnt;
3472 	dd->f_sdma_update_tail  = qib_sdma_update_6120_tail;
3473 	dd->f_sendctrl          = sendctrl_6120_mod;
3474 	dd->f_set_armlaunch     = qib_set_6120_armlaunch;
3475 	dd->f_set_cntr_sample   = qib_set_cntr_6120_sample;
3476 	dd->f_iblink_state      = qib_6120_iblink_state;
3477 	dd->f_ibphys_portstate  = qib_6120_phys_portstate;
3478 	dd->f_get_ib_cfg        = qib_6120_get_ib_cfg;
3479 	dd->f_set_ib_cfg        = qib_6120_set_ib_cfg;
3480 	dd->f_set_ib_loopback   = qib_6120_set_loopback;
3481 	dd->f_set_intr_state    = qib_6120_set_intr_state;
3482 	dd->f_setextled         = qib_6120_setup_setextled;
3483 	dd->f_txchk_change      = qib_6120_txchk_change;
3484 	dd->f_update_usrhead    = qib_update_6120_usrhead;
3485 	dd->f_wantpiobuf_intr   = qib_wantpiobuf_6120_intr;
3486 	dd->f_xgxs_reset        = qib_6120_xgxs_reset;
3487 	dd->f_writescratch      = writescratch;
3488 	dd->f_tempsense_rd	= qib_6120_tempsense_rd;
3489 #ifdef CONFIG_INFINIBAND_QIB_DCA
3490 	dd->f_notify_dca = qib_6120_notify_dca;
3491 #endif
3492 	/*
3493 	 * Do remaining pcie setup and save pcie values in dd.
3494 	 * Any error printing is already done by the init code.
3495 	 * On return, we have the chip mapped and accessible,
3496 	 * but chip registers are not set up until start of
3497 	 * init_6120_variables.
3498 	 */
3499 	ret = qib_pcie_ddinit(dd, pdev, ent);
3500 	if (ret < 0)
3501 		goto bail_free;
3502 
3503 	/* initialize chip-specific variables */
3504 	ret = init_6120_variables(dd);
3505 	if (ret)
3506 		goto bail_cleanup;
3507 
3508 	if (qib_mini_init)
3509 		goto bail;
3510 
3511 	if (qib_pcie_params(dd, 8, NULL))
3512 		qib_dev_err(dd,
3513 			"Failed to setup PCIe or interrupts; continuing anyway\n");
3514 	/* clear diagctrl register, in case diags were running and crashed */
3515 	qib_write_kreg(dd, kr_hwdiagctrl, 0);
3516 
3517 	if (qib_read_kreg64(dd, kr_hwerrstatus) &
3518 	    QLOGIC_IB_HWE_SERDESPLLFAILED)
3519 		qib_write_kreg(dd, kr_hwerrclear,
3520 			       QLOGIC_IB_HWE_SERDESPLLFAILED);
3521 
3522 	/* setup interrupt handler (interrupt type handled above) */
3523 	qib_setup_6120_interrupt(dd);
3524 	/* Note that qpn_mask is set by qib_6120_config_ctxts() first */
3525 	qib_6120_init_hwerrors(dd);
3526 
3527 	goto bail;
3528 
3529 bail_cleanup:
3530 	qib_pcie_ddcleanup(dd);
3531 bail_free:
3532 	qib_free_devdata(dd);
3533 	dd = ERR_PTR(ret);
3534 bail:
3535 	return dd;
3536 }
3537