1 /*
2  * Copyright (c) 2013 Intel Corporation. All rights reserved.
3  * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
4  * All rights reserved.
5  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 /*
36  * This file contains all of the code that is specific to the
37  * QLogic_IB 6120 PCIe chip.
38  */
39 
40 #include <linux/interrupt.h>
41 #include <linux/pci.h>
42 #include <linux/delay.h>
43 #include <rdma/ib_verbs.h>
44 
45 #include "qib.h"
46 #include "qib_6120_regs.h"
47 
48 static void qib_6120_setup_setextled(struct qib_pportdata *, u32);
49 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op);
50 static u8 qib_6120_phys_portstate(u64);
51 static u32 qib_6120_iblink_state(u64);
52 
53 /*
54  * This file contains all the chip-specific register information and
55  * access functions for the Intel Intel_IB PCI-Express chip.
56  *
57  */
58 
59 /* KREG_IDX uses machine-generated #defines */
60 #define KREG_IDX(regname) (QIB_6120_##regname##_OFFS / sizeof(u64))
61 
62 /* Use defines to tie machine-generated names to lower-case names */
63 #define kr_extctrl KREG_IDX(EXTCtrl)
64 #define kr_extstatus KREG_IDX(EXTStatus)
65 #define kr_gpio_clear KREG_IDX(GPIOClear)
66 #define kr_gpio_mask KREG_IDX(GPIOMask)
67 #define kr_gpio_out KREG_IDX(GPIOOut)
68 #define kr_gpio_status KREG_IDX(GPIOStatus)
69 #define kr_rcvctrl KREG_IDX(RcvCtrl)
70 #define kr_sendctrl KREG_IDX(SendCtrl)
71 #define kr_partitionkey KREG_IDX(RcvPartitionKey)
72 #define kr_hwdiagctrl KREG_IDX(HwDiagCtrl)
73 #define kr_ibcstatus KREG_IDX(IBCStatus)
74 #define kr_ibcctrl KREG_IDX(IBCCtrl)
75 #define kr_sendbuffererror KREG_IDX(SendBufErr0)
76 #define kr_rcvbthqp KREG_IDX(RcvBTHQP)
77 #define kr_counterregbase KREG_IDX(CntrRegBase)
78 #define kr_palign KREG_IDX(PageAlign)
79 #define kr_rcvegrbase KREG_IDX(RcvEgrBase)
80 #define kr_rcvegrcnt KREG_IDX(RcvEgrCnt)
81 #define kr_rcvhdrcnt KREG_IDX(RcvHdrCnt)
82 #define kr_rcvhdrentsize KREG_IDX(RcvHdrEntSize)
83 #define kr_rcvhdrsize KREG_IDX(RcvHdrSize)
84 #define kr_rcvtidbase KREG_IDX(RcvTIDBase)
85 #define kr_rcvtidcnt KREG_IDX(RcvTIDCnt)
86 #define kr_scratch KREG_IDX(Scratch)
87 #define kr_sendctrl KREG_IDX(SendCtrl)
88 #define kr_sendpioavailaddr KREG_IDX(SendPIOAvailAddr)
89 #define kr_sendpiobufbase KREG_IDX(SendPIOBufBase)
90 #define kr_sendpiobufcnt KREG_IDX(SendPIOBufCnt)
91 #define kr_sendpiosize KREG_IDX(SendPIOSize)
92 #define kr_sendregbase KREG_IDX(SendRegBase)
93 #define kr_userregbase KREG_IDX(UserRegBase)
94 #define kr_control KREG_IDX(Control)
95 #define kr_intclear KREG_IDX(IntClear)
96 #define kr_intmask KREG_IDX(IntMask)
97 #define kr_intstatus KREG_IDX(IntStatus)
98 #define kr_errclear KREG_IDX(ErrClear)
99 #define kr_errmask KREG_IDX(ErrMask)
100 #define kr_errstatus KREG_IDX(ErrStatus)
101 #define kr_hwerrclear KREG_IDX(HwErrClear)
102 #define kr_hwerrmask KREG_IDX(HwErrMask)
103 #define kr_hwerrstatus KREG_IDX(HwErrStatus)
104 #define kr_revision KREG_IDX(Revision)
105 #define kr_portcnt KREG_IDX(PortCnt)
106 #define kr_serdes_cfg0 KREG_IDX(SerdesCfg0)
107 #define kr_serdes_cfg1 (kr_serdes_cfg0 + 1)
108 #define kr_serdes_stat KREG_IDX(SerdesStat)
109 #define kr_xgxs_cfg KREG_IDX(XGXSCfg)
110 
111 /* These must only be written via qib_write_kreg_ctxt() */
112 #define kr_rcvhdraddr KREG_IDX(RcvHdrAddr0)
113 #define kr_rcvhdrtailaddr KREG_IDX(RcvHdrTailAddr0)
114 
115 #define CREG_IDX(regname) ((QIB_6120_##regname##_OFFS - \
116 			QIB_6120_LBIntCnt_OFFS) / sizeof(u64))
117 
118 #define cr_badformat CREG_IDX(RxBadFormatCnt)
119 #define cr_erricrc CREG_IDX(RxICRCErrCnt)
120 #define cr_errlink CREG_IDX(RxLinkProblemCnt)
121 #define cr_errlpcrc CREG_IDX(RxLPCRCErrCnt)
122 #define cr_errpkey CREG_IDX(RxPKeyMismatchCnt)
123 #define cr_rcvflowctrl_err CREG_IDX(RxFlowCtrlErrCnt)
124 #define cr_err_rlen CREG_IDX(RxLenErrCnt)
125 #define cr_errslen CREG_IDX(TxLenErrCnt)
126 #define cr_errtidfull CREG_IDX(RxTIDFullErrCnt)
127 #define cr_errtidvalid CREG_IDX(RxTIDValidErrCnt)
128 #define cr_errvcrc CREG_IDX(RxVCRCErrCnt)
129 #define cr_ibstatuschange CREG_IDX(IBStatusChangeCnt)
130 #define cr_lbint CREG_IDX(LBIntCnt)
131 #define cr_invalidrlen CREG_IDX(RxMaxMinLenErrCnt)
132 #define cr_invalidslen CREG_IDX(TxMaxMinLenErrCnt)
133 #define cr_lbflowstall CREG_IDX(LBFlowStallCnt)
134 #define cr_pktrcv CREG_IDX(RxDataPktCnt)
135 #define cr_pktrcvflowctrl CREG_IDX(RxFlowPktCnt)
136 #define cr_pktsend CREG_IDX(TxDataPktCnt)
137 #define cr_pktsendflow CREG_IDX(TxFlowPktCnt)
138 #define cr_portovfl CREG_IDX(RxP0HdrEgrOvflCnt)
139 #define cr_rcvebp CREG_IDX(RxEBPCnt)
140 #define cr_rcvovfl CREG_IDX(RxBufOvflCnt)
141 #define cr_senddropped CREG_IDX(TxDroppedPktCnt)
142 #define cr_sendstall CREG_IDX(TxFlowStallCnt)
143 #define cr_sendunderrun CREG_IDX(TxUnderrunCnt)
144 #define cr_wordrcv CREG_IDX(RxDwordCnt)
145 #define cr_wordsend CREG_IDX(TxDwordCnt)
146 #define cr_txunsupvl CREG_IDX(TxUnsupVLErrCnt)
147 #define cr_rxdroppkt CREG_IDX(RxDroppedPktCnt)
148 #define cr_iblinkerrrecov CREG_IDX(IBLinkErrRecoveryCnt)
149 #define cr_iblinkdown CREG_IDX(IBLinkDownedCnt)
150 #define cr_ibsymbolerr CREG_IDX(IBSymbolErrCnt)
151 
152 #define SYM_RMASK(regname, fldname) ((u64)              \
153 	QIB_6120_##regname##_##fldname##_RMASK)
154 #define SYM_MASK(regname, fldname) ((u64)               \
155 	QIB_6120_##regname##_##fldname##_RMASK <<       \
156 	 QIB_6120_##regname##_##fldname##_LSB)
157 #define SYM_LSB(regname, fldname) (QIB_6120_##regname##_##fldname##_LSB)
158 
159 #define SYM_FIELD(value, regname, fldname) ((u64) \
160 	(((value) >> SYM_LSB(regname, fldname)) & \
161 	 SYM_RMASK(regname, fldname)))
162 #define ERR_MASK(fldname) SYM_MASK(ErrMask, fldname##Mask)
163 #define HWE_MASK(fldname) SYM_MASK(HwErrMask, fldname##Mask)
164 
165 /* link training states, from IBC */
166 #define IB_6120_LT_STATE_DISABLED        0x00
167 #define IB_6120_LT_STATE_LINKUP          0x01
168 #define IB_6120_LT_STATE_POLLACTIVE      0x02
169 #define IB_6120_LT_STATE_POLLQUIET       0x03
170 #define IB_6120_LT_STATE_SLEEPDELAY      0x04
171 #define IB_6120_LT_STATE_SLEEPQUIET      0x05
172 #define IB_6120_LT_STATE_CFGDEBOUNCE     0x08
173 #define IB_6120_LT_STATE_CFGRCVFCFG      0x09
174 #define IB_6120_LT_STATE_CFGWAITRMT      0x0a
175 #define IB_6120_LT_STATE_CFGIDLE 0x0b
176 #define IB_6120_LT_STATE_RECOVERRETRAIN  0x0c
177 #define IB_6120_LT_STATE_RECOVERWAITRMT  0x0e
178 #define IB_6120_LT_STATE_RECOVERIDLE     0x0f
179 
180 /* link state machine states from IBC */
181 #define IB_6120_L_STATE_DOWN             0x0
182 #define IB_6120_L_STATE_INIT             0x1
183 #define IB_6120_L_STATE_ARM              0x2
184 #define IB_6120_L_STATE_ACTIVE           0x3
185 #define IB_6120_L_STATE_ACT_DEFER        0x4
186 
187 static const u8 qib_6120_physportstate[0x20] = {
188 	[IB_6120_LT_STATE_DISABLED] = IB_PHYSPORTSTATE_DISABLED,
189 	[IB_6120_LT_STATE_LINKUP] = IB_PHYSPORTSTATE_LINKUP,
190 	[IB_6120_LT_STATE_POLLACTIVE] = IB_PHYSPORTSTATE_POLL,
191 	[IB_6120_LT_STATE_POLLQUIET] = IB_PHYSPORTSTATE_POLL,
192 	[IB_6120_LT_STATE_SLEEPDELAY] = IB_PHYSPORTSTATE_SLEEP,
193 	[IB_6120_LT_STATE_SLEEPQUIET] = IB_PHYSPORTSTATE_SLEEP,
194 	[IB_6120_LT_STATE_CFGDEBOUNCE] =
195 		IB_PHYSPORTSTATE_CFG_TRAIN,
196 	[IB_6120_LT_STATE_CFGRCVFCFG] =
197 		IB_PHYSPORTSTATE_CFG_TRAIN,
198 	[IB_6120_LT_STATE_CFGWAITRMT] =
199 		IB_PHYSPORTSTATE_CFG_TRAIN,
200 	[IB_6120_LT_STATE_CFGIDLE] = IB_PHYSPORTSTATE_CFG_TRAIN,
201 	[IB_6120_LT_STATE_RECOVERRETRAIN] =
202 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
203 	[IB_6120_LT_STATE_RECOVERWAITRMT] =
204 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
205 	[IB_6120_LT_STATE_RECOVERIDLE] =
206 		IB_PHYSPORTSTATE_LINK_ERR_RECOVER,
207 	[0x10] = IB_PHYSPORTSTATE_CFG_TRAIN,
208 	[0x11] = IB_PHYSPORTSTATE_CFG_TRAIN,
209 	[0x12] = IB_PHYSPORTSTATE_CFG_TRAIN,
210 	[0x13] = IB_PHYSPORTSTATE_CFG_TRAIN,
211 	[0x14] = IB_PHYSPORTSTATE_CFG_TRAIN,
212 	[0x15] = IB_PHYSPORTSTATE_CFG_TRAIN,
213 	[0x16] = IB_PHYSPORTSTATE_CFG_TRAIN,
214 	[0x17] = IB_PHYSPORTSTATE_CFG_TRAIN
215 };
216 
217 
218 struct qib_chip_specific {
219 	u64 __iomem *cregbase;
220 	u64 *cntrs;
221 	u64 *portcntrs;
222 	void *dummy_hdrq;   /* used after ctxt close */
223 	dma_addr_t dummy_hdrq_phys;
224 	spinlock_t kernel_tid_lock; /* no back to back kernel TID writes */
225 	spinlock_t user_tid_lock; /* no back to back user TID writes */
226 	spinlock_t rcvmod_lock; /* protect rcvctrl shadow changes */
227 	spinlock_t gpio_lock; /* RMW of shadows/regs for ExtCtrl and GPIO */
228 	u64 hwerrmask;
229 	u64 errormask;
230 	u64 gpio_out; /* shadow of kr_gpio_out, for rmw ops */
231 	u64 gpio_mask; /* shadow the gpio mask register */
232 	u64 extctrl; /* shadow the gpio output enable, etc... */
233 	/*
234 	 * these 5 fields are used to establish deltas for IB symbol
235 	 * errors and linkrecovery errors.  They can be reported on
236 	 * some chips during link negotiation prior to INIT, and with
237 	 * DDR when faking DDR negotiations with non-IBTA switches.
238 	 * The chip counters are adjusted at driver unload if there is
239 	 * a non-zero delta.
240 	 */
241 	u64 ibdeltainprog;
242 	u64 ibsymdelta;
243 	u64 ibsymsnap;
244 	u64 iblnkerrdelta;
245 	u64 iblnkerrsnap;
246 	u64 ibcctrl; /* shadow for kr_ibcctrl */
247 	u32 lastlinkrecov; /* link recovery issue */
248 	int irq;
249 	u32 cntrnamelen;
250 	u32 portcntrnamelen;
251 	u32 ncntrs;
252 	u32 nportcntrs;
253 	/* used with gpio interrupts to implement IB counters */
254 	u32 rxfc_unsupvl_errs;
255 	u32 overrun_thresh_errs;
256 	/*
257 	 * these count only cases where _successive_ LocalLinkIntegrity
258 	 * errors were seen in the receive headers of IB standard packets
259 	 */
260 	u32 lli_errs;
261 	u32 lli_counter;
262 	u64 lli_thresh;
263 	u64 sword; /* total dwords sent (sample result) */
264 	u64 rword; /* total dwords received (sample result) */
265 	u64 spkts; /* total packets sent (sample result) */
266 	u64 rpkts; /* total packets received (sample result) */
267 	u64 xmit_wait; /* # of ticks no data sent (sample result) */
268 	struct timer_list pma_timer;
269 	char emsgbuf[128];
270 	char bitsmsgbuf[64];
271 	u8 pma_sample_status;
272 };
273 
274 /* ibcctrl bits */
275 #define QLOGIC_IB_IBCC_LINKINITCMD_DISABLE 1
276 /* cycle through TS1/TS2 till OK */
277 #define QLOGIC_IB_IBCC_LINKINITCMD_POLL 2
278 /* wait for TS1, then go on */
279 #define QLOGIC_IB_IBCC_LINKINITCMD_SLEEP 3
280 #define QLOGIC_IB_IBCC_LINKINITCMD_SHIFT 16
281 
282 #define QLOGIC_IB_IBCC_LINKCMD_DOWN 1           /* move to 0x11 */
283 #define QLOGIC_IB_IBCC_LINKCMD_ARMED 2          /* move to 0x21 */
284 #define QLOGIC_IB_IBCC_LINKCMD_ACTIVE 3 /* move to 0x31 */
285 #define QLOGIC_IB_IBCC_LINKCMD_SHIFT 18
286 
287 /*
288  * We could have a single register get/put routine, that takes a group type,
289  * but this is somewhat clearer and cleaner.  It also gives us some error
290  * checking.  64 bit register reads should always work, but are inefficient
291  * on opteron (the northbridge always generates 2 separate HT 32 bit reads),
292  * so we use kreg32 wherever possible.  User register and counter register
293  * reads are always 32 bit reads, so only one form of those routines.
294  */
295 
296 /**
297  * qib_read_ureg32 - read 32-bit virtualized per-context register
298  * @dd: device
299  * @regno: register number
300  * @ctxt: context number
301  *
302  * Return the contents of a register that is virtualized to be per context.
303  * Returns -1 on errors (not distinguishable from valid contents at
304  * runtime; we may add a separate error variable at some point).
305  */
306 static inline u32 qib_read_ureg32(const struct qib_devdata *dd,
307 				  enum qib_ureg regno, int ctxt)
308 {
309 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
310 		return 0;
311 
312 	if (dd->userbase)
313 		return readl(regno + (u64 __iomem *)
314 			     ((char __iomem *)dd->userbase +
315 			      dd->ureg_align * ctxt));
316 	else
317 		return readl(regno + (u64 __iomem *)
318 			     (dd->uregbase +
319 			      (char __iomem *)dd->kregbase +
320 			      dd->ureg_align * ctxt));
321 }
322 
323 /**
324  * qib_write_ureg - write 32-bit virtualized per-context register
325  * @dd: device
326  * @regno: register number
327  * @value: value
328  * @ctxt: context
329  *
330  * Write the contents of a register that is virtualized to be per context.
331  */
332 static inline void qib_write_ureg(const struct qib_devdata *dd,
333 				  enum qib_ureg regno, u64 value, int ctxt)
334 {
335 	u64 __iomem *ubase;
336 
337 	if (dd->userbase)
338 		ubase = (u64 __iomem *)
339 			((char __iomem *) dd->userbase +
340 			 dd->ureg_align * ctxt);
341 	else
342 		ubase = (u64 __iomem *)
343 			(dd->uregbase +
344 			 (char __iomem *) dd->kregbase +
345 			 dd->ureg_align * ctxt);
346 
347 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
348 		writeq(value, &ubase[regno]);
349 }
350 
351 static inline u32 qib_read_kreg32(const struct qib_devdata *dd,
352 				  const u16 regno)
353 {
354 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
355 		return -1;
356 	return readl((u32 __iomem *)&dd->kregbase[regno]);
357 }
358 
359 static inline u64 qib_read_kreg64(const struct qib_devdata *dd,
360 				  const u16 regno)
361 {
362 	if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
363 		return -1;
364 
365 	return readq(&dd->kregbase[regno]);
366 }
367 
368 static inline void qib_write_kreg(const struct qib_devdata *dd,
369 				  const u16 regno, u64 value)
370 {
371 	if (dd->kregbase && (dd->flags & QIB_PRESENT))
372 		writeq(value, &dd->kregbase[regno]);
373 }
374 
375 /**
376  * qib_write_kreg_ctxt - write a device's per-ctxt 64-bit kernel register
377  * @dd: the qlogic_ib device
378  * @regno: the register number to write
379  * @ctxt: the context containing the register
380  * @value: the value to write
381  */
382 static inline void qib_write_kreg_ctxt(const struct qib_devdata *dd,
383 				       const u16 regno, unsigned ctxt,
384 				       u64 value)
385 {
386 	qib_write_kreg(dd, regno + ctxt, value);
387 }
388 
389 static inline void write_6120_creg(const struct qib_devdata *dd,
390 				   u16 regno, u64 value)
391 {
392 	if (dd->cspec->cregbase && (dd->flags & QIB_PRESENT))
393 		writeq(value, &dd->cspec->cregbase[regno]);
394 }
395 
396 static inline u64 read_6120_creg(const struct qib_devdata *dd, u16 regno)
397 {
398 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
399 		return 0;
400 	return readq(&dd->cspec->cregbase[regno]);
401 }
402 
403 static inline u32 read_6120_creg32(const struct qib_devdata *dd, u16 regno)
404 {
405 	if (!dd->cspec->cregbase || !(dd->flags & QIB_PRESENT))
406 		return 0;
407 	return readl(&dd->cspec->cregbase[regno]);
408 }
409 
410 /* kr_control bits */
411 #define QLOGIC_IB_C_RESET 1U
412 
413 /* kr_intstatus, kr_intclear, kr_intmask bits */
414 #define QLOGIC_IB_I_RCVURG_MASK ((1U << 5) - 1)
415 #define QLOGIC_IB_I_RCVURG_SHIFT 0
416 #define QLOGIC_IB_I_RCVAVAIL_MASK ((1U << 5) - 1)
417 #define QLOGIC_IB_I_RCVAVAIL_SHIFT 12
418 
419 #define QLOGIC_IB_C_FREEZEMODE 0x00000002
420 #define QLOGIC_IB_C_LINKENABLE 0x00000004
421 #define QLOGIC_IB_I_ERROR               0x0000000080000000ULL
422 #define QLOGIC_IB_I_SPIOSENT            0x0000000040000000ULL
423 #define QLOGIC_IB_I_SPIOBUFAVAIL        0x0000000020000000ULL
424 #define QLOGIC_IB_I_GPIO                0x0000000010000000ULL
425 #define QLOGIC_IB_I_BITSEXTANT \
426 		((QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT) | \
427 		(QLOGIC_IB_I_RCVAVAIL_MASK << \
428 		 QLOGIC_IB_I_RCVAVAIL_SHIFT) | \
429 		QLOGIC_IB_I_ERROR | QLOGIC_IB_I_SPIOSENT | \
430 		QLOGIC_IB_I_SPIOBUFAVAIL | QLOGIC_IB_I_GPIO)
431 
432 /* kr_hwerrclear, kr_hwerrmask, kr_hwerrstatus, bits */
433 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK  0x000000000000003fULL
434 #define QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT 0
435 #define QLOGIC_IB_HWE_PCIEPOISONEDTLP      0x0000000010000000ULL
436 #define QLOGIC_IB_HWE_PCIECPLTIMEOUT       0x0000000020000000ULL
437 #define QLOGIC_IB_HWE_PCIEBUSPARITYXTLH    0x0000000040000000ULL
438 #define QLOGIC_IB_HWE_PCIEBUSPARITYXADM    0x0000000080000000ULL
439 #define QLOGIC_IB_HWE_PCIEBUSPARITYRADM    0x0000000100000000ULL
440 #define QLOGIC_IB_HWE_COREPLL_FBSLIP       0x0080000000000000ULL
441 #define QLOGIC_IB_HWE_COREPLL_RFSLIP       0x0100000000000000ULL
442 #define QLOGIC_IB_HWE_PCIE1PLLFAILED       0x0400000000000000ULL
443 #define QLOGIC_IB_HWE_PCIE0PLLFAILED       0x0800000000000000ULL
444 #define QLOGIC_IB_HWE_SERDESPLLFAILED      0x1000000000000000ULL
445 
446 
447 /* kr_extstatus bits */
448 #define QLOGIC_IB_EXTS_FREQSEL 0x2
449 #define QLOGIC_IB_EXTS_SERDESSEL 0x4
450 #define QLOGIC_IB_EXTS_MEMBIST_ENDTEST     0x0000000000004000
451 #define QLOGIC_IB_EXTS_MEMBIST_FOUND       0x0000000000008000
452 
453 /* kr_xgxsconfig bits */
454 #define QLOGIC_IB_XGXS_RESET          0x5ULL
455 
456 #define _QIB_GPIO_SDA_NUM 1
457 #define _QIB_GPIO_SCL_NUM 0
458 
459 /* Bits in GPIO for the added IB link interrupts */
460 #define GPIO_RXUVL_BIT 3
461 #define GPIO_OVRUN_BIT 4
462 #define GPIO_LLI_BIT 5
463 #define GPIO_ERRINTR_MASK 0x38
464 
465 
466 #define QLOGIC_IB_RT_BUFSIZE_MASK 0xe0000000ULL
467 #define QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid) \
468 	((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) >> 29) + 11 - 1)
469 #define QLOGIC_IB_RT_BUFSIZE(tid) (1 << QLOGIC_IB_RT_BUFSIZE_SHIFTVAL(tid))
470 #define QLOGIC_IB_RT_IS_VALID(tid) \
471 	(((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) && \
472 	 ((((tid) & QLOGIC_IB_RT_BUFSIZE_MASK) != QLOGIC_IB_RT_BUFSIZE_MASK)))
473 #define QLOGIC_IB_RT_ADDR_MASK 0x1FFFFFFFULL /* 29 bits valid */
474 #define QLOGIC_IB_RT_ADDR_SHIFT 10
475 
476 #define QLOGIC_IB_R_INTRAVAIL_SHIFT 16
477 #define QLOGIC_IB_R_TAILUPD_SHIFT 31
478 #define IBA6120_R_PKEY_DIS_SHIFT 30
479 
480 #define PBC_6120_VL15_SEND_CTRL (1ULL << 31) /* pbc; VL15; link_buf only */
481 
482 #define IBCBUSFRSPCPARITYERR HWE_MASK(IBCBusFromSPCParityErr)
483 #define IBCBUSTOSPCPARITYERR HWE_MASK(IBCBusToSPCParityErr)
484 
485 #define SYM_MASK_BIT(regname, fldname, bit) ((u64) \
486 	((1ULL << (SYM_LSB(regname, fldname) + (bit)))))
487 
488 #define TXEMEMPARITYERR_PIOBUF \
489 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 0)
490 #define TXEMEMPARITYERR_PIOPBC \
491 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 1)
492 #define TXEMEMPARITYERR_PIOLAUNCHFIFO \
493 	SYM_MASK_BIT(HwErrMask, TXEMemParityErrMask, 2)
494 
495 #define RXEMEMPARITYERR_RCVBUF \
496 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 0)
497 #define RXEMEMPARITYERR_LOOKUPQ \
498 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 1)
499 #define RXEMEMPARITYERR_EXPTID \
500 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 2)
501 #define RXEMEMPARITYERR_EAGERTID \
502 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 3)
503 #define RXEMEMPARITYERR_FLAGBUF \
504 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 4)
505 #define RXEMEMPARITYERR_DATAINFO \
506 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 5)
507 #define RXEMEMPARITYERR_HDRINFO \
508 	SYM_MASK_BIT(HwErrMask, RXEMemParityErrMask, 6)
509 
510 /* 6120 specific hardware errors... */
511 static const struct qib_hwerror_msgs qib_6120_hwerror_msgs[] = {
512 	/* generic hardware errors */
513 	QLOGIC_IB_HWE_MSG(IBCBUSFRSPCPARITYERR, "QIB2IB Parity"),
514 	QLOGIC_IB_HWE_MSG(IBCBUSTOSPCPARITYERR, "IB2QIB Parity"),
515 
516 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOBUF,
517 			  "TXE PIOBUF Memory Parity"),
518 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOPBC,
519 			  "TXE PIOPBC Memory Parity"),
520 	QLOGIC_IB_HWE_MSG(TXEMEMPARITYERR_PIOLAUNCHFIFO,
521 			  "TXE PIOLAUNCHFIFO Memory Parity"),
522 
523 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_RCVBUF,
524 			  "RXE RCVBUF Memory Parity"),
525 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_LOOKUPQ,
526 			  "RXE LOOKUPQ Memory Parity"),
527 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EAGERTID,
528 			  "RXE EAGERTID Memory Parity"),
529 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_EXPTID,
530 			  "RXE EXPTID Memory Parity"),
531 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_FLAGBUF,
532 			  "RXE FLAGBUF Memory Parity"),
533 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_DATAINFO,
534 			  "RXE DATAINFO Memory Parity"),
535 	QLOGIC_IB_HWE_MSG(RXEMEMPARITYERR_HDRINFO,
536 			  "RXE HDRINFO Memory Parity"),
537 
538 	/* chip-specific hardware errors */
539 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEPOISONEDTLP,
540 			  "PCIe Poisoned TLP"),
541 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIECPLTIMEOUT,
542 			  "PCIe completion timeout"),
543 	/*
544 	 * In practice, it's unlikely wthat we'll see PCIe PLL, or bus
545 	 * parity or memory parity error failures, because most likely we
546 	 * won't be able to talk to the core of the chip.  Nonetheless, we
547 	 * might see them, if they are in parts of the PCIe core that aren't
548 	 * essential.
549 	 */
550 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE1PLLFAILED,
551 			  "PCIePLL1"),
552 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIE0PLLFAILED,
553 			  "PCIePLL0"),
554 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXTLH,
555 			  "PCIe XTLH core parity"),
556 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYXADM,
557 			  "PCIe ADM TX core parity"),
558 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_PCIEBUSPARITYRADM,
559 			  "PCIe ADM RX core parity"),
560 	QLOGIC_IB_HWE_MSG(QLOGIC_IB_HWE_SERDESPLLFAILED,
561 			  "SerDes PLL"),
562 };
563 
564 #define TXE_PIO_PARITY (TXEMEMPARITYERR_PIOBUF | TXEMEMPARITYERR_PIOPBC)
565 #define _QIB_PLL_FAIL (QLOGIC_IB_HWE_COREPLL_FBSLIP |   \
566 		QLOGIC_IB_HWE_COREPLL_RFSLIP)
567 
568 	/* variables for sanity checking interrupt and errors */
569 #define IB_HWE_BITSEXTANT \
570 	(HWE_MASK(RXEMemParityErr) |					\
571 	 HWE_MASK(TXEMemParityErr) |					\
572 	 (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<			\
573 	  QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) |			\
574 	 QLOGIC_IB_HWE_PCIE1PLLFAILED |					\
575 	 QLOGIC_IB_HWE_PCIE0PLLFAILED |					\
576 	 QLOGIC_IB_HWE_PCIEPOISONEDTLP |				\
577 	 QLOGIC_IB_HWE_PCIECPLTIMEOUT |					\
578 	 QLOGIC_IB_HWE_PCIEBUSPARITYXTLH |				\
579 	 QLOGIC_IB_HWE_PCIEBUSPARITYXADM |				\
580 	 QLOGIC_IB_HWE_PCIEBUSPARITYRADM |				\
581 	 HWE_MASK(PowerOnBISTFailed) |					\
582 	 QLOGIC_IB_HWE_COREPLL_FBSLIP |					\
583 	 QLOGIC_IB_HWE_COREPLL_RFSLIP |					\
584 	 QLOGIC_IB_HWE_SERDESPLLFAILED |				\
585 	 HWE_MASK(IBCBusToSPCParityErr) |				\
586 	 HWE_MASK(IBCBusFromSPCParityErr))
587 
588 #define IB_E_BITSEXTANT \
589 	(ERR_MASK(RcvFormatErr) | ERR_MASK(RcvVCRCErr) |		\
590 	 ERR_MASK(RcvICRCErr) | ERR_MASK(RcvMinPktLenErr) |		\
591 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvLongPktLenErr) |	\
592 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvUnexpectedCharErr) | \
593 	 ERR_MASK(RcvUnsupportedVLErr) | ERR_MASK(RcvEBPErr) |		\
594 	 ERR_MASK(RcvIBFlowErr) | ERR_MASK(RcvBadVersionErr) |		\
595 	 ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr) |		\
596 	 ERR_MASK(RcvBadTidErr) | ERR_MASK(RcvHdrLenErr) |		\
597 	 ERR_MASK(RcvHdrErr) | ERR_MASK(RcvIBLostLinkErr) |		\
598 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendMaxPktLenErr) |	\
599 	 ERR_MASK(SendUnderRunErr) | ERR_MASK(SendPktLenErr) |		\
600 	 ERR_MASK(SendDroppedSmpPktErr) |				\
601 	 ERR_MASK(SendDroppedDataPktErr) |				\
602 	 ERR_MASK(SendPioArmLaunchErr) |				\
603 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
604 	 ERR_MASK(SendUnsupportedVLErr) | ERR_MASK(IBStatusChanged) |	\
605 	 ERR_MASK(InvalidAddrErr) | ERR_MASK(ResetNegated) |		\
606 	 ERR_MASK(HardwareErr))
607 
608 #define QLOGIC_IB_E_PKTERRS ( \
609 		ERR_MASK(SendPktLenErr) |				\
610 		ERR_MASK(SendDroppedDataPktErr) |			\
611 		ERR_MASK(RcvVCRCErr) |					\
612 		ERR_MASK(RcvICRCErr) |					\
613 		ERR_MASK(RcvShortPktLenErr) |				\
614 		ERR_MASK(RcvEBPErr))
615 
616 /* These are all rcv-related errors which we want to count for stats */
617 #define E_SUM_PKTERRS						\
618 	(ERR_MASK(RcvHdrLenErr) | ERR_MASK(RcvBadTidErr) |		\
619 	 ERR_MASK(RcvBadVersionErr) | ERR_MASK(RcvHdrErr) |		\
620 	 ERR_MASK(RcvLongPktLenErr) | ERR_MASK(RcvShortPktLenErr) |	\
621 	 ERR_MASK(RcvMaxPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
622 	 ERR_MASK(RcvFormatErr) | ERR_MASK(RcvUnsupportedVLErr) |	\
623 	 ERR_MASK(RcvUnexpectedCharErr) | ERR_MASK(RcvEBPErr))
624 
625 /* These are all send-related errors which we want to count for stats */
626 #define E_SUM_ERRS							\
627 	(ERR_MASK(SendPioArmLaunchErr) |				\
628 	 ERR_MASK(SendUnexpectedPktNumErr) |				\
629 	 ERR_MASK(SendDroppedDataPktErr) |				\
630 	 ERR_MASK(SendDroppedSmpPktErr) |				\
631 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendUnsupportedVLErr) |	\
632 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
633 	 ERR_MASK(InvalidAddrErr))
634 
635 /*
636  * this is similar to E_SUM_ERRS, but can't ignore armlaunch, don't ignore
637  * errors not related to freeze and cancelling buffers.  Can't ignore
638  * armlaunch because could get more while still cleaning up, and need
639  * to cancel those as they happen.
640  */
641 #define E_SPKT_ERRS_IGNORE \
642 	(ERR_MASK(SendDroppedDataPktErr) |				\
643 	 ERR_MASK(SendDroppedSmpPktErr) |				\
644 	 ERR_MASK(SendMaxPktLenErr) | ERR_MASK(SendMinPktLenErr) |	\
645 	 ERR_MASK(SendPktLenErr))
646 
647 /*
648  * these are errors that can occur when the link changes state while
649  * a packet is being sent or received.  This doesn't cover things
650  * like EBP or VCRC that can be the result of a sending having the
651  * link change state, so we receive a "known bad" packet.
652  */
653 #define E_SUM_LINK_PKTERRS		\
654 	(ERR_MASK(SendDroppedDataPktErr) |				\
655 	 ERR_MASK(SendDroppedSmpPktErr) |				\
656 	 ERR_MASK(SendMinPktLenErr) | ERR_MASK(SendPktLenErr) |		\
657 	 ERR_MASK(RcvShortPktLenErr) | ERR_MASK(RcvMinPktLenErr) |	\
658 	 ERR_MASK(RcvUnexpectedCharErr))
659 
660 static void qib_6120_put_tid_2(struct qib_devdata *, u64 __iomem *,
661 			       u32, unsigned long);
662 
663 /*
664  * On platforms using this chip, and not having ordered WC stores, we
665  * can get TXE parity errors due to speculative reads to the PIO buffers,
666  * and this, due to a chip issue can result in (many) false parity error
667  * reports.  So it's a debug print on those, and an info print on systems
668  * where the speculative reads don't occur.
669  */
670 static void qib_6120_txe_recover(struct qib_devdata *dd)
671 {
672 	if (!qib_unordered_wc())
673 		qib_devinfo(dd->pcidev,
674 			    "Recovering from TXE PIO parity error\n");
675 }
676 
677 /* enable/disable chip from delivering interrupts */
678 static void qib_6120_set_intr_state(struct qib_devdata *dd, u32 enable)
679 {
680 	if (enable) {
681 		if (dd->flags & QIB_BADINTR)
682 			return;
683 		qib_write_kreg(dd, kr_intmask, ~0ULL);
684 		/* force re-interrupt of any pending interrupts. */
685 		qib_write_kreg(dd, kr_intclear, 0ULL);
686 	} else
687 		qib_write_kreg(dd, kr_intmask, 0ULL);
688 }
689 
690 /*
691  * Try to cleanup as much as possible for anything that might have gone
692  * wrong while in freeze mode, such as pio buffers being written by user
693  * processes (causing armlaunch), send errors due to going into freeze mode,
694  * etc., and try to avoid causing extra interrupts while doing so.
695  * Forcibly update the in-memory pioavail register copies after cleanup
696  * because the chip won't do it while in freeze mode (the register values
697  * themselves are kept correct).
698  * Make sure that we don't lose any important interrupts by using the chip
699  * feature that says that writing 0 to a bit in *clear that is set in
700  * *status will cause an interrupt to be generated again (if allowed by
701  * the *mask value).
702  * This is in chip-specific code because of all of the register accesses,
703  * even though the details are similar on most chips
704  */
705 static void qib_6120_clear_freeze(struct qib_devdata *dd)
706 {
707 	/* disable error interrupts, to avoid confusion */
708 	qib_write_kreg(dd, kr_errmask, 0ULL);
709 
710 	/* also disable interrupts; errormask is sometimes overwriten */
711 	qib_6120_set_intr_state(dd, 0);
712 
713 	qib_cancel_sends(dd->pport);
714 
715 	/* clear the freeze, and be sure chip saw it */
716 	qib_write_kreg(dd, kr_control, dd->control);
717 	qib_read_kreg32(dd, kr_scratch);
718 
719 	/* force in-memory update now we are out of freeze */
720 	qib_force_pio_avail_update(dd);
721 
722 	/*
723 	 * force new interrupt if any hwerr, error or interrupt bits are
724 	 * still set, and clear "safe" send packet errors related to freeze
725 	 * and cancelling sends.  Re-enable error interrupts before possible
726 	 * force of re-interrupt on pending interrupts.
727 	 */
728 	qib_write_kreg(dd, kr_hwerrclear, 0ULL);
729 	qib_write_kreg(dd, kr_errclear, E_SPKT_ERRS_IGNORE);
730 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
731 	qib_6120_set_intr_state(dd, 1);
732 }
733 
734 /**
735  * qib_handle_6120_hwerrors - display hardware errors.
736  * @dd: the qlogic_ib device
737  * @msg: the output buffer
738  * @msgl: the size of the output buffer
739  *
740  * Use same msg buffer as regular errors to avoid excessive stack
741  * use.  Most hardware errors are catastrophic, but for right now,
742  * we'll print them and continue.  Reuse the same message buffer as
743  * handle_6120_errors() to avoid excessive stack usage.
744  */
745 static void qib_handle_6120_hwerrors(struct qib_devdata *dd, char *msg,
746 				     size_t msgl)
747 {
748 	u64 hwerrs;
749 	u32 bits, ctrl;
750 	int isfatal = 0;
751 	char *bitsmsg;
752 	int log_idx;
753 
754 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
755 	if (!hwerrs)
756 		return;
757 	if (hwerrs == ~0ULL) {
758 		qib_dev_err(dd,
759 			"Read of hardware error status failed (all bits set); ignoring\n");
760 		return;
761 	}
762 	qib_stats.sps_hwerrs++;
763 
764 	/* Always clear the error status register, except MEMBISTFAIL,
765 	 * regardless of whether we continue or stop using the chip.
766 	 * We want that set so we know it failed, even across driver reload.
767 	 * We'll still ignore it in the hwerrmask.  We do this partly for
768 	 * diagnostics, but also for support */
769 	qib_write_kreg(dd, kr_hwerrclear,
770 		       hwerrs & ~HWE_MASK(PowerOnBISTFailed));
771 
772 	hwerrs &= dd->cspec->hwerrmask;
773 
774 	/* We log some errors to EEPROM, check if we have any of those. */
775 	for (log_idx = 0; log_idx < QIB_EEP_LOG_CNT; ++log_idx)
776 		if (hwerrs & dd->eep_st_masks[log_idx].hwerrs_to_log)
777 			qib_inc_eeprom_err(dd, log_idx, 1);
778 
779 	/*
780 	 * Make sure we get this much out, unless told to be quiet,
781 	 * or it's occurred within the last 5 seconds.
782 	 */
783 	if (hwerrs & ~(TXE_PIO_PARITY | RXEMEMPARITYERR_EAGERTID))
784 		qib_devinfo(dd->pcidev,
785 			"Hardware error: hwerr=0x%llx (cleared)\n",
786 			(unsigned long long) hwerrs);
787 
788 	if (hwerrs & ~IB_HWE_BITSEXTANT)
789 		qib_dev_err(dd,
790 			"hwerror interrupt with unknown errors %llx set\n",
791 			(unsigned long long)(hwerrs & ~IB_HWE_BITSEXTANT));
792 
793 	ctrl = qib_read_kreg32(dd, kr_control);
794 	if ((ctrl & QLOGIC_IB_C_FREEZEMODE) && !dd->diag_client) {
795 		/*
796 		 * Parity errors in send memory are recoverable,
797 		 * just cancel the send (if indicated in * sendbuffererror),
798 		 * count the occurrence, unfreeze (if no other handled
799 		 * hardware error bits are set), and continue. They can
800 		 * occur if a processor speculative read is done to the PIO
801 		 * buffer while we are sending a packet, for example.
802 		 */
803 		if (hwerrs & TXE_PIO_PARITY) {
804 			qib_6120_txe_recover(dd);
805 			hwerrs &= ~TXE_PIO_PARITY;
806 		}
807 
808 		if (!hwerrs) {
809 			static u32 freeze_cnt;
810 
811 			freeze_cnt++;
812 			qib_6120_clear_freeze(dd);
813 		} else
814 			isfatal = 1;
815 	}
816 
817 	*msg = '\0';
818 
819 	if (hwerrs & HWE_MASK(PowerOnBISTFailed)) {
820 		isfatal = 1;
821 		strlcat(msg,
822 			"[Memory BIST test failed, InfiniPath hardware unusable]",
823 			msgl);
824 		/* ignore from now on, so disable until driver reloaded */
825 		dd->cspec->hwerrmask &= ~HWE_MASK(PowerOnBISTFailed);
826 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
827 	}
828 
829 	qib_format_hwerrors(hwerrs, qib_6120_hwerror_msgs,
830 			    ARRAY_SIZE(qib_6120_hwerror_msgs), msg, msgl);
831 
832 	bitsmsg = dd->cspec->bitsmsgbuf;
833 	if (hwerrs & (QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK <<
834 		      QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT)) {
835 		bits = (u32) ((hwerrs >>
836 			       QLOGIC_IB_HWE_PCIEMEMPARITYERR_SHIFT) &
837 			      QLOGIC_IB_HWE_PCIEMEMPARITYERR_MASK);
838 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
839 			 "[PCIe Mem Parity Errs %x] ", bits);
840 		strlcat(msg, bitsmsg, msgl);
841 	}
842 
843 	if (hwerrs & _QIB_PLL_FAIL) {
844 		isfatal = 1;
845 		snprintf(bitsmsg, sizeof(dd->cspec->bitsmsgbuf),
846 			 "[PLL failed (%llx), InfiniPath hardware unusable]",
847 			 (unsigned long long) hwerrs & _QIB_PLL_FAIL);
848 		strlcat(msg, bitsmsg, msgl);
849 		/* ignore from now on, so disable until driver reloaded */
850 		dd->cspec->hwerrmask &= ~(hwerrs & _QIB_PLL_FAIL);
851 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
852 	}
853 
854 	if (hwerrs & QLOGIC_IB_HWE_SERDESPLLFAILED) {
855 		/*
856 		 * If it occurs, it is left masked since the external
857 		 * interface is unused
858 		 */
859 		dd->cspec->hwerrmask &= ~QLOGIC_IB_HWE_SERDESPLLFAILED;
860 		qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
861 	}
862 
863 	if (hwerrs)
864 		/*
865 		 * if any set that we aren't ignoring; only
866 		 * make the complaint once, in case it's stuck
867 		 * or recurring, and we get here multiple
868 		 * times.
869 		 */
870 		qib_dev_err(dd, "%s hardware error\n", msg);
871 	else
872 		*msg = 0; /* recovered from all of them */
873 
874 	if (isfatal && !dd->diag_client) {
875 		qib_dev_err(dd,
876 			"Fatal Hardware Error, no longer usable, SN %.16s\n",
877 			dd->serial);
878 		/*
879 		 * for /sys status file and user programs to print; if no
880 		 * trailing brace is copied, we'll know it was truncated.
881 		 */
882 		if (dd->freezemsg)
883 			snprintf(dd->freezemsg, dd->freezelen,
884 				 "{%s}", msg);
885 		qib_disable_after_error(dd);
886 	}
887 }
888 
889 /*
890  * Decode the error status into strings, deciding whether to always
891  * print * it or not depending on "normal packet errors" vs everything
892  * else.   Return 1 if "real" errors, otherwise 0 if only packet
893  * errors, so caller can decide what to print with the string.
894  */
895 static int qib_decode_6120_err(struct qib_devdata *dd, char *buf, size_t blen,
896 			       u64 err)
897 {
898 	int iserr = 1;
899 
900 	*buf = '\0';
901 	if (err & QLOGIC_IB_E_PKTERRS) {
902 		if (!(err & ~QLOGIC_IB_E_PKTERRS))
903 			iserr = 0;
904 		if ((err & ERR_MASK(RcvICRCErr)) &&
905 		    !(err&(ERR_MASK(RcvVCRCErr)|ERR_MASK(RcvEBPErr))))
906 			strlcat(buf, "CRC ", blen);
907 		if (!iserr)
908 			goto done;
909 	}
910 	if (err & ERR_MASK(RcvHdrLenErr))
911 		strlcat(buf, "rhdrlen ", blen);
912 	if (err & ERR_MASK(RcvBadTidErr))
913 		strlcat(buf, "rbadtid ", blen);
914 	if (err & ERR_MASK(RcvBadVersionErr))
915 		strlcat(buf, "rbadversion ", blen);
916 	if (err & ERR_MASK(RcvHdrErr))
917 		strlcat(buf, "rhdr ", blen);
918 	if (err & ERR_MASK(RcvLongPktLenErr))
919 		strlcat(buf, "rlongpktlen ", blen);
920 	if (err & ERR_MASK(RcvMaxPktLenErr))
921 		strlcat(buf, "rmaxpktlen ", blen);
922 	if (err & ERR_MASK(RcvMinPktLenErr))
923 		strlcat(buf, "rminpktlen ", blen);
924 	if (err & ERR_MASK(SendMinPktLenErr))
925 		strlcat(buf, "sminpktlen ", blen);
926 	if (err & ERR_MASK(RcvFormatErr))
927 		strlcat(buf, "rformaterr ", blen);
928 	if (err & ERR_MASK(RcvUnsupportedVLErr))
929 		strlcat(buf, "runsupvl ", blen);
930 	if (err & ERR_MASK(RcvUnexpectedCharErr))
931 		strlcat(buf, "runexpchar ", blen);
932 	if (err & ERR_MASK(RcvIBFlowErr))
933 		strlcat(buf, "ribflow ", blen);
934 	if (err & ERR_MASK(SendUnderRunErr))
935 		strlcat(buf, "sunderrun ", blen);
936 	if (err & ERR_MASK(SendPioArmLaunchErr))
937 		strlcat(buf, "spioarmlaunch ", blen);
938 	if (err & ERR_MASK(SendUnexpectedPktNumErr))
939 		strlcat(buf, "sunexperrpktnum ", blen);
940 	if (err & ERR_MASK(SendDroppedSmpPktErr))
941 		strlcat(buf, "sdroppedsmppkt ", blen);
942 	if (err & ERR_MASK(SendMaxPktLenErr))
943 		strlcat(buf, "smaxpktlen ", blen);
944 	if (err & ERR_MASK(SendUnsupportedVLErr))
945 		strlcat(buf, "sunsupVL ", blen);
946 	if (err & ERR_MASK(InvalidAddrErr))
947 		strlcat(buf, "invalidaddr ", blen);
948 	if (err & ERR_MASK(RcvEgrFullErr))
949 		strlcat(buf, "rcvegrfull ", blen);
950 	if (err & ERR_MASK(RcvHdrFullErr))
951 		strlcat(buf, "rcvhdrfull ", blen);
952 	if (err & ERR_MASK(IBStatusChanged))
953 		strlcat(buf, "ibcstatuschg ", blen);
954 	if (err & ERR_MASK(RcvIBLostLinkErr))
955 		strlcat(buf, "riblostlink ", blen);
956 	if (err & ERR_MASK(HardwareErr))
957 		strlcat(buf, "hardware ", blen);
958 	if (err & ERR_MASK(ResetNegated))
959 		strlcat(buf, "reset ", blen);
960 done:
961 	return iserr;
962 }
963 
964 /*
965  * Called when we might have an error that is specific to a particular
966  * PIO buffer, and may need to cancel that buffer, so it can be re-used.
967  */
968 static void qib_disarm_6120_senderrbufs(struct qib_pportdata *ppd)
969 {
970 	unsigned long sbuf[2];
971 	struct qib_devdata *dd = ppd->dd;
972 
973 	/*
974 	 * It's possible that sendbuffererror could have bits set; might
975 	 * have already done this as a result of hardware error handling.
976 	 */
977 	sbuf[0] = qib_read_kreg64(dd, kr_sendbuffererror);
978 	sbuf[1] = qib_read_kreg64(dd, kr_sendbuffererror + 1);
979 
980 	if (sbuf[0] || sbuf[1])
981 		qib_disarm_piobufs_set(dd, sbuf,
982 				       dd->piobcnt2k + dd->piobcnt4k);
983 }
984 
985 static int chk_6120_linkrecovery(struct qib_devdata *dd, u64 ibcs)
986 {
987 	int ret = 1;
988 	u32 ibstate = qib_6120_iblink_state(ibcs);
989 	u32 linkrecov = read_6120_creg32(dd, cr_iblinkerrrecov);
990 
991 	if (linkrecov != dd->cspec->lastlinkrecov) {
992 		/* and no more until active again */
993 		dd->cspec->lastlinkrecov = 0;
994 		qib_set_linkstate(dd->pport, QIB_IB_LINKDOWN);
995 		ret = 0;
996 	}
997 	if (ibstate == IB_PORT_ACTIVE)
998 		dd->cspec->lastlinkrecov =
999 			read_6120_creg32(dd, cr_iblinkerrrecov);
1000 	return ret;
1001 }
1002 
1003 static void handle_6120_errors(struct qib_devdata *dd, u64 errs)
1004 {
1005 	char *msg;
1006 	u64 ignore_this_time = 0;
1007 	u64 iserr = 0;
1008 	int log_idx;
1009 	struct qib_pportdata *ppd = dd->pport;
1010 	u64 mask;
1011 
1012 	/* don't report errors that are masked */
1013 	errs &= dd->cspec->errormask;
1014 	msg = dd->cspec->emsgbuf;
1015 
1016 	/* do these first, they are most important */
1017 	if (errs & ERR_MASK(HardwareErr))
1018 		qib_handle_6120_hwerrors(dd, msg, sizeof(dd->cspec->emsgbuf));
1019 	else
1020 		for (log_idx = 0; log_idx < QIB_EEP_LOG_CNT; ++log_idx)
1021 			if (errs & dd->eep_st_masks[log_idx].errs_to_log)
1022 				qib_inc_eeprom_err(dd, log_idx, 1);
1023 
1024 	if (errs & ~IB_E_BITSEXTANT)
1025 		qib_dev_err(dd,
1026 			"error interrupt with unknown errors %llx set\n",
1027 			(unsigned long long) (errs & ~IB_E_BITSEXTANT));
1028 
1029 	if (errs & E_SUM_ERRS) {
1030 		qib_disarm_6120_senderrbufs(ppd);
1031 		if ((errs & E_SUM_LINK_PKTERRS) &&
1032 		    !(ppd->lflags & QIBL_LINKACTIVE)) {
1033 			/*
1034 			 * This can happen when trying to bring the link
1035 			 * up, but the IB link changes state at the "wrong"
1036 			 * time. The IB logic then complains that the packet
1037 			 * isn't valid.  We don't want to confuse people, so
1038 			 * we just don't print them, except at debug
1039 			 */
1040 			ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1041 		}
1042 	} else if ((errs & E_SUM_LINK_PKTERRS) &&
1043 		   !(ppd->lflags & QIBL_LINKACTIVE)) {
1044 		/*
1045 		 * This can happen when SMA is trying to bring the link
1046 		 * up, but the IB link changes state at the "wrong" time.
1047 		 * The IB logic then complains that the packet isn't
1048 		 * valid.  We don't want to confuse people, so we just
1049 		 * don't print them, except at debug
1050 		 */
1051 		ignore_this_time = errs & E_SUM_LINK_PKTERRS;
1052 	}
1053 
1054 	qib_write_kreg(dd, kr_errclear, errs);
1055 
1056 	errs &= ~ignore_this_time;
1057 	if (!errs)
1058 		goto done;
1059 
1060 	/*
1061 	 * The ones we mask off are handled specially below
1062 	 * or above.
1063 	 */
1064 	mask = ERR_MASK(IBStatusChanged) | ERR_MASK(RcvEgrFullErr) |
1065 		ERR_MASK(RcvHdrFullErr) | ERR_MASK(HardwareErr);
1066 	qib_decode_6120_err(dd, msg, sizeof(dd->cspec->emsgbuf), errs & ~mask);
1067 
1068 	if (errs & E_SUM_PKTERRS)
1069 		qib_stats.sps_rcverrs++;
1070 	if (errs & E_SUM_ERRS)
1071 		qib_stats.sps_txerrs++;
1072 
1073 	iserr = errs & ~(E_SUM_PKTERRS | QLOGIC_IB_E_PKTERRS);
1074 
1075 	if (errs & ERR_MASK(IBStatusChanged)) {
1076 		u64 ibcs = qib_read_kreg64(dd, kr_ibcstatus);
1077 		u32 ibstate = qib_6120_iblink_state(ibcs);
1078 		int handle = 1;
1079 
1080 		if (ibstate != IB_PORT_INIT && dd->cspec->lastlinkrecov)
1081 			handle = chk_6120_linkrecovery(dd, ibcs);
1082 		/*
1083 		 * Since going into a recovery state causes the link state
1084 		 * to go down and since recovery is transitory, it is better
1085 		 * if we "miss" ever seeing the link training state go into
1086 		 * recovery (i.e., ignore this transition for link state
1087 		 * special handling purposes) without updating lastibcstat.
1088 		 */
1089 		if (handle && qib_6120_phys_portstate(ibcs) ==
1090 					    IB_PHYSPORTSTATE_LINK_ERR_RECOVER)
1091 			handle = 0;
1092 		if (handle)
1093 			qib_handle_e_ibstatuschanged(ppd, ibcs);
1094 	}
1095 
1096 	if (errs & ERR_MASK(ResetNegated)) {
1097 		qib_dev_err(dd,
1098 			"Got reset, requires re-init (unload and reload driver)\n");
1099 		dd->flags &= ~QIB_INITTED;  /* needs re-init */
1100 		/* mark as having had error */
1101 		*dd->devstatusp |= QIB_STATUS_HWERROR;
1102 		*dd->pport->statusp &= ~QIB_STATUS_IB_CONF;
1103 	}
1104 
1105 	if (*msg && iserr)
1106 		qib_dev_porterr(dd, ppd->port, "%s error\n", msg);
1107 
1108 	if (ppd->state_wanted & ppd->lflags)
1109 		wake_up_interruptible(&ppd->state_wait);
1110 
1111 	/*
1112 	 * If there were hdrq or egrfull errors, wake up any processes
1113 	 * waiting in poll.  We used to try to check which contexts had
1114 	 * the overflow, but given the cost of that and the chip reads
1115 	 * to support it, it's better to just wake everybody up if we
1116 	 * get an overflow; waiters can poll again if it's not them.
1117 	 */
1118 	if (errs & (ERR_MASK(RcvEgrFullErr) | ERR_MASK(RcvHdrFullErr))) {
1119 		qib_handle_urcv(dd, ~0U);
1120 		if (errs & ERR_MASK(RcvEgrFullErr))
1121 			qib_stats.sps_buffull++;
1122 		else
1123 			qib_stats.sps_hdrfull++;
1124 	}
1125 done:
1126 	return;
1127 }
1128 
1129 /**
1130  * qib_6120_init_hwerrors - enable hardware errors
1131  * @dd: the qlogic_ib device
1132  *
1133  * now that we have finished initializing everything that might reasonably
1134  * cause a hardware error, and cleared those errors bits as they occur,
1135  * we can enable hardware errors in the mask (potentially enabling
1136  * freeze mode), and enable hardware errors as errors (along with
1137  * everything else) in errormask
1138  */
1139 static void qib_6120_init_hwerrors(struct qib_devdata *dd)
1140 {
1141 	u64 val;
1142 	u64 extsval;
1143 
1144 	extsval = qib_read_kreg64(dd, kr_extstatus);
1145 
1146 	if (!(extsval & QLOGIC_IB_EXTS_MEMBIST_ENDTEST))
1147 		qib_dev_err(dd, "MemBIST did not complete!\n");
1148 
1149 	/* init so all hwerrors interrupt, and enter freeze, ajdust below */
1150 	val = ~0ULL;
1151 	if (dd->minrev < 2) {
1152 		/*
1153 		 * Avoid problem with internal interface bus parity
1154 		 * checking. Fixed in Rev2.
1155 		 */
1156 		val &= ~QLOGIC_IB_HWE_PCIEBUSPARITYRADM;
1157 	}
1158 	/* avoid some intel cpu's speculative read freeze mode issue */
1159 	val &= ~TXEMEMPARITYERR_PIOBUF;
1160 
1161 	dd->cspec->hwerrmask = val;
1162 
1163 	qib_write_kreg(dd, kr_hwerrclear, ~HWE_MASK(PowerOnBISTFailed));
1164 	qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask);
1165 
1166 	/* clear all */
1167 	qib_write_kreg(dd, kr_errclear, ~0ULL);
1168 	/* enable errors that are masked, at least this first time. */
1169 	qib_write_kreg(dd, kr_errmask, ~0ULL);
1170 	dd->cspec->errormask = qib_read_kreg64(dd, kr_errmask);
1171 	/* clear any interrupts up to this point (ints still not enabled) */
1172 	qib_write_kreg(dd, kr_intclear, ~0ULL);
1173 
1174 	qib_write_kreg(dd, kr_rcvbthqp,
1175 		       dd->qpn_mask << (QIB_6120_RcvBTHQP_BTHQP_Mask_LSB - 1) |
1176 		       QIB_KD_QP);
1177 }
1178 
1179 /*
1180  * Disable and enable the armlaunch error.  Used for PIO bandwidth testing
1181  * on chips that are count-based, rather than trigger-based.  There is no
1182  * reference counting, but that's also fine, given the intended use.
1183  * Only chip-specific because it's all register accesses
1184  */
1185 static void qib_set_6120_armlaunch(struct qib_devdata *dd, u32 enable)
1186 {
1187 	if (enable) {
1188 		qib_write_kreg(dd, kr_errclear,
1189 			       ERR_MASK(SendPioArmLaunchErr));
1190 		dd->cspec->errormask |= ERR_MASK(SendPioArmLaunchErr);
1191 	} else
1192 		dd->cspec->errormask &= ~ERR_MASK(SendPioArmLaunchErr);
1193 	qib_write_kreg(dd, kr_errmask, dd->cspec->errormask);
1194 }
1195 
1196 /*
1197  * Formerly took parameter <which> in pre-shifted,
1198  * pre-merged form with LinkCmd and LinkInitCmd
1199  * together, and assuming the zero was NOP.
1200  */
1201 static void qib_set_ib_6120_lstate(struct qib_pportdata *ppd, u16 linkcmd,
1202 				   u16 linitcmd)
1203 {
1204 	u64 mod_wd;
1205 	struct qib_devdata *dd = ppd->dd;
1206 	unsigned long flags;
1207 
1208 	if (linitcmd == QLOGIC_IB_IBCC_LINKINITCMD_DISABLE) {
1209 		/*
1210 		 * If we are told to disable, note that so link-recovery
1211 		 * code does not attempt to bring us back up.
1212 		 */
1213 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1214 		ppd->lflags |= QIBL_IB_LINK_DISABLED;
1215 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1216 	} else if (linitcmd || linkcmd == QLOGIC_IB_IBCC_LINKCMD_DOWN) {
1217 		/*
1218 		 * Any other linkinitcmd will lead to LINKDOWN and then
1219 		 * to INIT (if all is well), so clear flag to let
1220 		 * link-recovery code attempt to bring us back up.
1221 		 */
1222 		spin_lock_irqsave(&ppd->lflags_lock, flags);
1223 		ppd->lflags &= ~QIBL_IB_LINK_DISABLED;
1224 		spin_unlock_irqrestore(&ppd->lflags_lock, flags);
1225 	}
1226 
1227 	mod_wd = (linkcmd << QLOGIC_IB_IBCC_LINKCMD_SHIFT) |
1228 		(linitcmd << QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1229 
1230 	qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl | mod_wd);
1231 	/* write to chip to prevent back-to-back writes of control reg */
1232 	qib_write_kreg(dd, kr_scratch, 0);
1233 }
1234 
1235 /**
1236  * qib_6120_bringup_serdes - bring up the serdes
1237  * @dd: the qlogic_ib device
1238  */
1239 static int qib_6120_bringup_serdes(struct qib_pportdata *ppd)
1240 {
1241 	struct qib_devdata *dd = ppd->dd;
1242 	u64 val, config1, prev_val, hwstat, ibc;
1243 
1244 	/* Put IBC in reset, sends disabled */
1245 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1246 	qib_write_kreg(dd, kr_control, 0ULL);
1247 
1248 	dd->cspec->ibdeltainprog = 1;
1249 	dd->cspec->ibsymsnap = read_6120_creg32(dd, cr_ibsymbolerr);
1250 	dd->cspec->iblnkerrsnap = read_6120_creg32(dd, cr_iblinkerrrecov);
1251 
1252 	/* flowcontrolwatermark is in units of KBytes */
1253 	ibc = 0x5ULL << SYM_LSB(IBCCtrl, FlowCtrlWaterMark);
1254 	/*
1255 	 * How often flowctrl sent.  More or less in usecs; balance against
1256 	 * watermark value, so that in theory senders always get a flow
1257 	 * control update in time to not let the IB link go idle.
1258 	 */
1259 	ibc |= 0x3ULL << SYM_LSB(IBCCtrl, FlowCtrlPeriod);
1260 	/* max error tolerance */
1261 	dd->cspec->lli_thresh = 0xf;
1262 	ibc |= (u64) dd->cspec->lli_thresh << SYM_LSB(IBCCtrl, PhyerrThreshold);
1263 	/* use "real" buffer space for */
1264 	ibc |= 4ULL << SYM_LSB(IBCCtrl, CreditScale);
1265 	/* IB credit flow control. */
1266 	ibc |= 0xfULL << SYM_LSB(IBCCtrl, OverrunThreshold);
1267 	/*
1268 	 * set initial max size pkt IBC will send, including ICRC; it's the
1269 	 * PIO buffer size in dwords, less 1; also see qib_set_mtu()
1270 	 */
1271 	ibc |= ((u64)(ppd->ibmaxlen >> 2) + 1) << SYM_LSB(IBCCtrl, MaxPktLen);
1272 	dd->cspec->ibcctrl = ibc; /* without linkcmd or linkinitcmd! */
1273 
1274 	/* initially come up waiting for TS1, without sending anything. */
1275 	val = dd->cspec->ibcctrl | (QLOGIC_IB_IBCC_LINKINITCMD_DISABLE <<
1276 		QLOGIC_IB_IBCC_LINKINITCMD_SHIFT);
1277 	qib_write_kreg(dd, kr_ibcctrl, val);
1278 
1279 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1280 	config1 = qib_read_kreg64(dd, kr_serdes_cfg1);
1281 
1282 	/*
1283 	 * Force reset on, also set rxdetect enable.  Must do before reading
1284 	 * serdesstatus at least for simulation, or some of the bits in
1285 	 * serdes status will come back as undefined and cause simulation
1286 	 * failures
1287 	 */
1288 	val |= SYM_MASK(SerdesCfg0, ResetPLL) |
1289 		SYM_MASK(SerdesCfg0, RxDetEnX) |
1290 		(SYM_MASK(SerdesCfg0, L1PwrDnA) |
1291 		 SYM_MASK(SerdesCfg0, L1PwrDnB) |
1292 		 SYM_MASK(SerdesCfg0, L1PwrDnC) |
1293 		 SYM_MASK(SerdesCfg0, L1PwrDnD));
1294 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1295 	/* be sure chip saw it */
1296 	qib_read_kreg64(dd, kr_scratch);
1297 	udelay(5);              /* need pll reset set at least for a bit */
1298 	/*
1299 	 * after PLL is reset, set the per-lane Resets and TxIdle and
1300 	 * clear the PLL reset and rxdetect (to get falling edge).
1301 	 * Leave L1PWR bits set (permanently)
1302 	 */
1303 	val &= ~(SYM_MASK(SerdesCfg0, RxDetEnX) |
1304 		 SYM_MASK(SerdesCfg0, ResetPLL) |
1305 		 (SYM_MASK(SerdesCfg0, L1PwrDnA) |
1306 		  SYM_MASK(SerdesCfg0, L1PwrDnB) |
1307 		  SYM_MASK(SerdesCfg0, L1PwrDnC) |
1308 		  SYM_MASK(SerdesCfg0, L1PwrDnD)));
1309 	val |= (SYM_MASK(SerdesCfg0, ResetA) |
1310 		SYM_MASK(SerdesCfg0, ResetB) |
1311 		SYM_MASK(SerdesCfg0, ResetC) |
1312 		SYM_MASK(SerdesCfg0, ResetD)) |
1313 		SYM_MASK(SerdesCfg0, TxIdeEnX);
1314 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1315 	/* be sure chip saw it */
1316 	(void) qib_read_kreg64(dd, kr_scratch);
1317 	/* need PLL reset clear for at least 11 usec before lane
1318 	 * resets cleared; give it a few more to be sure */
1319 	udelay(15);
1320 	val &= ~((SYM_MASK(SerdesCfg0, ResetA) |
1321 		  SYM_MASK(SerdesCfg0, ResetB) |
1322 		  SYM_MASK(SerdesCfg0, ResetC) |
1323 		  SYM_MASK(SerdesCfg0, ResetD)) |
1324 		 SYM_MASK(SerdesCfg0, TxIdeEnX));
1325 
1326 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1327 	/* be sure chip saw it */
1328 	(void) qib_read_kreg64(dd, kr_scratch);
1329 
1330 	val = qib_read_kreg64(dd, kr_xgxs_cfg);
1331 	prev_val = val;
1332 	if (val & QLOGIC_IB_XGXS_RESET)
1333 		val &= ~QLOGIC_IB_XGXS_RESET;
1334 	if (SYM_FIELD(val, XGXSCfg, polarity_inv) != ppd->rx_pol_inv) {
1335 		/* need to compensate for Tx inversion in partner */
1336 		val &= ~SYM_MASK(XGXSCfg, polarity_inv);
1337 		val |= (u64)ppd->rx_pol_inv << SYM_LSB(XGXSCfg, polarity_inv);
1338 	}
1339 	if (val != prev_val)
1340 		qib_write_kreg(dd, kr_xgxs_cfg, val);
1341 
1342 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1343 
1344 	/* clear current and de-emphasis bits */
1345 	config1 &= ~0x0ffffffff00ULL;
1346 	/* set current to 20ma */
1347 	config1 |= 0x00000000000ULL;
1348 	/* set de-emphasis to -5.68dB */
1349 	config1 |= 0x0cccc000000ULL;
1350 	qib_write_kreg(dd, kr_serdes_cfg1, config1);
1351 
1352 	/* base and port guid same for single port */
1353 	ppd->guid = dd->base_guid;
1354 
1355 	/*
1356 	 * the process of setting and un-resetting the serdes normally
1357 	 * causes a serdes PLL error, so check for that and clear it
1358 	 * here.  Also clearr hwerr bit in errstatus, but not others.
1359 	 */
1360 	hwstat = qib_read_kreg64(dd, kr_hwerrstatus);
1361 	if (hwstat) {
1362 		/* should just have PLL, clear all set, in an case */
1363 		qib_write_kreg(dd, kr_hwerrclear, hwstat);
1364 		qib_write_kreg(dd, kr_errclear, ERR_MASK(HardwareErr));
1365 	}
1366 
1367 	dd->control |= QLOGIC_IB_C_LINKENABLE;
1368 	dd->control &= ~QLOGIC_IB_C_FREEZEMODE;
1369 	qib_write_kreg(dd, kr_control, dd->control);
1370 
1371 	return 0;
1372 }
1373 
1374 /**
1375  * qib_6120_quiet_serdes - set serdes to txidle
1376  * @ppd: physical port of the qlogic_ib device
1377  * Called when driver is being unloaded
1378  */
1379 static void qib_6120_quiet_serdes(struct qib_pportdata *ppd)
1380 {
1381 	struct qib_devdata *dd = ppd->dd;
1382 	u64 val;
1383 
1384 	qib_set_ib_6120_lstate(ppd, 0, QLOGIC_IB_IBCC_LINKINITCMD_DISABLE);
1385 
1386 	/* disable IBC */
1387 	dd->control &= ~QLOGIC_IB_C_LINKENABLE;
1388 	qib_write_kreg(dd, kr_control,
1389 		       dd->control | QLOGIC_IB_C_FREEZEMODE);
1390 
1391 	if (dd->cspec->ibsymdelta || dd->cspec->iblnkerrdelta ||
1392 	    dd->cspec->ibdeltainprog) {
1393 		u64 diagc;
1394 
1395 		/* enable counter writes */
1396 		diagc = qib_read_kreg64(dd, kr_hwdiagctrl);
1397 		qib_write_kreg(dd, kr_hwdiagctrl,
1398 			       diagc | SYM_MASK(HwDiagCtrl, CounterWrEnable));
1399 
1400 		if (dd->cspec->ibsymdelta || dd->cspec->ibdeltainprog) {
1401 			val = read_6120_creg32(dd, cr_ibsymbolerr);
1402 			if (dd->cspec->ibdeltainprog)
1403 				val -= val - dd->cspec->ibsymsnap;
1404 			val -= dd->cspec->ibsymdelta;
1405 			write_6120_creg(dd, cr_ibsymbolerr, val);
1406 		}
1407 		if (dd->cspec->iblnkerrdelta || dd->cspec->ibdeltainprog) {
1408 			val = read_6120_creg32(dd, cr_iblinkerrrecov);
1409 			if (dd->cspec->ibdeltainprog)
1410 				val -= val - dd->cspec->iblnkerrsnap;
1411 			val -= dd->cspec->iblnkerrdelta;
1412 			write_6120_creg(dd, cr_iblinkerrrecov, val);
1413 		}
1414 
1415 		/* and disable counter writes */
1416 		qib_write_kreg(dd, kr_hwdiagctrl, diagc);
1417 	}
1418 
1419 	val = qib_read_kreg64(dd, kr_serdes_cfg0);
1420 	val |= SYM_MASK(SerdesCfg0, TxIdeEnX);
1421 	qib_write_kreg(dd, kr_serdes_cfg0, val);
1422 }
1423 
1424 /**
1425  * qib_6120_setup_setextled - set the state of the two external LEDs
1426  * @dd: the qlogic_ib device
1427  * @on: whether the link is up or not
1428  *
1429  * The exact combo of LEDs if on is true is determined by looking
1430  * at the ibcstatus.
1431 
1432  * These LEDs indicate the physical and logical state of IB link.
1433  * For this chip (at least with recommended board pinouts), LED1
1434  * is Yellow (logical state) and LED2 is Green (physical state),
1435  *
1436  * Note:  We try to match the Mellanox HCA LED behavior as best
1437  * we can.  Green indicates physical link state is OK (something is
1438  * plugged in, and we can train).
1439  * Amber indicates the link is logically up (ACTIVE).
1440  * Mellanox further blinks the amber LED to indicate data packet
1441  * activity, but we have no hardware support for that, so it would
1442  * require waking up every 10-20 msecs and checking the counters
1443  * on the chip, and then turning the LED off if appropriate.  That's
1444  * visible overhead, so not something we will do.
1445  *
1446  */
1447 static void qib_6120_setup_setextled(struct qib_pportdata *ppd, u32 on)
1448 {
1449 	u64 extctl, val, lst, ltst;
1450 	unsigned long flags;
1451 	struct qib_devdata *dd = ppd->dd;
1452 
1453 	/*
1454 	 * The diags use the LED to indicate diag info, so we leave
1455 	 * the external LED alone when the diags are running.
1456 	 */
1457 	if (dd->diag_client)
1458 		return;
1459 
1460 	/* Allow override of LED display for, e.g. Locating system in rack */
1461 	if (ppd->led_override) {
1462 		ltst = (ppd->led_override & QIB_LED_PHYS) ?
1463 			IB_PHYSPORTSTATE_LINKUP : IB_PHYSPORTSTATE_DISABLED,
1464 		lst = (ppd->led_override & QIB_LED_LOG) ?
1465 			IB_PORT_ACTIVE : IB_PORT_DOWN;
1466 	} else if (on) {
1467 		val = qib_read_kreg64(dd, kr_ibcstatus);
1468 		ltst = qib_6120_phys_portstate(val);
1469 		lst = qib_6120_iblink_state(val);
1470 	} else {
1471 		ltst = 0;
1472 		lst = 0;
1473 	}
1474 
1475 	spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
1476 	extctl = dd->cspec->extctrl & ~(SYM_MASK(EXTCtrl, LEDPriPortGreenOn) |
1477 				 SYM_MASK(EXTCtrl, LEDPriPortYellowOn));
1478 
1479 	if (ltst == IB_PHYSPORTSTATE_LINKUP)
1480 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortYellowOn);
1481 	if (lst == IB_PORT_ACTIVE)
1482 		extctl |= SYM_MASK(EXTCtrl, LEDPriPortGreenOn);
1483 	dd->cspec->extctrl = extctl;
1484 	qib_write_kreg(dd, kr_extctrl, extctl);
1485 	spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
1486 }
1487 
1488 static void qib_6120_free_irq(struct qib_devdata *dd)
1489 {
1490 	if (dd->cspec->irq) {
1491 		free_irq(dd->cspec->irq, dd);
1492 		dd->cspec->irq = 0;
1493 	}
1494 	qib_nomsi(dd);
1495 }
1496 
1497 /**
1498  * qib_6120_setup_cleanup - clean up any per-chip chip-specific stuff
1499  * @dd: the qlogic_ib device
1500  *
1501  * This is called during driver unload.
1502 */
1503 static void qib_6120_setup_cleanup(struct qib_devdata *dd)
1504 {
1505 	qib_6120_free_irq(dd);
1506 	kfree(dd->cspec->cntrs);
1507 	kfree(dd->cspec->portcntrs);
1508 	if (dd->cspec->dummy_hdrq) {
1509 		dma_free_coherent(&dd->pcidev->dev,
1510 				  ALIGN(dd->rcvhdrcnt *
1511 					dd->rcvhdrentsize *
1512 					sizeof(u32), PAGE_SIZE),
1513 				  dd->cspec->dummy_hdrq,
1514 				  dd->cspec->dummy_hdrq_phys);
1515 		dd->cspec->dummy_hdrq = NULL;
1516 	}
1517 }
1518 
1519 static void qib_wantpiobuf_6120_intr(struct qib_devdata *dd, u32 needint)
1520 {
1521 	unsigned long flags;
1522 
1523 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
1524 	if (needint)
1525 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOIntBufAvail);
1526 	else
1527 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOIntBufAvail);
1528 	qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
1529 	qib_write_kreg(dd, kr_scratch, 0ULL);
1530 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
1531 }
1532 
1533 /*
1534  * handle errors and unusual events first, separate function
1535  * to improve cache hits for fast path interrupt handling
1536  */
1537 static noinline void unlikely_6120_intr(struct qib_devdata *dd, u64 istat)
1538 {
1539 	if (unlikely(istat & ~QLOGIC_IB_I_BITSEXTANT))
1540 		qib_dev_err(dd, "interrupt with unknown interrupts %Lx set\n",
1541 			    istat & ~QLOGIC_IB_I_BITSEXTANT);
1542 
1543 	if (istat & QLOGIC_IB_I_ERROR) {
1544 		u64 estat = 0;
1545 
1546 		qib_stats.sps_errints++;
1547 		estat = qib_read_kreg64(dd, kr_errstatus);
1548 		if (!estat)
1549 			qib_devinfo(dd->pcidev,
1550 				"error interrupt (%Lx), but no error bits set!\n",
1551 				istat);
1552 		handle_6120_errors(dd, estat);
1553 	}
1554 
1555 	if (istat & QLOGIC_IB_I_GPIO) {
1556 		u32 gpiostatus;
1557 		u32 to_clear = 0;
1558 
1559 		/*
1560 		 * GPIO_3..5 on IBA6120 Rev2 chips indicate
1561 		 * errors that we need to count.
1562 		 */
1563 		gpiostatus = qib_read_kreg32(dd, kr_gpio_status);
1564 		/* First the error-counter case. */
1565 		if (gpiostatus & GPIO_ERRINTR_MASK) {
1566 			/* want to clear the bits we see asserted. */
1567 			to_clear |= (gpiostatus & GPIO_ERRINTR_MASK);
1568 
1569 			/*
1570 			 * Count appropriately, clear bits out of our copy,
1571 			 * as they have been "handled".
1572 			 */
1573 			if (gpiostatus & (1 << GPIO_RXUVL_BIT))
1574 				dd->cspec->rxfc_unsupvl_errs++;
1575 			if (gpiostatus & (1 << GPIO_OVRUN_BIT))
1576 				dd->cspec->overrun_thresh_errs++;
1577 			if (gpiostatus & (1 << GPIO_LLI_BIT))
1578 				dd->cspec->lli_errs++;
1579 			gpiostatus &= ~GPIO_ERRINTR_MASK;
1580 		}
1581 		if (gpiostatus) {
1582 			/*
1583 			 * Some unexpected bits remain. If they could have
1584 			 * caused the interrupt, complain and clear.
1585 			 * To avoid repetition of this condition, also clear
1586 			 * the mask. It is almost certainly due to error.
1587 			 */
1588 			const u32 mask = qib_read_kreg32(dd, kr_gpio_mask);
1589 
1590 			/*
1591 			 * Also check that the chip reflects our shadow,
1592 			 * and report issues, If they caused the interrupt.
1593 			 * we will suppress by refreshing from the shadow.
1594 			 */
1595 			if (mask & gpiostatus) {
1596 				to_clear |= (gpiostatus & mask);
1597 				dd->cspec->gpio_mask &= ~(gpiostatus & mask);
1598 				qib_write_kreg(dd, kr_gpio_mask,
1599 					       dd->cspec->gpio_mask);
1600 			}
1601 		}
1602 		if (to_clear)
1603 			qib_write_kreg(dd, kr_gpio_clear, (u64) to_clear);
1604 	}
1605 }
1606 
1607 static irqreturn_t qib_6120intr(int irq, void *data)
1608 {
1609 	struct qib_devdata *dd = data;
1610 	irqreturn_t ret;
1611 	u32 istat, ctxtrbits, rmask, crcs = 0;
1612 	unsigned i;
1613 
1614 	if ((dd->flags & (QIB_PRESENT | QIB_BADINTR)) != QIB_PRESENT) {
1615 		/*
1616 		 * This return value is not great, but we do not want the
1617 		 * interrupt core code to remove our interrupt handler
1618 		 * because we don't appear to be handling an interrupt
1619 		 * during a chip reset.
1620 		 */
1621 		ret = IRQ_HANDLED;
1622 		goto bail;
1623 	}
1624 
1625 	istat = qib_read_kreg32(dd, kr_intstatus);
1626 
1627 	if (unlikely(!istat)) {
1628 		ret = IRQ_NONE; /* not our interrupt, or already handled */
1629 		goto bail;
1630 	}
1631 	if (unlikely(istat == -1)) {
1632 		qib_bad_intrstatus(dd);
1633 		/* don't know if it was our interrupt or not */
1634 		ret = IRQ_NONE;
1635 		goto bail;
1636 	}
1637 
1638 	this_cpu_inc(*dd->int_counter);
1639 
1640 	if (unlikely(istat & (~QLOGIC_IB_I_BITSEXTANT |
1641 			      QLOGIC_IB_I_GPIO | QLOGIC_IB_I_ERROR)))
1642 		unlikely_6120_intr(dd, istat);
1643 
1644 	/*
1645 	 * Clear the interrupt bits we found set, relatively early, so we
1646 	 * "know" know the chip will have seen this by the time we process
1647 	 * the queue, and will re-interrupt if necessary.  The processor
1648 	 * itself won't take the interrupt again until we return.
1649 	 */
1650 	qib_write_kreg(dd, kr_intclear, istat);
1651 
1652 	/*
1653 	 * Handle kernel receive queues before checking for pio buffers
1654 	 * available since receives can overflow; piobuf waiters can afford
1655 	 * a few extra cycles, since they were waiting anyway.
1656 	 */
1657 	ctxtrbits = istat &
1658 		((QLOGIC_IB_I_RCVAVAIL_MASK << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1659 		 (QLOGIC_IB_I_RCVURG_MASK << QLOGIC_IB_I_RCVURG_SHIFT));
1660 	if (ctxtrbits) {
1661 		rmask = (1U << QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1662 			(1U << QLOGIC_IB_I_RCVURG_SHIFT);
1663 		for (i = 0; i < dd->first_user_ctxt; i++) {
1664 			if (ctxtrbits & rmask) {
1665 				ctxtrbits &= ~rmask;
1666 				crcs += qib_kreceive(dd->rcd[i],
1667 						     &dd->cspec->lli_counter,
1668 						     NULL);
1669 			}
1670 			rmask <<= 1;
1671 		}
1672 		if (crcs) {
1673 			u32 cntr = dd->cspec->lli_counter;
1674 
1675 			cntr += crcs;
1676 			if (cntr) {
1677 				if (cntr > dd->cspec->lli_thresh) {
1678 					dd->cspec->lli_counter = 0;
1679 					dd->cspec->lli_errs++;
1680 				} else
1681 					dd->cspec->lli_counter += cntr;
1682 			}
1683 		}
1684 
1685 
1686 		if (ctxtrbits) {
1687 			ctxtrbits =
1688 				(ctxtrbits >> QLOGIC_IB_I_RCVAVAIL_SHIFT) |
1689 				(ctxtrbits >> QLOGIC_IB_I_RCVURG_SHIFT);
1690 			qib_handle_urcv(dd, ctxtrbits);
1691 		}
1692 	}
1693 
1694 	if ((istat & QLOGIC_IB_I_SPIOBUFAVAIL) && (dd->flags & QIB_INITTED))
1695 		qib_ib_piobufavail(dd);
1696 
1697 	ret = IRQ_HANDLED;
1698 bail:
1699 	return ret;
1700 }
1701 
1702 /*
1703  * Set up our chip-specific interrupt handler
1704  * The interrupt type has already been setup, so
1705  * we just need to do the registration and error checking.
1706  */
1707 static void qib_setup_6120_interrupt(struct qib_devdata *dd)
1708 {
1709 	/*
1710 	 * If the chip supports added error indication via GPIO pins,
1711 	 * enable interrupts on those bits so the interrupt routine
1712 	 * can count the events. Also set flag so interrupt routine
1713 	 * can know they are expected.
1714 	 */
1715 	if (SYM_FIELD(dd->revision, Revision_R,
1716 		      ChipRevMinor) > 1) {
1717 		/* Rev2+ reports extra errors via internal GPIO pins */
1718 		dd->cspec->gpio_mask |= GPIO_ERRINTR_MASK;
1719 		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1720 	}
1721 
1722 	if (!dd->cspec->irq)
1723 		qib_dev_err(dd,
1724 			"irq is 0, BIOS error?  Interrupts won't work\n");
1725 	else {
1726 		int ret;
1727 
1728 		ret = request_irq(dd->cspec->irq, qib_6120intr, 0,
1729 				  QIB_DRV_NAME, dd);
1730 		if (ret)
1731 			qib_dev_err(dd,
1732 				"Couldn't setup interrupt (irq=%d): %d\n",
1733 				dd->cspec->irq, ret);
1734 	}
1735 }
1736 
1737 /**
1738  * pe_boardname - fill in the board name
1739  * @dd: the qlogic_ib device
1740  *
1741  * info is based on the board revision register
1742  */
1743 static void pe_boardname(struct qib_devdata *dd)
1744 {
1745 	char *n;
1746 	u32 boardid, namelen;
1747 
1748 	boardid = SYM_FIELD(dd->revision, Revision,
1749 			    BoardID);
1750 
1751 	switch (boardid) {
1752 	case 2:
1753 		n = "InfiniPath_QLE7140";
1754 		break;
1755 	default:
1756 		qib_dev_err(dd, "Unknown 6120 board with ID %u\n", boardid);
1757 		n = "Unknown_InfiniPath_6120";
1758 		break;
1759 	}
1760 	namelen = strlen(n) + 1;
1761 	dd->boardname = kmalloc(namelen, GFP_KERNEL);
1762 	if (!dd->boardname)
1763 		qib_dev_err(dd, "Failed allocation for board name: %s\n", n);
1764 	else
1765 		snprintf(dd->boardname, namelen, "%s", n);
1766 
1767 	if (dd->majrev != 4 || !dd->minrev || dd->minrev > 2)
1768 		qib_dev_err(dd,
1769 			"Unsupported InfiniPath hardware revision %u.%u!\n",
1770 			dd->majrev, dd->minrev);
1771 
1772 	snprintf(dd->boardversion, sizeof(dd->boardversion),
1773 		 "ChipABI %u.%u, %s, InfiniPath%u %u.%u, SW Compat %u\n",
1774 		 QIB_CHIP_VERS_MAJ, QIB_CHIP_VERS_MIN, dd->boardname,
1775 		 (unsigned)SYM_FIELD(dd->revision, Revision_R, Arch),
1776 		 dd->majrev, dd->minrev,
1777 		 (unsigned)SYM_FIELD(dd->revision, Revision_R, SW));
1778 
1779 }
1780 
1781 /*
1782  * This routine sleeps, so it can only be called from user context, not
1783  * from interrupt context.  If we need interrupt context, we can split
1784  * it into two routines.
1785  */
1786 static int qib_6120_setup_reset(struct qib_devdata *dd)
1787 {
1788 	u64 val;
1789 	int i;
1790 	int ret;
1791 	u16 cmdval;
1792 	u8 int_line, clinesz;
1793 
1794 	qib_pcie_getcmd(dd, &cmdval, &int_line, &clinesz);
1795 
1796 	/* Use ERROR so it shows up in logs, etc. */
1797 	qib_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->unit);
1798 
1799 	/* no interrupts till re-initted */
1800 	qib_6120_set_intr_state(dd, 0);
1801 
1802 	dd->cspec->ibdeltainprog = 0;
1803 	dd->cspec->ibsymdelta = 0;
1804 	dd->cspec->iblnkerrdelta = 0;
1805 
1806 	/*
1807 	 * Keep chip from being accessed until we are ready.  Use
1808 	 * writeq() directly, to allow the write even though QIB_PRESENT
1809 	 * isn't set.
1810 	 */
1811 	dd->flags &= ~(QIB_INITTED | QIB_PRESENT);
1812 	/* so we check interrupts work again */
1813 	dd->z_int_counter = qib_int_counter(dd);
1814 	val = dd->control | QLOGIC_IB_C_RESET;
1815 	writeq(val, &dd->kregbase[kr_control]);
1816 	mb(); /* prevent compiler re-ordering around actual reset */
1817 
1818 	for (i = 1; i <= 5; i++) {
1819 		/*
1820 		 * Allow MBIST, etc. to complete; longer on each retry.
1821 		 * We sometimes get machine checks from bus timeout if no
1822 		 * response, so for now, make it *really* long.
1823 		 */
1824 		msleep(1000 + (1 + i) * 2000);
1825 
1826 		qib_pcie_reenable(dd, cmdval, int_line, clinesz);
1827 
1828 		/*
1829 		 * Use readq directly, so we don't need to mark it as PRESENT
1830 		 * until we get a successful indication that all is well.
1831 		 */
1832 		val = readq(&dd->kregbase[kr_revision]);
1833 		if (val == dd->revision) {
1834 			dd->flags |= QIB_PRESENT; /* it's back */
1835 			ret = qib_reinit_intr(dd);
1836 			goto bail;
1837 		}
1838 	}
1839 	ret = 0; /* failed */
1840 
1841 bail:
1842 	if (ret) {
1843 		if (qib_pcie_params(dd, dd->lbus_width, NULL, NULL))
1844 			qib_dev_err(dd,
1845 				"Reset failed to setup PCIe or interrupts; continuing anyway\n");
1846 		/* clear the reset error, init error/hwerror mask */
1847 		qib_6120_init_hwerrors(dd);
1848 		/* for Rev2 error interrupts; nop for rev 1 */
1849 		qib_write_kreg(dd, kr_gpio_mask, dd->cspec->gpio_mask);
1850 		/* clear the reset error, init error/hwerror mask */
1851 		qib_6120_init_hwerrors(dd);
1852 	}
1853 	return ret;
1854 }
1855 
1856 /**
1857  * qib_6120_put_tid - write a TID in chip
1858  * @dd: the qlogic_ib device
1859  * @tidptr: pointer to the expected TID (in chip) to update
1860  * @tidtype: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1861  * for expected
1862  * @pa: physical address of in memory buffer; tidinvalid if freeing
1863  *
1864  * This exists as a separate routine to allow for special locking etc.
1865  * It's used for both the full cleanup on exit, as well as the normal
1866  * setup and teardown.
1867  */
1868 static void qib_6120_put_tid(struct qib_devdata *dd, u64 __iomem *tidptr,
1869 			     u32 type, unsigned long pa)
1870 {
1871 	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1872 	unsigned long flags;
1873 	int tidx;
1874 	spinlock_t *tidlockp; /* select appropriate spinlock */
1875 
1876 	if (!dd->kregbase)
1877 		return;
1878 
1879 	if (pa != dd->tidinvalid) {
1880 		if (pa & ((1U << 11) - 1)) {
1881 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1882 				    pa);
1883 			return;
1884 		}
1885 		pa >>= 11;
1886 		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1887 			qib_dev_err(dd,
1888 				"Physical page address 0x%lx larger than supported\n",
1889 				pa);
1890 			return;
1891 		}
1892 
1893 		if (type == RCVHQ_RCV_TYPE_EAGER)
1894 			pa |= dd->tidtemplate;
1895 		else /* for now, always full 4KB page */
1896 			pa |= 2 << 29;
1897 	}
1898 
1899 	/*
1900 	 * Avoid chip issue by writing the scratch register
1901 	 * before and after the TID, and with an io write barrier.
1902 	 * We use a spinlock around the writes, so they can't intermix
1903 	 * with other TID (eager or expected) writes (the chip problem
1904 	 * is triggered by back to back TID writes). Unfortunately, this
1905 	 * call can be done from interrupt level for the ctxt 0 eager TIDs,
1906 	 * so we have to use irqsave locks.
1907 	 */
1908 	/*
1909 	 * Assumes tidptr always > egrtidbase
1910 	 * if type == RCVHQ_RCV_TYPE_EAGER.
1911 	 */
1912 	tidx = tidptr - dd->egrtidbase;
1913 
1914 	tidlockp = (type == RCVHQ_RCV_TYPE_EAGER && tidx < dd->rcvhdrcnt)
1915 		? &dd->cspec->kernel_tid_lock : &dd->cspec->user_tid_lock;
1916 	spin_lock_irqsave(tidlockp, flags);
1917 	qib_write_kreg(dd, kr_scratch, 0xfeeddeaf);
1918 	writel(pa, tidp32);
1919 	qib_write_kreg(dd, kr_scratch, 0xdeadbeef);
1920 	mmiowb();
1921 	spin_unlock_irqrestore(tidlockp, flags);
1922 }
1923 
1924 /**
1925  * qib_6120_put_tid_2 - write a TID in chip, Revision 2 or higher
1926  * @dd: the qlogic_ib device
1927  * @tidptr: pointer to the expected TID (in chip) to update
1928  * @tidtype: RCVHQ_RCV_TYPE_EAGER (1) for eager, RCVHQ_RCV_TYPE_EXPECTED (0)
1929  * for expected
1930  * @pa: physical address of in memory buffer; tidinvalid if freeing
1931  *
1932  * This exists as a separate routine to allow for selection of the
1933  * appropriate "flavor". The static calls in cleanup just use the
1934  * revision-agnostic form, as they are not performance critical.
1935  */
1936 static void qib_6120_put_tid_2(struct qib_devdata *dd, u64 __iomem *tidptr,
1937 			       u32 type, unsigned long pa)
1938 {
1939 	u32 __iomem *tidp32 = (u32 __iomem *)tidptr;
1940 	u32 tidx;
1941 
1942 	if (!dd->kregbase)
1943 		return;
1944 
1945 	if (pa != dd->tidinvalid) {
1946 		if (pa & ((1U << 11) - 1)) {
1947 			qib_dev_err(dd, "Physaddr %lx not 2KB aligned!\n",
1948 				    pa);
1949 			return;
1950 		}
1951 		pa >>= 11;
1952 		if (pa & ~QLOGIC_IB_RT_ADDR_MASK) {
1953 			qib_dev_err(dd,
1954 				"Physical page address 0x%lx larger than supported\n",
1955 				pa);
1956 			return;
1957 		}
1958 
1959 		if (type == RCVHQ_RCV_TYPE_EAGER)
1960 			pa |= dd->tidtemplate;
1961 		else /* for now, always full 4KB page */
1962 			pa |= 2 << 29;
1963 	}
1964 	tidx = tidptr - dd->egrtidbase;
1965 	writel(pa, tidp32);
1966 	mmiowb();
1967 }
1968 
1969 
1970 /**
1971  * qib_6120_clear_tids - clear all TID entries for a context, expected and eager
1972  * @dd: the qlogic_ib device
1973  * @ctxt: the context
1974  *
1975  * clear all TID entries for a context, expected and eager.
1976  * Used from qib_close().  On this chip, TIDs are only 32 bits,
1977  * not 64, but they are still on 64 bit boundaries, so tidbase
1978  * is declared as u64 * for the pointer math, even though we write 32 bits
1979  */
1980 static void qib_6120_clear_tids(struct qib_devdata *dd,
1981 				struct qib_ctxtdata *rcd)
1982 {
1983 	u64 __iomem *tidbase;
1984 	unsigned long tidinv;
1985 	u32 ctxt;
1986 	int i;
1987 
1988 	if (!dd->kregbase || !rcd)
1989 		return;
1990 
1991 	ctxt = rcd->ctxt;
1992 
1993 	tidinv = dd->tidinvalid;
1994 	tidbase = (u64 __iomem *)
1995 		((char __iomem *)(dd->kregbase) +
1996 		 dd->rcvtidbase +
1997 		 ctxt * dd->rcvtidcnt * sizeof(*tidbase));
1998 
1999 	for (i = 0; i < dd->rcvtidcnt; i++)
2000 		/* use func pointer because could be one of two funcs */
2001 		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EXPECTED,
2002 				  tidinv);
2003 
2004 	tidbase = (u64 __iomem *)
2005 		((char __iomem *)(dd->kregbase) +
2006 		 dd->rcvegrbase +
2007 		 rcd->rcvegr_tid_base * sizeof(*tidbase));
2008 
2009 	for (i = 0; i < rcd->rcvegrcnt; i++)
2010 		/* use func pointer because could be one of two funcs */
2011 		dd->f_put_tid(dd, &tidbase[i], RCVHQ_RCV_TYPE_EAGER,
2012 				  tidinv);
2013 }
2014 
2015 /**
2016  * qib_6120_tidtemplate - setup constants for TID updates
2017  * @dd: the qlogic_ib device
2018  *
2019  * We setup stuff that we use a lot, to avoid calculating each time
2020  */
2021 static void qib_6120_tidtemplate(struct qib_devdata *dd)
2022 {
2023 	u32 egrsize = dd->rcvegrbufsize;
2024 
2025 	/*
2026 	 * For now, we always allocate 4KB buffers (at init) so we can
2027 	 * receive max size packets.  We may want a module parameter to
2028 	 * specify 2KB or 4KB and/or make be per ctxt instead of per device
2029 	 * for those who want to reduce memory footprint.  Note that the
2030 	 * rcvhdrentsize size must be large enough to hold the largest
2031 	 * IB header (currently 96 bytes) that we expect to handle (plus of
2032 	 * course the 2 dwords of RHF).
2033 	 */
2034 	if (egrsize == 2048)
2035 		dd->tidtemplate = 1U << 29;
2036 	else if (egrsize == 4096)
2037 		dd->tidtemplate = 2U << 29;
2038 	dd->tidinvalid = 0;
2039 }
2040 
2041 int __attribute__((weak)) qib_unordered_wc(void)
2042 {
2043 	return 0;
2044 }
2045 
2046 /**
2047  * qib_6120_get_base_info - set chip-specific flags for user code
2048  * @rcd: the qlogic_ib ctxt
2049  * @kbase: qib_base_info pointer
2050  *
2051  * We set the PCIE flag because the lower bandwidth on PCIe vs
2052  * HyperTransport can affect some user packet algorithms.
2053  */
2054 static int qib_6120_get_base_info(struct qib_ctxtdata *rcd,
2055 				  struct qib_base_info *kinfo)
2056 {
2057 	if (qib_unordered_wc())
2058 		kinfo->spi_runtime_flags |= QIB_RUNTIME_FORCE_WC_ORDER;
2059 
2060 	kinfo->spi_runtime_flags |= QIB_RUNTIME_PCIE |
2061 		QIB_RUNTIME_FORCE_PIOAVAIL | QIB_RUNTIME_PIO_REGSWAPPED;
2062 	return 0;
2063 }
2064 
2065 
2066 static struct qib_message_header *
2067 qib_6120_get_msgheader(struct qib_devdata *dd, __le32 *rhf_addr)
2068 {
2069 	return (struct qib_message_header *)
2070 		&rhf_addr[sizeof(u64) / sizeof(u32)];
2071 }
2072 
2073 static void qib_6120_config_ctxts(struct qib_devdata *dd)
2074 {
2075 	dd->ctxtcnt = qib_read_kreg32(dd, kr_portcnt);
2076 	if (qib_n_krcv_queues > 1) {
2077 		dd->first_user_ctxt = qib_n_krcv_queues * dd->num_pports;
2078 		if (dd->first_user_ctxt > dd->ctxtcnt)
2079 			dd->first_user_ctxt = dd->ctxtcnt;
2080 		dd->qpn_mask = dd->first_user_ctxt <= 2 ? 2 : 6;
2081 	} else
2082 		dd->first_user_ctxt = dd->num_pports;
2083 	dd->n_krcv_queues = dd->first_user_ctxt;
2084 }
2085 
2086 static void qib_update_6120_usrhead(struct qib_ctxtdata *rcd, u64 hd,
2087 				    u32 updegr, u32 egrhd, u32 npkts)
2088 {
2089 	if (updegr)
2090 		qib_write_ureg(rcd->dd, ur_rcvegrindexhead, egrhd, rcd->ctxt);
2091 	mmiowb();
2092 	qib_write_ureg(rcd->dd, ur_rcvhdrhead, hd, rcd->ctxt);
2093 	mmiowb();
2094 }
2095 
2096 static u32 qib_6120_hdrqempty(struct qib_ctxtdata *rcd)
2097 {
2098 	u32 head, tail;
2099 
2100 	head = qib_read_ureg32(rcd->dd, ur_rcvhdrhead, rcd->ctxt);
2101 	if (rcd->rcvhdrtail_kvaddr)
2102 		tail = qib_get_rcvhdrtail(rcd);
2103 	else
2104 		tail = qib_read_ureg32(rcd->dd, ur_rcvhdrtail, rcd->ctxt);
2105 	return head == tail;
2106 }
2107 
2108 /*
2109  * Used when we close any ctxt, for DMA already in flight
2110  * at close.  Can't be done until we know hdrq size, so not
2111  * early in chip init.
2112  */
2113 static void alloc_dummy_hdrq(struct qib_devdata *dd)
2114 {
2115 	dd->cspec->dummy_hdrq = dma_alloc_coherent(&dd->pcidev->dev,
2116 					dd->rcd[0]->rcvhdrq_size,
2117 					&dd->cspec->dummy_hdrq_phys,
2118 					GFP_ATOMIC | __GFP_COMP);
2119 	if (!dd->cspec->dummy_hdrq) {
2120 		qib_devinfo(dd->pcidev, "Couldn't allocate dummy hdrq\n");
2121 		/* fallback to just 0'ing */
2122 		dd->cspec->dummy_hdrq_phys = 0UL;
2123 	}
2124 }
2125 
2126 /*
2127  * Modify the RCVCTRL register in chip-specific way. This
2128  * is a function because bit positions and (future) register
2129  * location is chip-specific, but the needed operations are
2130  * generic. <op> is a bit-mask because we often want to
2131  * do multiple modifications.
2132  */
2133 static void rcvctrl_6120_mod(struct qib_pportdata *ppd, unsigned int op,
2134 			     int ctxt)
2135 {
2136 	struct qib_devdata *dd = ppd->dd;
2137 	u64 mask, val;
2138 	unsigned long flags;
2139 
2140 	spin_lock_irqsave(&dd->cspec->rcvmod_lock, flags);
2141 
2142 	if (op & QIB_RCVCTRL_TAILUPD_ENB)
2143 		dd->rcvctrl |= (1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2144 	if (op & QIB_RCVCTRL_TAILUPD_DIS)
2145 		dd->rcvctrl &= ~(1ULL << QLOGIC_IB_R_TAILUPD_SHIFT);
2146 	if (op & QIB_RCVCTRL_PKEY_ENB)
2147 		dd->rcvctrl &= ~(1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2148 	if (op & QIB_RCVCTRL_PKEY_DIS)
2149 		dd->rcvctrl |= (1ULL << IBA6120_R_PKEY_DIS_SHIFT);
2150 	if (ctxt < 0)
2151 		mask = (1ULL << dd->ctxtcnt) - 1;
2152 	else
2153 		mask = (1ULL << ctxt);
2154 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2155 		/* always done for specific ctxt */
2156 		dd->rcvctrl |= (mask << SYM_LSB(RcvCtrl, PortEnable));
2157 		if (!(dd->flags & QIB_NODMA_RTAIL))
2158 			dd->rcvctrl |= 1ULL << QLOGIC_IB_R_TAILUPD_SHIFT;
2159 		/* Write these registers before the context is enabled. */
2160 		qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2161 			dd->rcd[ctxt]->rcvhdrqtailaddr_phys);
2162 		qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2163 			dd->rcd[ctxt]->rcvhdrq_phys);
2164 
2165 		if (ctxt == 0 && !dd->cspec->dummy_hdrq)
2166 			alloc_dummy_hdrq(dd);
2167 	}
2168 	if (op & QIB_RCVCTRL_CTXT_DIS)
2169 		dd->rcvctrl &= ~(mask << SYM_LSB(RcvCtrl, PortEnable));
2170 	if (op & QIB_RCVCTRL_INTRAVAIL_ENB)
2171 		dd->rcvctrl |= (mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2172 	if (op & QIB_RCVCTRL_INTRAVAIL_DIS)
2173 		dd->rcvctrl &= ~(mask << QLOGIC_IB_R_INTRAVAIL_SHIFT);
2174 	qib_write_kreg(dd, kr_rcvctrl, dd->rcvctrl);
2175 	if ((op & QIB_RCVCTRL_INTRAVAIL_ENB) && dd->rhdrhead_intr_off) {
2176 		/* arm rcv interrupt */
2177 		val = qib_read_ureg32(dd, ur_rcvhdrhead, ctxt) |
2178 			dd->rhdrhead_intr_off;
2179 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2180 	}
2181 	if (op & QIB_RCVCTRL_CTXT_ENB) {
2182 		/*
2183 		 * Init the context registers also; if we were
2184 		 * disabled, tail and head should both be zero
2185 		 * already from the enable, but since we don't
2186 		 * know, we have to do it explicitly.
2187 		 */
2188 		val = qib_read_ureg32(dd, ur_rcvegrindextail, ctxt);
2189 		qib_write_ureg(dd, ur_rcvegrindexhead, val, ctxt);
2190 
2191 		val = qib_read_ureg32(dd, ur_rcvhdrtail, ctxt);
2192 		dd->rcd[ctxt]->head = val;
2193 		/* If kctxt, interrupt on next receive. */
2194 		if (ctxt < dd->first_user_ctxt)
2195 			val |= dd->rhdrhead_intr_off;
2196 		qib_write_ureg(dd, ur_rcvhdrhead, val, ctxt);
2197 	}
2198 	if (op & QIB_RCVCTRL_CTXT_DIS) {
2199 		/*
2200 		 * Be paranoid, and never write 0's to these, just use an
2201 		 * unused page.  Of course,
2202 		 * rcvhdraddr points to a large chunk of memory, so this
2203 		 * could still trash things, but at least it won't trash
2204 		 * page 0, and by disabling the ctxt, it should stop "soon",
2205 		 * even if a packet or two is in already in flight after we
2206 		 * disabled the ctxt.  Only 6120 has this issue.
2207 		 */
2208 		if (ctxt >= 0) {
2209 			qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr, ctxt,
2210 					    dd->cspec->dummy_hdrq_phys);
2211 			qib_write_kreg_ctxt(dd, kr_rcvhdraddr, ctxt,
2212 					    dd->cspec->dummy_hdrq_phys);
2213 		} else {
2214 			unsigned i;
2215 
2216 			for (i = 0; i < dd->cfgctxts; i++) {
2217 				qib_write_kreg_ctxt(dd, kr_rcvhdrtailaddr,
2218 					    i, dd->cspec->dummy_hdrq_phys);
2219 				qib_write_kreg_ctxt(dd, kr_rcvhdraddr,
2220 					    i, dd->cspec->dummy_hdrq_phys);
2221 			}
2222 		}
2223 	}
2224 	spin_unlock_irqrestore(&dd->cspec->rcvmod_lock, flags);
2225 }
2226 
2227 /*
2228  * Modify the SENDCTRL register in chip-specific way. This
2229  * is a function there may be multiple such registers with
2230  * slightly different layouts. Only operations actually used
2231  * are implemented yet.
2232  * Chip requires no back-back sendctrl writes, so write
2233  * scratch register after writing sendctrl
2234  */
2235 static void sendctrl_6120_mod(struct qib_pportdata *ppd, u32 op)
2236 {
2237 	struct qib_devdata *dd = ppd->dd;
2238 	u64 tmp_dd_sendctrl;
2239 	unsigned long flags;
2240 
2241 	spin_lock_irqsave(&dd->sendctrl_lock, flags);
2242 
2243 	/* First the ones that are "sticky", saved in shadow */
2244 	if (op & QIB_SENDCTRL_CLEAR)
2245 		dd->sendctrl = 0;
2246 	if (op & QIB_SENDCTRL_SEND_DIS)
2247 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOEnable);
2248 	else if (op & QIB_SENDCTRL_SEND_ENB)
2249 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOEnable);
2250 	if (op & QIB_SENDCTRL_AVAIL_DIS)
2251 		dd->sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2252 	else if (op & QIB_SENDCTRL_AVAIL_ENB)
2253 		dd->sendctrl |= SYM_MASK(SendCtrl, PIOBufAvailUpd);
2254 
2255 	if (op & QIB_SENDCTRL_DISARM_ALL) {
2256 		u32 i, last;
2257 
2258 		tmp_dd_sendctrl = dd->sendctrl;
2259 		/*
2260 		 * disarm any that are not yet launched, disabling sends
2261 		 * and updates until done.
2262 		 */
2263 		last = dd->piobcnt2k + dd->piobcnt4k;
2264 		tmp_dd_sendctrl &=
2265 			~(SYM_MASK(SendCtrl, PIOEnable) |
2266 			  SYM_MASK(SendCtrl, PIOBufAvailUpd));
2267 		for (i = 0; i < last; i++) {
2268 			qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl |
2269 				       SYM_MASK(SendCtrl, Disarm) | i);
2270 			qib_write_kreg(dd, kr_scratch, 0);
2271 		}
2272 	}
2273 
2274 	tmp_dd_sendctrl = dd->sendctrl;
2275 
2276 	if (op & QIB_SENDCTRL_FLUSH)
2277 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Abort);
2278 	if (op & QIB_SENDCTRL_DISARM)
2279 		tmp_dd_sendctrl |= SYM_MASK(SendCtrl, Disarm) |
2280 			((op & QIB_6120_SendCtrl_DisarmPIOBuf_RMASK) <<
2281 			 SYM_LSB(SendCtrl, DisarmPIOBuf));
2282 	if (op & QIB_SENDCTRL_AVAIL_BLIP)
2283 		tmp_dd_sendctrl &= ~SYM_MASK(SendCtrl, PIOBufAvailUpd);
2284 
2285 	qib_write_kreg(dd, kr_sendctrl, tmp_dd_sendctrl);
2286 	qib_write_kreg(dd, kr_scratch, 0);
2287 
2288 	if (op & QIB_SENDCTRL_AVAIL_BLIP) {
2289 		qib_write_kreg(dd, kr_sendctrl, dd->sendctrl);
2290 		qib_write_kreg(dd, kr_scratch, 0);
2291 	}
2292 
2293 	spin_unlock_irqrestore(&dd->sendctrl_lock, flags);
2294 
2295 	if (op & QIB_SENDCTRL_FLUSH) {
2296 		u32 v;
2297 		/*
2298 		 * ensure writes have hit chip, then do a few
2299 		 * more reads, to allow DMA of pioavail registers
2300 		 * to occur, so in-memory copy is in sync with
2301 		 * the chip.  Not always safe to sleep.
2302 		 */
2303 		v = qib_read_kreg32(dd, kr_scratch);
2304 		qib_write_kreg(dd, kr_scratch, v);
2305 		v = qib_read_kreg32(dd, kr_scratch);
2306 		qib_write_kreg(dd, kr_scratch, v);
2307 		qib_read_kreg32(dd, kr_scratch);
2308 	}
2309 }
2310 
2311 /**
2312  * qib_portcntr_6120 - read a per-port counter
2313  * @dd: the qlogic_ib device
2314  * @creg: the counter to snapshot
2315  */
2316 static u64 qib_portcntr_6120(struct qib_pportdata *ppd, u32 reg)
2317 {
2318 	u64 ret = 0ULL;
2319 	struct qib_devdata *dd = ppd->dd;
2320 	u16 creg;
2321 	/* 0xffff for unimplemented or synthesized counters */
2322 	static const u16 xlator[] = {
2323 		[QIBPORTCNTR_PKTSEND] = cr_pktsend,
2324 		[QIBPORTCNTR_WORDSEND] = cr_wordsend,
2325 		[QIBPORTCNTR_PSXMITDATA] = 0xffff,
2326 		[QIBPORTCNTR_PSXMITPKTS] = 0xffff,
2327 		[QIBPORTCNTR_PSXMITWAIT] = 0xffff,
2328 		[QIBPORTCNTR_SENDSTALL] = cr_sendstall,
2329 		[QIBPORTCNTR_PKTRCV] = cr_pktrcv,
2330 		[QIBPORTCNTR_PSRCVDATA] = 0xffff,
2331 		[QIBPORTCNTR_PSRCVPKTS] = 0xffff,
2332 		[QIBPORTCNTR_RCVEBP] = cr_rcvebp,
2333 		[QIBPORTCNTR_RCVOVFL] = cr_rcvovfl,
2334 		[QIBPORTCNTR_WORDRCV] = cr_wordrcv,
2335 		[QIBPORTCNTR_RXDROPPKT] = cr_rxdroppkt,
2336 		[QIBPORTCNTR_RXLOCALPHYERR] = 0xffff,
2337 		[QIBPORTCNTR_RXVLERR] = 0xffff,
2338 		[QIBPORTCNTR_ERRICRC] = cr_erricrc,
2339 		[QIBPORTCNTR_ERRVCRC] = cr_errvcrc,
2340 		[QIBPORTCNTR_ERRLPCRC] = cr_errlpcrc,
2341 		[QIBPORTCNTR_BADFORMAT] = cr_badformat,
2342 		[QIBPORTCNTR_ERR_RLEN] = cr_err_rlen,
2343 		[QIBPORTCNTR_IBSYMBOLERR] = cr_ibsymbolerr,
2344 		[QIBPORTCNTR_INVALIDRLEN] = cr_invalidrlen,
2345 		[QIBPORTCNTR_UNSUPVL] = cr_txunsupvl,
2346 		[QIBPORTCNTR_EXCESSBUFOVFL] = 0xffff,
2347 		[QIBPORTCNTR_ERRLINK] = cr_errlink,
2348 		[QIBPORTCNTR_IBLINKDOWN] = cr_iblinkdown,
2349 		[QIBPORTCNTR_IBLINKERRRECOV] = cr_iblinkerrrecov,
2350 		[QIBPORTCNTR_LLI] = 0xffff,
2351 		[QIBPORTCNTR_PSINTERVAL] = 0xffff,
2352 		[QIBPORTCNTR_PSSTART] = 0xffff,
2353 		[QIBPORTCNTR_PSSTAT] = 0xffff,
2354 		[QIBPORTCNTR_VL15PKTDROP] = 0xffff,
2355 		[QIBPORTCNTR_ERRPKEY] = cr_errpkey,
2356 		[QIBPORTCNTR_KHDROVFL] = 0xffff,
2357 	};
2358 
2359 	if (reg >= ARRAY_SIZE(xlator)) {
2360 		qib_devinfo(ppd->dd->pcidev,
2361 			 "Unimplemented portcounter %u\n", reg);
2362 		goto done;
2363 	}
2364 	creg = xlator[reg];
2365 
2366 	/* handle counters requests not implemented as chip counters */
2367 	if (reg == QIBPORTCNTR_LLI)
2368 		ret = dd->cspec->lli_errs;
2369 	else if (reg == QIBPORTCNTR_EXCESSBUFOVFL)
2370 		ret = dd->cspec->overrun_thresh_errs;
2371 	else if (reg == QIBPORTCNTR_KHDROVFL) {
2372 		int i;
2373 
2374 		/* sum over all kernel contexts */
2375 		for (i = 0; i < dd->first_user_ctxt; i++)
2376 			ret += read_6120_creg32(dd, cr_portovfl + i);
2377 	} else if (reg == QIBPORTCNTR_PSSTAT)
2378 		ret = dd->cspec->pma_sample_status;
2379 	if (creg == 0xffff)
2380 		goto done;
2381 
2382 	/*
2383 	 * only fast incrementing counters are 64bit; use 32 bit reads to
2384 	 * avoid two independent reads when on opteron
2385 	 */
2386 	if (creg == cr_wordsend || creg == cr_wordrcv ||
2387 	    creg == cr_pktsend || creg == cr_pktrcv)
2388 		ret = read_6120_creg(dd, creg);
2389 	else
2390 		ret = read_6120_creg32(dd, creg);
2391 	if (creg == cr_ibsymbolerr) {
2392 		if (dd->cspec->ibdeltainprog)
2393 			ret -= ret - dd->cspec->ibsymsnap;
2394 		ret -= dd->cspec->ibsymdelta;
2395 	} else if (creg == cr_iblinkerrrecov) {
2396 		if (dd->cspec->ibdeltainprog)
2397 			ret -= ret - dd->cspec->iblnkerrsnap;
2398 		ret -= dd->cspec->iblnkerrdelta;
2399 	}
2400 	if (reg == QIBPORTCNTR_RXDROPPKT) /* add special cased count */
2401 		ret += dd->cspec->rxfc_unsupvl_errs;
2402 
2403 done:
2404 	return ret;
2405 }
2406 
2407 /*
2408  * Device counter names (not port-specific), one line per stat,
2409  * single string.  Used by utilities like ipathstats to print the stats
2410  * in a way which works for different versions of drivers, without changing
2411  * the utility.  Names need to be 12 chars or less (w/o newline), for proper
2412  * display by utility.
2413  * Non-error counters are first.
2414  * Start of "error" conters is indicated by a leading "E " on the first
2415  * "error" counter, and doesn't count in label length.
2416  * The EgrOvfl list needs to be last so we truncate them at the configured
2417  * context count for the device.
2418  * cntr6120indices contains the corresponding register indices.
2419  */
2420 static const char cntr6120names[] =
2421 	"Interrupts\n"
2422 	"HostBusStall\n"
2423 	"E RxTIDFull\n"
2424 	"RxTIDInvalid\n"
2425 	"Ctxt0EgrOvfl\n"
2426 	"Ctxt1EgrOvfl\n"
2427 	"Ctxt2EgrOvfl\n"
2428 	"Ctxt3EgrOvfl\n"
2429 	"Ctxt4EgrOvfl\n";
2430 
2431 static const size_t cntr6120indices[] = {
2432 	cr_lbint,
2433 	cr_lbflowstall,
2434 	cr_errtidfull,
2435 	cr_errtidvalid,
2436 	cr_portovfl + 0,
2437 	cr_portovfl + 1,
2438 	cr_portovfl + 2,
2439 	cr_portovfl + 3,
2440 	cr_portovfl + 4,
2441 };
2442 
2443 /*
2444  * same as cntr6120names and cntr6120indices, but for port-specific counters.
2445  * portcntr6120indices is somewhat complicated by some registers needing
2446  * adjustments of various kinds, and those are ORed with _PORT_VIRT_FLAG
2447  */
2448 static const char portcntr6120names[] =
2449 	"TxPkt\n"
2450 	"TxFlowPkt\n"
2451 	"TxWords\n"
2452 	"RxPkt\n"
2453 	"RxFlowPkt\n"
2454 	"RxWords\n"
2455 	"TxFlowStall\n"
2456 	"E IBStatusChng\n"
2457 	"IBLinkDown\n"
2458 	"IBLnkRecov\n"
2459 	"IBRxLinkErr\n"
2460 	"IBSymbolErr\n"
2461 	"RxLLIErr\n"
2462 	"RxBadFormat\n"
2463 	"RxBadLen\n"
2464 	"RxBufOvrfl\n"
2465 	"RxEBP\n"
2466 	"RxFlowCtlErr\n"
2467 	"RxICRCerr\n"
2468 	"RxLPCRCerr\n"
2469 	"RxVCRCerr\n"
2470 	"RxInvalLen\n"
2471 	"RxInvalPKey\n"
2472 	"RxPktDropped\n"
2473 	"TxBadLength\n"
2474 	"TxDropped\n"
2475 	"TxInvalLen\n"
2476 	"TxUnderrun\n"
2477 	"TxUnsupVL\n"
2478 	;
2479 
2480 #define _PORT_VIRT_FLAG 0x8000 /* "virtual", need adjustments */
2481 static const size_t portcntr6120indices[] = {
2482 	QIBPORTCNTR_PKTSEND | _PORT_VIRT_FLAG,
2483 	cr_pktsendflow,
2484 	QIBPORTCNTR_WORDSEND | _PORT_VIRT_FLAG,
2485 	QIBPORTCNTR_PKTRCV | _PORT_VIRT_FLAG,
2486 	cr_pktrcvflowctrl,
2487 	QIBPORTCNTR_WORDRCV | _PORT_VIRT_FLAG,
2488 	QIBPORTCNTR_SENDSTALL | _PORT_VIRT_FLAG,
2489 	cr_ibstatuschange,
2490 	QIBPORTCNTR_IBLINKDOWN | _PORT_VIRT_FLAG,
2491 	QIBPORTCNTR_IBLINKERRRECOV | _PORT_VIRT_FLAG,
2492 	QIBPORTCNTR_ERRLINK | _PORT_VIRT_FLAG,
2493 	QIBPORTCNTR_IBSYMBOLERR | _PORT_VIRT_FLAG,
2494 	QIBPORTCNTR_LLI | _PORT_VIRT_FLAG,
2495 	QIBPORTCNTR_BADFORMAT | _PORT_VIRT_FLAG,
2496 	QIBPORTCNTR_ERR_RLEN | _PORT_VIRT_FLAG,
2497 	QIBPORTCNTR_RCVOVFL | _PORT_VIRT_FLAG,
2498 	QIBPORTCNTR_RCVEBP | _PORT_VIRT_FLAG,
2499 	cr_rcvflowctrl_err,
2500 	QIBPORTCNTR_ERRICRC | _PORT_VIRT_FLAG,
2501 	QIBPORTCNTR_ERRLPCRC | _PORT_VIRT_FLAG,
2502 	QIBPORTCNTR_ERRVCRC | _PORT_VIRT_FLAG,
2503 	QIBPORTCNTR_INVALIDRLEN | _PORT_VIRT_FLAG,
2504 	QIBPORTCNTR_ERRPKEY | _PORT_VIRT_FLAG,
2505 	QIBPORTCNTR_RXDROPPKT | _PORT_VIRT_FLAG,
2506 	cr_invalidslen,
2507 	cr_senddropped,
2508 	cr_errslen,
2509 	cr_sendunderrun,
2510 	cr_txunsupvl,
2511 };
2512 
2513 /* do all the setup to make the counter reads efficient later */
2514 static void init_6120_cntrnames(struct qib_devdata *dd)
2515 {
2516 	int i, j = 0;
2517 	char *s;
2518 
2519 	for (i = 0, s = (char *)cntr6120names; s && j <= dd->cfgctxts;
2520 	     i++) {
2521 		/* we always have at least one counter before the egrovfl */
2522 		if (!j && !strncmp("Ctxt0EgrOvfl", s + 1, 12))
2523 			j = 1;
2524 		s = strchr(s + 1, '\n');
2525 		if (s && j)
2526 			j++;
2527 	}
2528 	dd->cspec->ncntrs = i;
2529 	if (!s)
2530 		/* full list; size is without terminating null */
2531 		dd->cspec->cntrnamelen = sizeof(cntr6120names) - 1;
2532 	else
2533 		dd->cspec->cntrnamelen = 1 + s - cntr6120names;
2534 	dd->cspec->cntrs = kmalloc(dd->cspec->ncntrs
2535 		* sizeof(u64), GFP_KERNEL);
2536 	if (!dd->cspec->cntrs)
2537 		qib_dev_err(dd, "Failed allocation for counters\n");
2538 
2539 	for (i = 0, s = (char *)portcntr6120names; s; i++)
2540 		s = strchr(s + 1, '\n');
2541 	dd->cspec->nportcntrs = i - 1;
2542 	dd->cspec->portcntrnamelen = sizeof(portcntr6120names) - 1;
2543 	dd->cspec->portcntrs = kmalloc(dd->cspec->nportcntrs
2544 		* sizeof(u64), GFP_KERNEL);
2545 	if (!dd->cspec->portcntrs)
2546 		qib_dev_err(dd, "Failed allocation for portcounters\n");
2547 }
2548 
2549 static u32 qib_read_6120cntrs(struct qib_devdata *dd, loff_t pos, char **namep,
2550 			      u64 **cntrp)
2551 {
2552 	u32 ret;
2553 
2554 	if (namep) {
2555 		ret = dd->cspec->cntrnamelen;
2556 		if (pos >= ret)
2557 			ret = 0; /* final read after getting everything */
2558 		else
2559 			*namep = (char *)cntr6120names;
2560 	} else {
2561 		u64 *cntr = dd->cspec->cntrs;
2562 		int i;
2563 
2564 		ret = dd->cspec->ncntrs * sizeof(u64);
2565 		if (!cntr || pos >= ret) {
2566 			/* everything read, or couldn't get memory */
2567 			ret = 0;
2568 			goto done;
2569 		}
2570 		if (pos >= ret) {
2571 			ret = 0; /* final read after getting everything */
2572 			goto done;
2573 		}
2574 		*cntrp = cntr;
2575 		for (i = 0; i < dd->cspec->ncntrs; i++)
2576 			*cntr++ = read_6120_creg32(dd, cntr6120indices[i]);
2577 	}
2578 done:
2579 	return ret;
2580 }
2581 
2582 static u32 qib_read_6120portcntrs(struct qib_devdata *dd, loff_t pos, u32 port,
2583 				  char **namep, u64 **cntrp)
2584 {
2585 	u32 ret;
2586 
2587 	if (namep) {
2588 		ret = dd->cspec->portcntrnamelen;
2589 		if (pos >= ret)
2590 			ret = 0; /* final read after getting everything */
2591 		else
2592 			*namep = (char *)portcntr6120names;
2593 	} else {
2594 		u64 *cntr = dd->cspec->portcntrs;
2595 		struct qib_pportdata *ppd = &dd->pport[port];
2596 		int i;
2597 
2598 		ret = dd->cspec->nportcntrs * sizeof(u64);
2599 		if (!cntr || pos >= ret) {
2600 			/* everything read, or couldn't get memory */
2601 			ret = 0;
2602 			goto done;
2603 		}
2604 		*cntrp = cntr;
2605 		for (i = 0; i < dd->cspec->nportcntrs; i++) {
2606 			if (portcntr6120indices[i] & _PORT_VIRT_FLAG)
2607 				*cntr++ = qib_portcntr_6120(ppd,
2608 					portcntr6120indices[i] &
2609 					~_PORT_VIRT_FLAG);
2610 			else
2611 				*cntr++ = read_6120_creg32(dd,
2612 					   portcntr6120indices[i]);
2613 		}
2614 	}
2615 done:
2616 	return ret;
2617 }
2618 
2619 static void qib_chk_6120_errormask(struct qib_devdata *dd)
2620 {
2621 	static u32 fixed;
2622 	u32 ctrl;
2623 	unsigned long errormask;
2624 	unsigned long hwerrs;
2625 
2626 	if (!dd->cspec->errormask || !(dd->flags & QIB_INITTED))
2627 		return;
2628 
2629 	errormask = qib_read_kreg64(dd, kr_errmask);
2630 
2631 	if (errormask == dd->cspec->errormask)
2632 		return;
2633 	fixed++;
2634 
2635 	hwerrs = qib_read_kreg64(dd, kr_hwerrstatus);
2636 	ctrl = qib_read_kreg32(dd, kr_control);
2637 
2638 	qib_write_kreg(dd, kr_errmask,
2639 		dd->cspec->errormask);
2640 
2641 	if ((hwerrs & dd->cspec->hwerrmask) ||
2642 	    (ctrl & QLOGIC_IB_C_FREEZEMODE)) {
2643 		qib_write_kreg(dd, kr_hwerrclear, 0ULL);
2644 		qib_write_kreg(dd, kr_errclear, 0ULL);
2645 		/* force re-interrupt of pending events, just in case */
2646 		qib_write_kreg(dd, kr_intclear, 0ULL);
2647 		qib_devinfo(dd->pcidev,
2648 			 "errormask fixed(%u) %lx->%lx, ctrl %x hwerr %lx\n",
2649 			 fixed, errormask, (unsigned long)dd->cspec->errormask,
2650 			 ctrl, hwerrs);
2651 	}
2652 }
2653 
2654 /**
2655  * qib_get_faststats - get word counters from chip before they overflow
2656  * @opaque - contains a pointer to the qlogic_ib device qib_devdata
2657  *
2658  * This needs more work; in particular, decision on whether we really
2659  * need traffic_wds done the way it is
2660  * called from add_timer
2661  */
2662 static void qib_get_6120_faststats(unsigned long opaque)
2663 {
2664 	struct qib_devdata *dd = (struct qib_devdata *) opaque;
2665 	struct qib_pportdata *ppd = dd->pport;
2666 	unsigned long flags;
2667 	u64 traffic_wds;
2668 
2669 	/*
2670 	 * don't access the chip while running diags, or memory diags can
2671 	 * fail
2672 	 */
2673 	if (!(dd->flags & QIB_INITTED) || dd->diag_client)
2674 		/* but re-arm the timer, for diags case; won't hurt other */
2675 		goto done;
2676 
2677 	/*
2678 	 * We now try to maintain an activity timer, based on traffic
2679 	 * exceeding a threshold, so we need to check the word-counts
2680 	 * even if they are 64-bit.
2681 	 */
2682 	traffic_wds = qib_portcntr_6120(ppd, cr_wordsend) +
2683 		qib_portcntr_6120(ppd, cr_wordrcv);
2684 	spin_lock_irqsave(&dd->eep_st_lock, flags);
2685 	traffic_wds -= dd->traffic_wds;
2686 	dd->traffic_wds += traffic_wds;
2687 	spin_unlock_irqrestore(&dd->eep_st_lock, flags);
2688 
2689 	qib_chk_6120_errormask(dd);
2690 done:
2691 	mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
2692 }
2693 
2694 /* no interrupt fallback for these chips */
2695 static int qib_6120_nointr_fallback(struct qib_devdata *dd)
2696 {
2697 	return 0;
2698 }
2699 
2700 /*
2701  * reset the XGXS (between serdes and IBC).  Slightly less intrusive
2702  * than resetting the IBC or external link state, and useful in some
2703  * cases to cause some retraining.  To do this right, we reset IBC
2704  * as well.
2705  */
2706 static void qib_6120_xgxs_reset(struct qib_pportdata *ppd)
2707 {
2708 	u64 val, prev_val;
2709 	struct qib_devdata *dd = ppd->dd;
2710 
2711 	prev_val = qib_read_kreg64(dd, kr_xgxs_cfg);
2712 	val = prev_val | QLOGIC_IB_XGXS_RESET;
2713 	prev_val &= ~QLOGIC_IB_XGXS_RESET; /* be sure */
2714 	qib_write_kreg(dd, kr_control,
2715 		       dd->control & ~QLOGIC_IB_C_LINKENABLE);
2716 	qib_write_kreg(dd, kr_xgxs_cfg, val);
2717 	qib_read_kreg32(dd, kr_scratch);
2718 	qib_write_kreg(dd, kr_xgxs_cfg, prev_val);
2719 	qib_write_kreg(dd, kr_control, dd->control);
2720 }
2721 
2722 static int qib_6120_get_ib_cfg(struct qib_pportdata *ppd, int which)
2723 {
2724 	int ret;
2725 
2726 	switch (which) {
2727 	case QIB_IB_CFG_LWID:
2728 		ret = ppd->link_width_active;
2729 		break;
2730 
2731 	case QIB_IB_CFG_SPD:
2732 		ret = ppd->link_speed_active;
2733 		break;
2734 
2735 	case QIB_IB_CFG_LWID_ENB:
2736 		ret = ppd->link_width_enabled;
2737 		break;
2738 
2739 	case QIB_IB_CFG_SPD_ENB:
2740 		ret = ppd->link_speed_enabled;
2741 		break;
2742 
2743 	case QIB_IB_CFG_OP_VLS:
2744 		ret = ppd->vls_operational;
2745 		break;
2746 
2747 	case QIB_IB_CFG_VL_HIGH_CAP:
2748 		ret = 0;
2749 		break;
2750 
2751 	case QIB_IB_CFG_VL_LOW_CAP:
2752 		ret = 0;
2753 		break;
2754 
2755 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2756 		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2757 				OverrunThreshold);
2758 		break;
2759 
2760 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2761 		ret = SYM_FIELD(ppd->dd->cspec->ibcctrl, IBCCtrl,
2762 				PhyerrThreshold);
2763 		break;
2764 
2765 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2766 		/* will only take effect when the link state changes */
2767 		ret = (ppd->dd->cspec->ibcctrl &
2768 		       SYM_MASK(IBCCtrl, LinkDownDefaultState)) ?
2769 			IB_LINKINITCMD_SLEEP : IB_LINKINITCMD_POLL;
2770 		break;
2771 
2772 	case QIB_IB_CFG_HRTBT: /* Get Heartbeat off/enable/auto */
2773 		ret = 0; /* no heartbeat on this chip */
2774 		break;
2775 
2776 	case QIB_IB_CFG_PMA_TICKS:
2777 		ret = 250; /* 1 usec. */
2778 		break;
2779 
2780 	default:
2781 		ret =  -EINVAL;
2782 		break;
2783 	}
2784 	return ret;
2785 }
2786 
2787 /*
2788  * We assume range checking is already done, if needed.
2789  */
2790 static int qib_6120_set_ib_cfg(struct qib_pportdata *ppd, int which, u32 val)
2791 {
2792 	struct qib_devdata *dd = ppd->dd;
2793 	int ret = 0;
2794 	u64 val64;
2795 	u16 lcmd, licmd;
2796 
2797 	switch (which) {
2798 	case QIB_IB_CFG_LWID_ENB:
2799 		ppd->link_width_enabled = val;
2800 		break;
2801 
2802 	case QIB_IB_CFG_SPD_ENB:
2803 		ppd->link_speed_enabled = val;
2804 		break;
2805 
2806 	case QIB_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */
2807 		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2808 				  OverrunThreshold);
2809 		if (val64 != val) {
2810 			dd->cspec->ibcctrl &=
2811 				~SYM_MASK(IBCCtrl, OverrunThreshold);
2812 			dd->cspec->ibcctrl |= (u64) val <<
2813 				SYM_LSB(IBCCtrl, OverrunThreshold);
2814 			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2815 			qib_write_kreg(dd, kr_scratch, 0);
2816 		}
2817 		break;
2818 
2819 	case QIB_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */
2820 		val64 = SYM_FIELD(dd->cspec->ibcctrl, IBCCtrl,
2821 				  PhyerrThreshold);
2822 		if (val64 != val) {
2823 			dd->cspec->ibcctrl &=
2824 				~SYM_MASK(IBCCtrl, PhyerrThreshold);
2825 			dd->cspec->ibcctrl |= (u64) val <<
2826 				SYM_LSB(IBCCtrl, PhyerrThreshold);
2827 			qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2828 			qib_write_kreg(dd, kr_scratch, 0);
2829 		}
2830 		break;
2831 
2832 	case QIB_IB_CFG_PKEYS: /* update pkeys */
2833 		val64 = (u64) ppd->pkeys[0] | ((u64) ppd->pkeys[1] << 16) |
2834 			((u64) ppd->pkeys[2] << 32) |
2835 			((u64) ppd->pkeys[3] << 48);
2836 		qib_write_kreg(dd, kr_partitionkey, val64);
2837 		break;
2838 
2839 	case QIB_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */
2840 		/* will only take effect when the link state changes */
2841 		if (val == IB_LINKINITCMD_POLL)
2842 			dd->cspec->ibcctrl &=
2843 				~SYM_MASK(IBCCtrl, LinkDownDefaultState);
2844 		else /* SLEEP */
2845 			dd->cspec->ibcctrl |=
2846 				SYM_MASK(IBCCtrl, LinkDownDefaultState);
2847 		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2848 		qib_write_kreg(dd, kr_scratch, 0);
2849 		break;
2850 
2851 	case QIB_IB_CFG_MTU: /* update the MTU in IBC */
2852 		/*
2853 		 * Update our housekeeping variables, and set IBC max
2854 		 * size, same as init code; max IBC is max we allow in
2855 		 * buffer, less the qword pbc, plus 1 for ICRC, in dwords
2856 		 * Set even if it's unchanged, print debug message only
2857 		 * on changes.
2858 		 */
2859 		val = (ppd->ibmaxlen >> 2) + 1;
2860 		dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, MaxPktLen);
2861 		dd->cspec->ibcctrl |= (u64)val <<
2862 			SYM_LSB(IBCCtrl, MaxPktLen);
2863 		qib_write_kreg(dd, kr_ibcctrl, dd->cspec->ibcctrl);
2864 		qib_write_kreg(dd, kr_scratch, 0);
2865 		break;
2866 
2867 	case QIB_IB_CFG_LSTATE: /* set the IB link state */
2868 		switch (val & 0xffff0000) {
2869 		case IB_LINKCMD_DOWN:
2870 			lcmd = QLOGIC_IB_IBCC_LINKCMD_DOWN;
2871 			if (!dd->cspec->ibdeltainprog) {
2872 				dd->cspec->ibdeltainprog = 1;
2873 				dd->cspec->ibsymsnap =
2874 					read_6120_creg32(dd, cr_ibsymbolerr);
2875 				dd->cspec->iblnkerrsnap =
2876 					read_6120_creg32(dd, cr_iblinkerrrecov);
2877 			}
2878 			break;
2879 
2880 		case IB_LINKCMD_ARMED:
2881 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ARMED;
2882 			break;
2883 
2884 		case IB_LINKCMD_ACTIVE:
2885 			lcmd = QLOGIC_IB_IBCC_LINKCMD_ACTIVE;
2886 			break;
2887 
2888 		default:
2889 			ret = -EINVAL;
2890 			qib_dev_err(dd, "bad linkcmd req 0x%x\n", val >> 16);
2891 			goto bail;
2892 		}
2893 		switch (val & 0xffff) {
2894 		case IB_LINKINITCMD_NOP:
2895 			licmd = 0;
2896 			break;
2897 
2898 		case IB_LINKINITCMD_POLL:
2899 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_POLL;
2900 			break;
2901 
2902 		case IB_LINKINITCMD_SLEEP:
2903 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_SLEEP;
2904 			break;
2905 
2906 		case IB_LINKINITCMD_DISABLE:
2907 			licmd = QLOGIC_IB_IBCC_LINKINITCMD_DISABLE;
2908 			break;
2909 
2910 		default:
2911 			ret = -EINVAL;
2912 			qib_dev_err(dd, "bad linkinitcmd req 0x%x\n",
2913 				    val & 0xffff);
2914 			goto bail;
2915 		}
2916 		qib_set_ib_6120_lstate(ppd, lcmd, licmd);
2917 		goto bail;
2918 
2919 	case QIB_IB_CFG_HRTBT:
2920 		ret = -EINVAL;
2921 		break;
2922 
2923 	default:
2924 		ret = -EINVAL;
2925 	}
2926 bail:
2927 	return ret;
2928 }
2929 
2930 static int qib_6120_set_loopback(struct qib_pportdata *ppd, const char *what)
2931 {
2932 	int ret = 0;
2933 
2934 	if (!strncmp(what, "ibc", 3)) {
2935 		ppd->dd->cspec->ibcctrl |= SYM_MASK(IBCCtrl, Loopback);
2936 		qib_devinfo(ppd->dd->pcidev, "Enabling IB%u:%u IBC loopback\n",
2937 			 ppd->dd->unit, ppd->port);
2938 	} else if (!strncmp(what, "off", 3)) {
2939 		ppd->dd->cspec->ibcctrl &= ~SYM_MASK(IBCCtrl, Loopback);
2940 		qib_devinfo(ppd->dd->pcidev,
2941 			"Disabling IB%u:%u IBC loopback (normal)\n",
2942 			ppd->dd->unit, ppd->port);
2943 	} else
2944 		ret = -EINVAL;
2945 	if (!ret) {
2946 		qib_write_kreg(ppd->dd, kr_ibcctrl, ppd->dd->cspec->ibcctrl);
2947 		qib_write_kreg(ppd->dd, kr_scratch, 0);
2948 	}
2949 	return ret;
2950 }
2951 
2952 static void pma_6120_timer(unsigned long data)
2953 {
2954 	struct qib_pportdata *ppd = (struct qib_pportdata *)data;
2955 	struct qib_chip_specific *cs = ppd->dd->cspec;
2956 	struct qib_ibport *ibp = &ppd->ibport_data;
2957 	unsigned long flags;
2958 
2959 	spin_lock_irqsave(&ibp->rvp.lock, flags);
2960 	if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED) {
2961 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2962 		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2963 				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2964 		mod_timer(&cs->pma_timer,
2965 		      jiffies + usecs_to_jiffies(ibp->rvp.pma_sample_interval));
2966 	} else if (cs->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
2967 		u64 ta, tb, tc, td, te;
2968 
2969 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2970 		qib_snapshot_counters(ppd, &ta, &tb, &tc, &td, &te);
2971 
2972 		cs->sword = ta - cs->sword;
2973 		cs->rword = tb - cs->rword;
2974 		cs->spkts = tc - cs->spkts;
2975 		cs->rpkts = td - cs->rpkts;
2976 		cs->xmit_wait = te - cs->xmit_wait;
2977 	}
2978 	spin_unlock_irqrestore(&ibp->rvp.lock, flags);
2979 }
2980 
2981 /*
2982  * Note that the caller has the ibp->rvp.lock held.
2983  */
2984 static void qib_set_cntr_6120_sample(struct qib_pportdata *ppd, u32 intv,
2985 				     u32 start)
2986 {
2987 	struct qib_chip_specific *cs = ppd->dd->cspec;
2988 
2989 	if (start && intv) {
2990 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_STARTED;
2991 		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(start));
2992 	} else if (intv) {
2993 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
2994 		qib_snapshot_counters(ppd, &cs->sword, &cs->rword,
2995 				      &cs->spkts, &cs->rpkts, &cs->xmit_wait);
2996 		mod_timer(&cs->pma_timer, jiffies + usecs_to_jiffies(intv));
2997 	} else {
2998 		cs->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
2999 		cs->sword = 0;
3000 		cs->rword = 0;
3001 		cs->spkts = 0;
3002 		cs->rpkts = 0;
3003 		cs->xmit_wait = 0;
3004 	}
3005 }
3006 
3007 static u32 qib_6120_iblink_state(u64 ibcs)
3008 {
3009 	u32 state = (u32)SYM_FIELD(ibcs, IBCStatus, LinkState);
3010 
3011 	switch (state) {
3012 	case IB_6120_L_STATE_INIT:
3013 		state = IB_PORT_INIT;
3014 		break;
3015 	case IB_6120_L_STATE_ARM:
3016 		state = IB_PORT_ARMED;
3017 		break;
3018 	case IB_6120_L_STATE_ACTIVE:
3019 		/* fall through */
3020 	case IB_6120_L_STATE_ACT_DEFER:
3021 		state = IB_PORT_ACTIVE;
3022 		break;
3023 	default: /* fall through */
3024 	case IB_6120_L_STATE_DOWN:
3025 		state = IB_PORT_DOWN;
3026 		break;
3027 	}
3028 	return state;
3029 }
3030 
3031 /* returns the IBTA port state, rather than the IBC link training state */
3032 static u8 qib_6120_phys_portstate(u64 ibcs)
3033 {
3034 	u8 state = (u8)SYM_FIELD(ibcs, IBCStatus, LinkTrainingState);
3035 	return qib_6120_physportstate[state];
3036 }
3037 
3038 static int qib_6120_ib_updown(struct qib_pportdata *ppd, int ibup, u64 ibcs)
3039 {
3040 	unsigned long flags;
3041 
3042 	spin_lock_irqsave(&ppd->lflags_lock, flags);
3043 	ppd->lflags &= ~QIBL_IB_FORCE_NOTIFY;
3044 	spin_unlock_irqrestore(&ppd->lflags_lock, flags);
3045 
3046 	if (ibup) {
3047 		if (ppd->dd->cspec->ibdeltainprog) {
3048 			ppd->dd->cspec->ibdeltainprog = 0;
3049 			ppd->dd->cspec->ibsymdelta +=
3050 				read_6120_creg32(ppd->dd, cr_ibsymbolerr) -
3051 					ppd->dd->cspec->ibsymsnap;
3052 			ppd->dd->cspec->iblnkerrdelta +=
3053 				read_6120_creg32(ppd->dd, cr_iblinkerrrecov) -
3054 					ppd->dd->cspec->iblnkerrsnap;
3055 		}
3056 		qib_hol_init(ppd);
3057 	} else {
3058 		ppd->dd->cspec->lli_counter = 0;
3059 		if (!ppd->dd->cspec->ibdeltainprog) {
3060 			ppd->dd->cspec->ibdeltainprog = 1;
3061 			ppd->dd->cspec->ibsymsnap =
3062 				read_6120_creg32(ppd->dd, cr_ibsymbolerr);
3063 			ppd->dd->cspec->iblnkerrsnap =
3064 				read_6120_creg32(ppd->dd, cr_iblinkerrrecov);
3065 		}
3066 		qib_hol_down(ppd);
3067 	}
3068 
3069 	qib_6120_setup_setextled(ppd, ibup);
3070 
3071 	return 0;
3072 }
3073 
3074 /* Does read/modify/write to appropriate registers to
3075  * set output and direction bits selected by mask.
3076  * these are in their canonical postions (e.g. lsb of
3077  * dir will end up in D48 of extctrl on existing chips).
3078  * returns contents of GP Inputs.
3079  */
3080 static int gpio_6120_mod(struct qib_devdata *dd, u32 out, u32 dir, u32 mask)
3081 {
3082 	u64 read_val, new_out;
3083 	unsigned long flags;
3084 
3085 	if (mask) {
3086 		/* some bits being written, lock access to GPIO */
3087 		dir &= mask;
3088 		out &= mask;
3089 		spin_lock_irqsave(&dd->cspec->gpio_lock, flags);
3090 		dd->cspec->extctrl &= ~((u64)mask << SYM_LSB(EXTCtrl, GPIOOe));
3091 		dd->cspec->extctrl |= ((u64) dir << SYM_LSB(EXTCtrl, GPIOOe));
3092 		new_out = (dd->cspec->gpio_out & ~mask) | out;
3093 
3094 		qib_write_kreg(dd, kr_extctrl, dd->cspec->extctrl);
3095 		qib_write_kreg(dd, kr_gpio_out, new_out);
3096 		dd->cspec->gpio_out = new_out;
3097 		spin_unlock_irqrestore(&dd->cspec->gpio_lock, flags);
3098 	}
3099 	/*
3100 	 * It is unlikely that a read at this time would get valid
3101 	 * data on a pin whose direction line was set in the same
3102 	 * call to this function. We include the read here because
3103 	 * that allows us to potentially combine a change on one pin with
3104 	 * a read on another, and because the old code did something like
3105 	 * this.
3106 	 */
3107 	read_val = qib_read_kreg64(dd, kr_extstatus);
3108 	return SYM_FIELD(read_val, EXTStatus, GPIOIn);
3109 }
3110 
3111 /*
3112  * Read fundamental info we need to use the chip.  These are
3113  * the registers that describe chip capabilities, and are
3114  * saved in shadow registers.
3115  */
3116 static void get_6120_chip_params(struct qib_devdata *dd)
3117 {
3118 	u64 val;
3119 	u32 piobufs;
3120 	int mtu;
3121 
3122 	dd->uregbase = qib_read_kreg32(dd, kr_userregbase);
3123 
3124 	dd->rcvtidcnt = qib_read_kreg32(dd, kr_rcvtidcnt);
3125 	dd->rcvtidbase = qib_read_kreg32(dd, kr_rcvtidbase);
3126 	dd->rcvegrbase = qib_read_kreg32(dd, kr_rcvegrbase);
3127 	dd->palign = qib_read_kreg32(dd, kr_palign);
3128 	dd->piobufbase = qib_read_kreg64(dd, kr_sendpiobufbase);
3129 	dd->pio2k_bufbase = dd->piobufbase & 0xffffffff;
3130 
3131 	dd->rcvhdrcnt = qib_read_kreg32(dd, kr_rcvegrcnt);
3132 
3133 	val = qib_read_kreg64(dd, kr_sendpiosize);
3134 	dd->piosize2k = val & ~0U;
3135 	dd->piosize4k = val >> 32;
3136 
3137 	mtu = ib_mtu_enum_to_int(qib_ibmtu);
3138 	if (mtu == -1)
3139 		mtu = QIB_DEFAULT_MTU;
3140 	dd->pport->ibmtu = (u32)mtu;
3141 
3142 	val = qib_read_kreg64(dd, kr_sendpiobufcnt);
3143 	dd->piobcnt2k = val & ~0U;
3144 	dd->piobcnt4k = val >> 32;
3145 	dd->last_pio = dd->piobcnt4k + dd->piobcnt2k - 1;
3146 	/* these may be adjusted in init_chip_wc_pat() */
3147 	dd->pio2kbase = (u32 __iomem *)
3148 		(((char __iomem *)dd->kregbase) + dd->pio2k_bufbase);
3149 	if (dd->piobcnt4k) {
3150 		dd->pio4kbase = (u32 __iomem *)
3151 			(((char __iomem *) dd->kregbase) +
3152 			 (dd->piobufbase >> 32));
3153 		/*
3154 		 * 4K buffers take 2 pages; we use roundup just to be
3155 		 * paranoid; we calculate it once here, rather than on
3156 		 * ever buf allocate
3157 		 */
3158 		dd->align4k = ALIGN(dd->piosize4k, dd->palign);
3159 	}
3160 
3161 	piobufs = dd->piobcnt4k + dd->piobcnt2k;
3162 
3163 	dd->pioavregs = ALIGN(piobufs, sizeof(u64) * BITS_PER_BYTE / 2) /
3164 		(sizeof(u64) * BITS_PER_BYTE / 2);
3165 }
3166 
3167 /*
3168  * The chip base addresses in cspec and cpspec have to be set
3169  * after possible init_chip_wc_pat(), rather than in
3170  * get_6120_chip_params(), so split out as separate function
3171  */
3172 static void set_6120_baseaddrs(struct qib_devdata *dd)
3173 {
3174 	u32 cregbase;
3175 
3176 	cregbase = qib_read_kreg32(dd, kr_counterregbase);
3177 	dd->cspec->cregbase = (u64 __iomem *)
3178 		((char __iomem *) dd->kregbase + cregbase);
3179 
3180 	dd->egrtidbase = (u64 __iomem *)
3181 		((char __iomem *) dd->kregbase + dd->rcvegrbase);
3182 }
3183 
3184 /*
3185  * Write the final few registers that depend on some of the
3186  * init setup.  Done late in init, just before bringing up
3187  * the serdes.
3188  */
3189 static int qib_late_6120_initreg(struct qib_devdata *dd)
3190 {
3191 	int ret = 0;
3192 	u64 val;
3193 
3194 	qib_write_kreg(dd, kr_rcvhdrentsize, dd->rcvhdrentsize);
3195 	qib_write_kreg(dd, kr_rcvhdrsize, dd->rcvhdrsize);
3196 	qib_write_kreg(dd, kr_rcvhdrcnt, dd->rcvhdrcnt);
3197 	qib_write_kreg(dd, kr_sendpioavailaddr, dd->pioavailregs_phys);
3198 	val = qib_read_kreg64(dd, kr_sendpioavailaddr);
3199 	if (val != dd->pioavailregs_phys) {
3200 		qib_dev_err(dd,
3201 			"Catastrophic software error, SendPIOAvailAddr written as %lx, read back as %llx\n",
3202 			(unsigned long) dd->pioavailregs_phys,
3203 			(unsigned long long) val);
3204 		ret = -EINVAL;
3205 	}
3206 	return ret;
3207 }
3208 
3209 static int init_6120_variables(struct qib_devdata *dd)
3210 {
3211 	int ret = 0;
3212 	struct qib_pportdata *ppd;
3213 	u32 sbufs;
3214 
3215 	ppd = (struct qib_pportdata *)(dd + 1);
3216 	dd->pport = ppd;
3217 	dd->num_pports = 1;
3218 
3219 	dd->cspec = (struct qib_chip_specific *)(ppd + dd->num_pports);
3220 	ppd->cpspec = NULL; /* not used in this chip */
3221 
3222 	spin_lock_init(&dd->cspec->kernel_tid_lock);
3223 	spin_lock_init(&dd->cspec->user_tid_lock);
3224 	spin_lock_init(&dd->cspec->rcvmod_lock);
3225 	spin_lock_init(&dd->cspec->gpio_lock);
3226 
3227 	/* we haven't yet set QIB_PRESENT, so use read directly */
3228 	dd->revision = readq(&dd->kregbase[kr_revision]);
3229 
3230 	if ((dd->revision & 0xffffffffU) == 0xffffffffU) {
3231 		qib_dev_err(dd,
3232 			"Revision register read failure, giving up initialization\n");
3233 		ret = -ENODEV;
3234 		goto bail;
3235 	}
3236 	dd->flags |= QIB_PRESENT;  /* now register routines work */
3237 
3238 	dd->majrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3239 				    ChipRevMajor);
3240 	dd->minrev = (u8) SYM_FIELD(dd->revision, Revision_R,
3241 				    ChipRevMinor);
3242 
3243 	get_6120_chip_params(dd);
3244 	pe_boardname(dd); /* fill in boardname */
3245 
3246 	/*
3247 	 * GPIO bits for TWSI data and clock,
3248 	 * used for serial EEPROM.
3249 	 */
3250 	dd->gpio_sda_num = _QIB_GPIO_SDA_NUM;
3251 	dd->gpio_scl_num = _QIB_GPIO_SCL_NUM;
3252 	dd->twsi_eeprom_dev = QIB_TWSI_NO_DEV;
3253 
3254 	if (qib_unordered_wc())
3255 		dd->flags |= QIB_PIO_FLUSH_WC;
3256 
3257 	/*
3258 	 * EEPROM error log 0 is TXE Parity errors. 1 is RXE Parity.
3259 	 * 2 is Some Misc, 3 is reserved for future.
3260 	 */
3261 	dd->eep_st_masks[0].hwerrs_to_log = HWE_MASK(TXEMemParityErr);
3262 
3263 	/* Ignore errors in PIO/PBC on systems with unordered write-combining */
3264 	if (qib_unordered_wc())
3265 		dd->eep_st_masks[0].hwerrs_to_log &= ~TXE_PIO_PARITY;
3266 
3267 	dd->eep_st_masks[1].hwerrs_to_log = HWE_MASK(RXEMemParityErr);
3268 
3269 	dd->eep_st_masks[2].errs_to_log = ERR_MASK(ResetNegated);
3270 
3271 	ret = qib_init_pportdata(ppd, dd, 0, 1);
3272 	if (ret)
3273 		goto bail;
3274 	ppd->link_width_supported = IB_WIDTH_1X | IB_WIDTH_4X;
3275 	ppd->link_speed_supported = QIB_IB_SDR;
3276 	ppd->link_width_enabled = IB_WIDTH_4X;
3277 	ppd->link_speed_enabled = ppd->link_speed_supported;
3278 	/* these can't change for this chip, so set once */
3279 	ppd->link_width_active = ppd->link_width_enabled;
3280 	ppd->link_speed_active = ppd->link_speed_enabled;
3281 	ppd->vls_supported = IB_VL_VL0;
3282 	ppd->vls_operational = ppd->vls_supported;
3283 
3284 	dd->rcvhdrentsize = QIB_RCVHDR_ENTSIZE;
3285 	dd->rcvhdrsize = QIB_DFLT_RCVHDRSIZE;
3286 	dd->rhf_offset = 0;
3287 
3288 	/* we always allocate at least 2048 bytes for eager buffers */
3289 	ret = ib_mtu_enum_to_int(qib_ibmtu);
3290 	dd->rcvegrbufsize = ret != -1 ? max(ret, 2048) : QIB_DEFAULT_MTU;
3291 	BUG_ON(!is_power_of_2(dd->rcvegrbufsize));
3292 	dd->rcvegrbufsize_shift = ilog2(dd->rcvegrbufsize);
3293 
3294 	qib_6120_tidtemplate(dd);
3295 
3296 	/*
3297 	 * We can request a receive interrupt for 1 or
3298 	 * more packets from current offset.  For now, we set this
3299 	 * up for a single packet.
3300 	 */
3301 	dd->rhdrhead_intr_off = 1ULL << 32;
3302 
3303 	/* setup the stats timer; the add_timer is done at end of init */
3304 	init_timer(&dd->stats_timer);
3305 	dd->stats_timer.function = qib_get_6120_faststats;
3306 	dd->stats_timer.data = (unsigned long) dd;
3307 
3308 	init_timer(&dd->cspec->pma_timer);
3309 	dd->cspec->pma_timer.function = pma_6120_timer;
3310 	dd->cspec->pma_timer.data = (unsigned long) ppd;
3311 
3312 	dd->ureg_align = qib_read_kreg32(dd, kr_palign);
3313 
3314 	dd->piosize2kmax_dwords = dd->piosize2k >> 2;
3315 	qib_6120_config_ctxts(dd);
3316 	qib_set_ctxtcnt(dd);
3317 
3318 	ret = init_chip_wc_pat(dd, 0);
3319 	if (ret)
3320 		goto bail;
3321 	set_6120_baseaddrs(dd); /* set chip access pointers now */
3322 
3323 	ret = 0;
3324 	if (qib_mini_init)
3325 		goto bail;
3326 
3327 	qib_num_cfg_vls = 1; /* if any 6120's, only one VL */
3328 
3329 	ret = qib_create_ctxts(dd);
3330 	init_6120_cntrnames(dd);
3331 
3332 	/* use all of 4KB buffers for the kernel, otherwise 16 */
3333 	sbufs = dd->piobcnt4k ?  dd->piobcnt4k : 16;
3334 
3335 	dd->lastctxt_piobuf = dd->piobcnt2k + dd->piobcnt4k - sbufs;
3336 	dd->pbufsctxt = dd->lastctxt_piobuf /
3337 		(dd->cfgctxts - dd->first_user_ctxt);
3338 
3339 	if (ret)
3340 		goto bail;
3341 bail:
3342 	return ret;
3343 }
3344 
3345 /*
3346  * For this chip, we want to use the same buffer every time
3347  * when we are trying to bring the link up (they are always VL15
3348  * packets).  At that link state the packet should always go out immediately
3349  * (or at least be discarded at the tx interface if the link is down).
3350  * If it doesn't, and the buffer isn't available, that means some other
3351  * sender has gotten ahead of us, and is preventing our packet from going
3352  * out.  In that case, we flush all packets, and try again.  If that still
3353  * fails, we fail the request, and hope things work the next time around.
3354  *
3355  * We don't need very complicated heuristics on whether the packet had
3356  * time to go out or not, since even at SDR 1X, it goes out in very short
3357  * time periods, covered by the chip reads done here and as part of the
3358  * flush.
3359  */
3360 static u32 __iomem *get_6120_link_buf(struct qib_pportdata *ppd, u32 *bnum)
3361 {
3362 	u32 __iomem *buf;
3363 	u32 lbuf = ppd->dd->piobcnt2k + ppd->dd->piobcnt4k - 1;
3364 
3365 	/*
3366 	 * always blip to get avail list updated, since it's almost
3367 	 * always needed, and is fairly cheap.
3368 	 */
3369 	sendctrl_6120_mod(ppd->dd->pport, QIB_SENDCTRL_AVAIL_BLIP);
3370 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3371 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3372 	if (buf)
3373 		goto done;
3374 
3375 	sendctrl_6120_mod(ppd, QIB_SENDCTRL_DISARM_ALL | QIB_SENDCTRL_FLUSH |
3376 			  QIB_SENDCTRL_AVAIL_BLIP);
3377 	ppd->dd->upd_pio_shadow  = 1; /* update our idea of what's busy */
3378 	qib_read_kreg64(ppd->dd, kr_scratch); /* extra chip flush */
3379 	buf = qib_getsendbuf_range(ppd->dd, bnum, lbuf, lbuf);
3380 done:
3381 	return buf;
3382 }
3383 
3384 static u32 __iomem *qib_6120_getsendbuf(struct qib_pportdata *ppd, u64 pbc,
3385 					u32 *pbufnum)
3386 {
3387 	u32 first, last, plen = pbc & QIB_PBC_LENGTH_MASK;
3388 	struct qib_devdata *dd = ppd->dd;
3389 	u32 __iomem *buf;
3390 
3391 	if (((pbc >> 32) & PBC_6120_VL15_SEND_CTRL) &&
3392 		!(ppd->lflags & (QIBL_IB_AUTONEG_INPROG | QIBL_LINKACTIVE)))
3393 		buf = get_6120_link_buf(ppd, pbufnum);
3394 	else {
3395 
3396 		if ((plen + 1) > dd->piosize2kmax_dwords)
3397 			first = dd->piobcnt2k;
3398 		else
3399 			first = 0;
3400 		/* try 4k if all 2k busy, so same last for both sizes */
3401 		last = dd->piobcnt2k + dd->piobcnt4k - 1;
3402 		buf = qib_getsendbuf_range(dd, pbufnum, first, last);
3403 	}
3404 	return buf;
3405 }
3406 
3407 static int init_sdma_6120_regs(struct qib_pportdata *ppd)
3408 {
3409 	return -ENODEV;
3410 }
3411 
3412 static u16 qib_sdma_6120_gethead(struct qib_pportdata *ppd)
3413 {
3414 	return 0;
3415 }
3416 
3417 static int qib_sdma_6120_busy(struct qib_pportdata *ppd)
3418 {
3419 	return 0;
3420 }
3421 
3422 static void qib_sdma_update_6120_tail(struct qib_pportdata *ppd, u16 tail)
3423 {
3424 }
3425 
3426 static void qib_6120_sdma_sendctrl(struct qib_pportdata *ppd, unsigned op)
3427 {
3428 }
3429 
3430 static void qib_sdma_set_6120_desc_cnt(struct qib_pportdata *ppd, unsigned cnt)
3431 {
3432 }
3433 
3434 /*
3435  * the pbc doesn't need a VL15 indicator, but we need it for link_buf.
3436  * The chip ignores the bit if set.
3437  */
3438 static u32 qib_6120_setpbc_control(struct qib_pportdata *ppd, u32 plen,
3439 				   u8 srate, u8 vl)
3440 {
3441 	return vl == 15 ? PBC_6120_VL15_SEND_CTRL : 0;
3442 }
3443 
3444 static void qib_6120_initvl15_bufs(struct qib_devdata *dd)
3445 {
3446 }
3447 
3448 static void qib_6120_init_ctxt(struct qib_ctxtdata *rcd)
3449 {
3450 	rcd->rcvegrcnt = rcd->dd->rcvhdrcnt;
3451 	rcd->rcvegr_tid_base = rcd->ctxt * rcd->rcvegrcnt;
3452 }
3453 
3454 static void qib_6120_txchk_change(struct qib_devdata *dd, u32 start,
3455 	u32 len, u32 avail, struct qib_ctxtdata *rcd)
3456 {
3457 }
3458 
3459 static void writescratch(struct qib_devdata *dd, u32 val)
3460 {
3461 	(void) qib_write_kreg(dd, kr_scratch, val);
3462 }
3463 
3464 static int qib_6120_tempsense_rd(struct qib_devdata *dd, int regnum)
3465 {
3466 	return -ENXIO;
3467 }
3468 
3469 #ifdef CONFIG_INFINIBAND_QIB_DCA
3470 static int qib_6120_notify_dca(struct qib_devdata *dd, unsigned long event)
3471 {
3472 	return 0;
3473 }
3474 #endif
3475 
3476 /* Dummy function, as 6120 boards never disable EEPROM Write */
3477 static int qib_6120_eeprom_wen(struct qib_devdata *dd, int wen)
3478 {
3479 	return 1;
3480 }
3481 
3482 /**
3483  * qib_init_iba6120_funcs - set up the chip-specific function pointers
3484  * @pdev: pci_dev of the qlogic_ib device
3485  * @ent: pci_device_id matching this chip
3486  *
3487  * This is global, and is called directly at init to set up the
3488  * chip-specific function pointers for later use.
3489  *
3490  * It also allocates/partially-inits the qib_devdata struct for
3491  * this device.
3492  */
3493 struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *pdev,
3494 					   const struct pci_device_id *ent)
3495 {
3496 	struct qib_devdata *dd;
3497 	int ret;
3498 
3499 	dd = qib_alloc_devdata(pdev, sizeof(struct qib_pportdata) +
3500 			       sizeof(struct qib_chip_specific));
3501 	if (IS_ERR(dd))
3502 		goto bail;
3503 
3504 	dd->f_bringup_serdes    = qib_6120_bringup_serdes;
3505 	dd->f_cleanup           = qib_6120_setup_cleanup;
3506 	dd->f_clear_tids        = qib_6120_clear_tids;
3507 	dd->f_free_irq          = qib_6120_free_irq;
3508 	dd->f_get_base_info     = qib_6120_get_base_info;
3509 	dd->f_get_msgheader     = qib_6120_get_msgheader;
3510 	dd->f_getsendbuf        = qib_6120_getsendbuf;
3511 	dd->f_gpio_mod          = gpio_6120_mod;
3512 	dd->f_eeprom_wen	= qib_6120_eeprom_wen;
3513 	dd->f_hdrqempty         = qib_6120_hdrqempty;
3514 	dd->f_ib_updown         = qib_6120_ib_updown;
3515 	dd->f_init_ctxt         = qib_6120_init_ctxt;
3516 	dd->f_initvl15_bufs     = qib_6120_initvl15_bufs;
3517 	dd->f_intr_fallback     = qib_6120_nointr_fallback;
3518 	dd->f_late_initreg      = qib_late_6120_initreg;
3519 	dd->f_setpbc_control    = qib_6120_setpbc_control;
3520 	dd->f_portcntr          = qib_portcntr_6120;
3521 	dd->f_put_tid           = (dd->minrev >= 2) ?
3522 				      qib_6120_put_tid_2 :
3523 				      qib_6120_put_tid;
3524 	dd->f_quiet_serdes      = qib_6120_quiet_serdes;
3525 	dd->f_rcvctrl           = rcvctrl_6120_mod;
3526 	dd->f_read_cntrs        = qib_read_6120cntrs;
3527 	dd->f_read_portcntrs    = qib_read_6120portcntrs;
3528 	dd->f_reset             = qib_6120_setup_reset;
3529 	dd->f_init_sdma_regs    = init_sdma_6120_regs;
3530 	dd->f_sdma_busy         = qib_sdma_6120_busy;
3531 	dd->f_sdma_gethead      = qib_sdma_6120_gethead;
3532 	dd->f_sdma_sendctrl     = qib_6120_sdma_sendctrl;
3533 	dd->f_sdma_set_desc_cnt = qib_sdma_set_6120_desc_cnt;
3534 	dd->f_sdma_update_tail  = qib_sdma_update_6120_tail;
3535 	dd->f_sendctrl          = sendctrl_6120_mod;
3536 	dd->f_set_armlaunch     = qib_set_6120_armlaunch;
3537 	dd->f_set_cntr_sample   = qib_set_cntr_6120_sample;
3538 	dd->f_iblink_state      = qib_6120_iblink_state;
3539 	dd->f_ibphys_portstate  = qib_6120_phys_portstate;
3540 	dd->f_get_ib_cfg        = qib_6120_get_ib_cfg;
3541 	dd->f_set_ib_cfg        = qib_6120_set_ib_cfg;
3542 	dd->f_set_ib_loopback   = qib_6120_set_loopback;
3543 	dd->f_set_intr_state    = qib_6120_set_intr_state;
3544 	dd->f_setextled         = qib_6120_setup_setextled;
3545 	dd->f_txchk_change      = qib_6120_txchk_change;
3546 	dd->f_update_usrhead    = qib_update_6120_usrhead;
3547 	dd->f_wantpiobuf_intr   = qib_wantpiobuf_6120_intr;
3548 	dd->f_xgxs_reset        = qib_6120_xgxs_reset;
3549 	dd->f_writescratch      = writescratch;
3550 	dd->f_tempsense_rd	= qib_6120_tempsense_rd;
3551 #ifdef CONFIG_INFINIBAND_QIB_DCA
3552 	dd->f_notify_dca = qib_6120_notify_dca;
3553 #endif
3554 	/*
3555 	 * Do remaining pcie setup and save pcie values in dd.
3556 	 * Any error printing is already done by the init code.
3557 	 * On return, we have the chip mapped and accessible,
3558 	 * but chip registers are not set up until start of
3559 	 * init_6120_variables.
3560 	 */
3561 	ret = qib_pcie_ddinit(dd, pdev, ent);
3562 	if (ret < 0)
3563 		goto bail_free;
3564 
3565 	/* initialize chip-specific variables */
3566 	ret = init_6120_variables(dd);
3567 	if (ret)
3568 		goto bail_cleanup;
3569 
3570 	if (qib_mini_init)
3571 		goto bail;
3572 
3573 	if (qib_pcie_params(dd, 8, NULL, NULL))
3574 		qib_dev_err(dd,
3575 			"Failed to setup PCIe or interrupts; continuing anyway\n");
3576 	dd->cspec->irq = pdev->irq; /* save IRQ */
3577 
3578 	/* clear diagctrl register, in case diags were running and crashed */
3579 	qib_write_kreg(dd, kr_hwdiagctrl, 0);
3580 
3581 	if (qib_read_kreg64(dd, kr_hwerrstatus) &
3582 	    QLOGIC_IB_HWE_SERDESPLLFAILED)
3583 		qib_write_kreg(dd, kr_hwerrclear,
3584 			       QLOGIC_IB_HWE_SERDESPLLFAILED);
3585 
3586 	/* setup interrupt handler (interrupt type handled above) */
3587 	qib_setup_6120_interrupt(dd);
3588 	/* Note that qpn_mask is set by qib_6120_config_ctxts() first */
3589 	qib_6120_init_hwerrors(dd);
3590 
3591 	goto bail;
3592 
3593 bail_cleanup:
3594 	qib_pcie_ddcleanup(dd);
3595 bail_free:
3596 	qib_free_devdata(dd);
3597 	dd = ERR_PTR(ret);
3598 bail:
3599 	return dd;
3600 }
3601