xref: /openbmc/linux/drivers/infiniband/hw/qib/qib.h (revision 68198dca)
1 #ifndef _QIB_KERNEL_H
2 #define _QIB_KERNEL_H
3 /*
4  * Copyright (c) 2012 - 2017 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
6  * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
7  *
8  * This software is available to you under a choice of one of two
9  * licenses.  You may choose to be licensed under the terms of the GNU
10  * General Public License (GPL) Version 2, available from the file
11  * COPYING in the main directory of this source tree, or the
12  * OpenIB.org BSD license below:
13  *
14  *     Redistribution and use in source and binary forms, with or
15  *     without modification, are permitted provided that the following
16  *     conditions are met:
17  *
18  *      - Redistributions of source code must retain the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer.
21  *
22  *      - Redistributions in binary form must reproduce the above
23  *        copyright notice, this list of conditions and the following
24  *        disclaimer in the documentation and/or other materials
25  *        provided with the distribution.
26  *
27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34  * SOFTWARE.
35  */
36 
37 /*
38  * This header file is the base header file for qlogic_ib kernel code
39  * qib_user.h serves a similar purpose for user code.
40  */
41 
42 #include <linux/interrupt.h>
43 #include <linux/pci.h>
44 #include <linux/dma-mapping.h>
45 #include <linux/mutex.h>
46 #include <linux/list.h>
47 #include <linux/scatterlist.h>
48 #include <linux/slab.h>
49 #include <linux/io.h>
50 #include <linux/fs.h>
51 #include <linux/completion.h>
52 #include <linux/kref.h>
53 #include <linux/sched.h>
54 #include <linux/kthread.h>
55 #include <rdma/ib_hdrs.h>
56 #include <rdma/rdma_vt.h>
57 
58 #include "qib_common.h"
59 #include "qib_verbs.h"
60 
61 /* only s/w major version of QLogic_IB we can handle */
62 #define QIB_CHIP_VERS_MAJ 2U
63 
64 /* don't care about this except printing */
65 #define QIB_CHIP_VERS_MIN 0U
66 
67 /* The Organization Unique Identifier (Mfg code), and its position in GUID */
68 #define QIB_OUI 0x001175
69 #define QIB_OUI_LSB 40
70 
71 /*
72  * per driver stats, either not device nor port-specific, or
73  * summed over all of the devices and ports.
74  * They are described by name via ipathfs filesystem, so layout
75  * and number of elements can change without breaking compatibility.
76  * If members are added or deleted qib_statnames[] in qib_fs.c must
77  * change to match.
78  */
79 struct qlogic_ib_stats {
80 	__u64 sps_ints; /* number of interrupts handled */
81 	__u64 sps_errints; /* number of error interrupts */
82 	__u64 sps_txerrs; /* tx-related packet errors */
83 	__u64 sps_rcverrs; /* non-crc rcv packet errors */
84 	__u64 sps_hwerrs; /* hardware errors reported (parity, etc.) */
85 	__u64 sps_nopiobufs; /* no pio bufs avail from kernel */
86 	__u64 sps_ctxts; /* number of contexts currently open */
87 	__u64 sps_lenerrs; /* number of kernel packets where RHF != LRH len */
88 	__u64 sps_buffull;
89 	__u64 sps_hdrfull;
90 };
91 
92 extern struct qlogic_ib_stats qib_stats;
93 extern const struct pci_error_handlers qib_pci_err_handler;
94 
95 #define QIB_CHIP_SWVERSION QIB_CHIP_VERS_MAJ
96 /*
97  * First-cut critierion for "device is active" is
98  * two thousand dwords combined Tx, Rx traffic per
99  * 5-second interval. SMA packets are 64 dwords,
100  * and occur "a few per second", presumably each way.
101  */
102 #define QIB_TRAFFIC_ACTIVE_THRESHOLD (2000)
103 
104 /*
105  * Below contains all data related to a single context (formerly called port).
106  */
107 
108 #ifdef CONFIG_DEBUG_FS
109 struct qib_opcode_stats_perctx;
110 #endif
111 
112 struct qib_ctxtdata {
113 	void **rcvegrbuf;
114 	dma_addr_t *rcvegrbuf_phys;
115 	/* rcvhdrq base, needs mmap before useful */
116 	void *rcvhdrq;
117 	/* kernel virtual address where hdrqtail is updated */
118 	void *rcvhdrtail_kvaddr;
119 	/*
120 	 * temp buffer for expected send setup, allocated at open, instead
121 	 * of each setup call
122 	 */
123 	void *tid_pg_list;
124 	/*
125 	 * Shared page for kernel to signal user processes that send buffers
126 	 * need disarming.  The process should call QIB_CMD_DISARM_BUFS
127 	 * or QIB_CMD_ACK_EVENT with IPATH_EVENT_DISARM_BUFS set.
128 	 */
129 	unsigned long *user_event_mask;
130 	/* when waiting for rcv or pioavail */
131 	wait_queue_head_t wait;
132 	/*
133 	 * rcvegr bufs base, physical, must fit
134 	 * in 44 bits so 32 bit programs mmap64 44 bit works)
135 	 */
136 	dma_addr_t rcvegr_phys;
137 	/* mmap of hdrq, must fit in 44 bits */
138 	dma_addr_t rcvhdrq_phys;
139 	dma_addr_t rcvhdrqtailaddr_phys;
140 
141 	/*
142 	 * number of opens (including slave sub-contexts) on this instance
143 	 * (ignoring forks, dup, etc. for now)
144 	 */
145 	int cnt;
146 	/*
147 	 * how much space to leave at start of eager TID entries for
148 	 * protocol use, on each TID
149 	 */
150 	/* instead of calculating it */
151 	unsigned ctxt;
152 	/* local node of context */
153 	int node_id;
154 	/* non-zero if ctxt is being shared. */
155 	u16 subctxt_cnt;
156 	/* non-zero if ctxt is being shared. */
157 	u16 subctxt_id;
158 	/* number of eager TID entries. */
159 	u16 rcvegrcnt;
160 	/* index of first eager TID entry. */
161 	u16 rcvegr_tid_base;
162 	/* number of pio bufs for this ctxt (all procs, if shared) */
163 	u32 piocnt;
164 	/* first pio buffer for this ctxt */
165 	u32 pio_base;
166 	/* chip offset of PIO buffers for this ctxt */
167 	u32 piobufs;
168 	/* how many alloc_pages() chunks in rcvegrbuf_pages */
169 	u32 rcvegrbuf_chunks;
170 	/* how many egrbufs per chunk */
171 	u16 rcvegrbufs_perchunk;
172 	/* ilog2 of above */
173 	u16 rcvegrbufs_perchunk_shift;
174 	/* order for rcvegrbuf_pages */
175 	size_t rcvegrbuf_size;
176 	/* rcvhdrq size (for freeing) */
177 	size_t rcvhdrq_size;
178 	/* per-context flags for fileops/intr communication */
179 	unsigned long flag;
180 	/* next expected TID to check when looking for free */
181 	u32 tidcursor;
182 	/* WAIT_RCV that timed out, no interrupt */
183 	u32 rcvwait_to;
184 	/* WAIT_PIO that timed out, no interrupt */
185 	u32 piowait_to;
186 	/* WAIT_RCV already happened, no wait */
187 	u32 rcvnowait;
188 	/* WAIT_PIO already happened, no wait */
189 	u32 pionowait;
190 	/* total number of polled urgent packets */
191 	u32 urgent;
192 	/* saved total number of polled urgent packets for poll edge trigger */
193 	u32 urgent_poll;
194 	/* pid of process using this ctxt */
195 	pid_t pid;
196 	pid_t subpid[QLOGIC_IB_MAX_SUBCTXT];
197 	/* same size as task_struct .comm[], command that opened context */
198 	char comm[16];
199 	/* pkeys set by this use of this ctxt */
200 	u16 pkeys[4];
201 	/* so file ops can get at unit */
202 	struct qib_devdata *dd;
203 	/* so funcs that need physical port can get it easily */
204 	struct qib_pportdata *ppd;
205 	/* A page of memory for rcvhdrhead, rcvegrhead, rcvegrtail * N */
206 	void *subctxt_uregbase;
207 	/* An array of pages for the eager receive buffers * N */
208 	void *subctxt_rcvegrbuf;
209 	/* An array of pages for the eager header queue entries * N */
210 	void *subctxt_rcvhdr_base;
211 	/* The version of the library which opened this ctxt */
212 	u32 userversion;
213 	/* Bitmask of active slaves */
214 	u32 active_slaves;
215 	/* Type of packets or conditions we want to poll for */
216 	u16 poll_type;
217 	/* receive packet sequence counter */
218 	u8 seq_cnt;
219 	u8 redirect_seq_cnt;
220 	/* ctxt rcvhdrq head offset */
221 	u32 head;
222 	/* QPs waiting for context processing */
223 	struct list_head qp_wait_list;
224 #ifdef CONFIG_DEBUG_FS
225 	/* verbs stats per CTX */
226 	struct qib_opcode_stats_perctx *opstats;
227 #endif
228 };
229 
230 struct rvt_sge_state;
231 
232 struct qib_sdma_txreq {
233 	int                 flags;
234 	int                 sg_count;
235 	dma_addr_t          addr;
236 	void              (*callback)(struct qib_sdma_txreq *, int);
237 	u16                 start_idx;  /* sdma private */
238 	u16                 next_descq_idx;  /* sdma private */
239 	struct list_head    list;       /* sdma private */
240 };
241 
242 struct qib_sdma_desc {
243 	__le64 qw[2];
244 };
245 
246 struct qib_verbs_txreq {
247 	struct qib_sdma_txreq   txreq;
248 	struct rvt_qp           *qp;
249 	struct rvt_swqe         *wqe;
250 	u32                     dwords;
251 	u16                     hdr_dwords;
252 	u16                     hdr_inx;
253 	struct qib_pio_header	*align_buf;
254 	struct rvt_mregion	*mr;
255 	struct rvt_sge_state    *ss;
256 };
257 
258 #define QIB_SDMA_TXREQ_F_USELARGEBUF  0x1
259 #define QIB_SDMA_TXREQ_F_HEADTOHOST   0x2
260 #define QIB_SDMA_TXREQ_F_INTREQ       0x4
261 #define QIB_SDMA_TXREQ_F_FREEBUF      0x8
262 #define QIB_SDMA_TXREQ_F_FREEDESC     0x10
263 
264 #define QIB_SDMA_TXREQ_S_OK        0
265 #define QIB_SDMA_TXREQ_S_SENDERROR 1
266 #define QIB_SDMA_TXREQ_S_ABORTED   2
267 #define QIB_SDMA_TXREQ_S_SHUTDOWN  3
268 
269 /*
270  * Get/Set IB link-level config parameters for f_get/set_ib_cfg()
271  * Mostly for MADs that set or query link parameters, also ipath
272  * config interfaces
273  */
274 #define QIB_IB_CFG_LIDLMC 0 /* LID (LS16b) and Mask (MS16b) */
275 #define QIB_IB_CFG_LWID_ENB 2 /* allowed Link-width */
276 #define QIB_IB_CFG_LWID 3 /* currently active Link-width */
277 #define QIB_IB_CFG_SPD_ENB 4 /* allowed Link speeds */
278 #define QIB_IB_CFG_SPD 5 /* current Link spd */
279 #define QIB_IB_CFG_RXPOL_ENB 6 /* Auto-RX-polarity enable */
280 #define QIB_IB_CFG_LREV_ENB 7 /* Auto-Lane-reversal enable */
281 #define QIB_IB_CFG_LINKLATENCY 8 /* Link Latency (IB1.2 only) */
282 #define QIB_IB_CFG_HRTBT 9 /* IB heartbeat off/enable/auto; DDR/QDR only */
283 #define QIB_IB_CFG_OP_VLS 10 /* operational VLs */
284 #define QIB_IB_CFG_VL_HIGH_CAP 11 /* num of VL high priority weights */
285 #define QIB_IB_CFG_VL_LOW_CAP 12 /* num of VL low priority weights */
286 #define QIB_IB_CFG_OVERRUN_THRESH 13 /* IB overrun threshold */
287 #define QIB_IB_CFG_PHYERR_THRESH 14 /* IB PHY error threshold */
288 #define QIB_IB_CFG_LINKDEFAULT 15 /* IB link default (sleep/poll) */
289 #define QIB_IB_CFG_PKEYS 16 /* update partition keys */
290 #define QIB_IB_CFG_MTU 17 /* update MTU in IBC */
291 #define QIB_IB_CFG_LSTATE 18 /* update linkcmd and linkinitcmd in IBC */
292 #define QIB_IB_CFG_VL_HIGH_LIMIT 19
293 #define QIB_IB_CFG_PMA_TICKS 20 /* PMA sample tick resolution */
294 #define QIB_IB_CFG_PORT 21 /* switch port we are connected to */
295 
296 /*
297  * for CFG_LSTATE: LINKCMD in upper 16 bits, LINKINITCMD in lower 16
298  * IB_LINKINITCMD_POLL and SLEEP are also used as set/get values for
299  * QIB_IB_CFG_LINKDEFAULT cmd
300  */
301 #define   IB_LINKCMD_DOWN   (0 << 16)
302 #define   IB_LINKCMD_ARMED  (1 << 16)
303 #define   IB_LINKCMD_ACTIVE (2 << 16)
304 #define   IB_LINKINITCMD_NOP     0
305 #define   IB_LINKINITCMD_POLL    1
306 #define   IB_LINKINITCMD_SLEEP   2
307 #define   IB_LINKINITCMD_DISABLE 3
308 
309 /*
310  * valid states passed to qib_set_linkstate() user call
311  */
312 #define QIB_IB_LINKDOWN         0
313 #define QIB_IB_LINKARM          1
314 #define QIB_IB_LINKACTIVE       2
315 #define QIB_IB_LINKDOWN_ONLY    3
316 #define QIB_IB_LINKDOWN_SLEEP   4
317 #define QIB_IB_LINKDOWN_DISABLE 5
318 
319 /*
320  * These 7 values (SDR, DDR, and QDR may be ORed for auto-speed
321  * negotiation) are used for the 3rd argument to path_f_set_ib_cfg
322  * with cmd QIB_IB_CFG_SPD_ENB, by direct calls or via sysfs.  They
323  * are also the the possible values for qib_link_speed_enabled and active
324  * The values were chosen to match values used within the IB spec.
325  */
326 #define QIB_IB_SDR 1
327 #define QIB_IB_DDR 2
328 #define QIB_IB_QDR 4
329 
330 #define QIB_DEFAULT_MTU 4096
331 
332 /* max number of IB ports supported per HCA */
333 #define QIB_MAX_IB_PORTS 2
334 
335 /*
336  * Possible IB config parameters for f_get/set_ib_table()
337  */
338 #define QIB_IB_TBL_VL_HIGH_ARB 1 /* Get/set VL high priority weights */
339 #define QIB_IB_TBL_VL_LOW_ARB 2 /* Get/set VL low priority weights */
340 
341 /*
342  * Possible "operations" for f_rcvctrl(ppd, op, ctxt)
343  * these are bits so they can be combined, e.g.
344  * QIB_RCVCTRL_INTRAVAIL_ENB | QIB_RCVCTRL_CTXT_ENB
345  */
346 #define QIB_RCVCTRL_TAILUPD_ENB 0x01
347 #define QIB_RCVCTRL_TAILUPD_DIS 0x02
348 #define QIB_RCVCTRL_CTXT_ENB 0x04
349 #define QIB_RCVCTRL_CTXT_DIS 0x08
350 #define QIB_RCVCTRL_INTRAVAIL_ENB 0x10
351 #define QIB_RCVCTRL_INTRAVAIL_DIS 0x20
352 #define QIB_RCVCTRL_PKEY_ENB 0x40  /* Note, default is enabled */
353 #define QIB_RCVCTRL_PKEY_DIS 0x80
354 #define QIB_RCVCTRL_BP_ENB 0x0100
355 #define QIB_RCVCTRL_BP_DIS 0x0200
356 #define QIB_RCVCTRL_TIDFLOW_ENB 0x0400
357 #define QIB_RCVCTRL_TIDFLOW_DIS 0x0800
358 
359 /*
360  * Possible "operations" for f_sendctrl(ppd, op, var)
361  * these are bits so they can be combined, e.g.
362  * QIB_SENDCTRL_BUFAVAIL_ENB | QIB_SENDCTRL_ENB
363  * Some operations (e.g. DISARM, ABORT) are known to
364  * be "one-shot", so do not modify shadow.
365  */
366 #define QIB_SENDCTRL_DISARM       (0x1000)
367 #define QIB_SENDCTRL_DISARM_BUF(bufn) ((bufn) | QIB_SENDCTRL_DISARM)
368 	/* available (0x2000) */
369 #define QIB_SENDCTRL_AVAIL_DIS    (0x4000)
370 #define QIB_SENDCTRL_AVAIL_ENB    (0x8000)
371 #define QIB_SENDCTRL_AVAIL_BLIP  (0x10000)
372 #define QIB_SENDCTRL_SEND_DIS    (0x20000)
373 #define QIB_SENDCTRL_SEND_ENB    (0x40000)
374 #define QIB_SENDCTRL_FLUSH       (0x80000)
375 #define QIB_SENDCTRL_CLEAR      (0x100000)
376 #define QIB_SENDCTRL_DISARM_ALL (0x200000)
377 
378 /*
379  * These are the generic indices for requesting per-port
380  * counter values via the f_portcntr function.  They
381  * are always returned as 64 bit values, although most
382  * are 32 bit counters.
383  */
384 /* send-related counters */
385 #define QIBPORTCNTR_PKTSEND         0U
386 #define QIBPORTCNTR_WORDSEND        1U
387 #define QIBPORTCNTR_PSXMITDATA      2U
388 #define QIBPORTCNTR_PSXMITPKTS      3U
389 #define QIBPORTCNTR_PSXMITWAIT      4U
390 #define QIBPORTCNTR_SENDSTALL       5U
391 /* receive-related counters */
392 #define QIBPORTCNTR_PKTRCV          6U
393 #define QIBPORTCNTR_PSRCVDATA       7U
394 #define QIBPORTCNTR_PSRCVPKTS       8U
395 #define QIBPORTCNTR_RCVEBP          9U
396 #define QIBPORTCNTR_RCVOVFL         10U
397 #define QIBPORTCNTR_WORDRCV         11U
398 /* IB link related error counters */
399 #define QIBPORTCNTR_RXLOCALPHYERR   12U
400 #define QIBPORTCNTR_RXVLERR         13U
401 #define QIBPORTCNTR_ERRICRC         14U
402 #define QIBPORTCNTR_ERRVCRC         15U
403 #define QIBPORTCNTR_ERRLPCRC        16U
404 #define QIBPORTCNTR_BADFORMAT       17U
405 #define QIBPORTCNTR_ERR_RLEN        18U
406 #define QIBPORTCNTR_IBSYMBOLERR     19U
407 #define QIBPORTCNTR_INVALIDRLEN     20U
408 #define QIBPORTCNTR_UNSUPVL         21U
409 #define QIBPORTCNTR_EXCESSBUFOVFL   22U
410 #define QIBPORTCNTR_ERRLINK         23U
411 #define QIBPORTCNTR_IBLINKDOWN      24U
412 #define QIBPORTCNTR_IBLINKERRRECOV  25U
413 #define QIBPORTCNTR_LLI             26U
414 /* other error counters */
415 #define QIBPORTCNTR_RXDROPPKT       27U
416 #define QIBPORTCNTR_VL15PKTDROP     28U
417 #define QIBPORTCNTR_ERRPKEY         29U
418 #define QIBPORTCNTR_KHDROVFL        30U
419 /* sampling counters (these are actually control registers) */
420 #define QIBPORTCNTR_PSINTERVAL      31U
421 #define QIBPORTCNTR_PSSTART         32U
422 #define QIBPORTCNTR_PSSTAT          33U
423 
424 /* how often we check for packet activity for "power on hours (in seconds) */
425 #define ACTIVITY_TIMER 5
426 
427 #define MAX_NAME_SIZE 64
428 
429 #ifdef CONFIG_INFINIBAND_QIB_DCA
430 struct qib_irq_notify;
431 #endif
432 
433 struct qib_msix_entry {
434 	void *arg;
435 #ifdef CONFIG_INFINIBAND_QIB_DCA
436 	int dca;
437 	int rcv;
438 	struct qib_irq_notify *notifier;
439 #endif
440 	cpumask_var_t mask;
441 };
442 
443 /* Below is an opaque struct. Each chip (device) can maintain
444  * private data needed for its operation, but not germane to the
445  * rest of the driver.  For convenience, we define another that
446  * is chip-specific, per-port
447  */
448 struct qib_chip_specific;
449 struct qib_chipport_specific;
450 
451 enum qib_sdma_states {
452 	qib_sdma_state_s00_hw_down,
453 	qib_sdma_state_s10_hw_start_up_wait,
454 	qib_sdma_state_s20_idle,
455 	qib_sdma_state_s30_sw_clean_up_wait,
456 	qib_sdma_state_s40_hw_clean_up_wait,
457 	qib_sdma_state_s50_hw_halt_wait,
458 	qib_sdma_state_s99_running,
459 };
460 
461 enum qib_sdma_events {
462 	qib_sdma_event_e00_go_hw_down,
463 	qib_sdma_event_e10_go_hw_start,
464 	qib_sdma_event_e20_hw_started,
465 	qib_sdma_event_e30_go_running,
466 	qib_sdma_event_e40_sw_cleaned,
467 	qib_sdma_event_e50_hw_cleaned,
468 	qib_sdma_event_e60_hw_halted,
469 	qib_sdma_event_e70_go_idle,
470 	qib_sdma_event_e7220_err_halted,
471 	qib_sdma_event_e7322_err_halted,
472 	qib_sdma_event_e90_timer_tick,
473 };
474 
475 extern char *qib_sdma_state_names[];
476 extern char *qib_sdma_event_names[];
477 
478 struct sdma_set_state_action {
479 	unsigned op_enable:1;
480 	unsigned op_intenable:1;
481 	unsigned op_halt:1;
482 	unsigned op_drain:1;
483 	unsigned go_s99_running_tofalse:1;
484 	unsigned go_s99_running_totrue:1;
485 };
486 
487 struct qib_sdma_state {
488 	struct kref          kref;
489 	struct completion    comp;
490 	enum qib_sdma_states current_state;
491 	struct sdma_set_state_action *set_state_action;
492 	unsigned             current_op;
493 	unsigned             go_s99_running;
494 	unsigned             first_sendbuf;
495 	unsigned             last_sendbuf; /* really last +1 */
496 	/* debugging/devel */
497 	enum qib_sdma_states previous_state;
498 	unsigned             previous_op;
499 	enum qib_sdma_events last_event;
500 };
501 
502 struct xmit_wait {
503 	struct timer_list timer;
504 	u64 counter;
505 	u8 flags;
506 	struct cache {
507 		u64 psxmitdata;
508 		u64 psrcvdata;
509 		u64 psxmitpkts;
510 		u64 psrcvpkts;
511 		u64 psxmitwait;
512 	} counter_cache;
513 };
514 
515 /*
516  * The structure below encapsulates data relevant to a physical IB Port.
517  * Current chips support only one such port, but the separation
518  * clarifies things a bit. Note that to conform to IB conventions,
519  * port-numbers are one-based. The first or only port is port1.
520  */
521 struct qib_pportdata {
522 	struct qib_ibport ibport_data;
523 
524 	struct qib_devdata *dd;
525 	struct qib_chippport_specific *cpspec; /* chip-specific per-port */
526 	struct kobject pport_kobj;
527 	struct kobject pport_cc_kobj;
528 	struct kobject sl2vl_kobj;
529 	struct kobject diagc_kobj;
530 
531 	/* GUID for this interface, in network order */
532 	__be64 guid;
533 
534 	/* QIB_POLL, etc. link-state specific flags, per port */
535 	u32 lflags;
536 	/* qib_lflags driver is waiting for */
537 	u32 state_wanted;
538 	spinlock_t lflags_lock;
539 
540 	/* ref count for each pkey */
541 	atomic_t pkeyrefs[4];
542 
543 	/*
544 	 * this address is mapped readonly into user processes so they can
545 	 * get status cheaply, whenever they want.  One qword of status per port
546 	 */
547 	u64 *statusp;
548 
549 	/* SendDMA related entries */
550 
551 	/* read mostly */
552 	struct qib_sdma_desc *sdma_descq;
553 	struct workqueue_struct *qib_wq;
554 	struct qib_sdma_state sdma_state;
555 	dma_addr_t       sdma_descq_phys;
556 	volatile __le64 *sdma_head_dma; /* DMA'ed by chip */
557 	dma_addr_t       sdma_head_phys;
558 	u16                   sdma_descq_cnt;
559 
560 	/* read/write using lock */
561 	spinlock_t            sdma_lock ____cacheline_aligned_in_smp;
562 	struct list_head      sdma_activelist;
563 	struct list_head      sdma_userpending;
564 	u64                   sdma_descq_added;
565 	u64                   sdma_descq_removed;
566 	u16                   sdma_descq_tail;
567 	u16                   sdma_descq_head;
568 	u8                    sdma_generation;
569 	u8                    sdma_intrequest;
570 
571 	struct tasklet_struct sdma_sw_clean_up_task
572 		____cacheline_aligned_in_smp;
573 
574 	wait_queue_head_t state_wait; /* for state_wanted */
575 
576 	/* HoL blocking for SMP replies */
577 	unsigned          hol_state;
578 	struct timer_list hol_timer;
579 
580 	/*
581 	 * Shadow copies of registers; size indicates read access size.
582 	 * Most of them are readonly, but some are write-only register,
583 	 * where we manipulate the bits in the shadow copy, and then write
584 	 * the shadow copy to qlogic_ib.
585 	 *
586 	 * We deliberately make most of these 32 bits, since they have
587 	 * restricted range.  For any that we read, we won't to generate 32
588 	 * bit accesses, since Opteron will generate 2 separate 32 bit HT
589 	 * transactions for a 64 bit read, and we want to avoid unnecessary
590 	 * bus transactions.
591 	 */
592 
593 	/* This is the 64 bit group */
594 	/* last ibcstatus.  opaque outside chip-specific code */
595 	u64 lastibcstat;
596 
597 	/* these are the "32 bit" regs */
598 
599 	/*
600 	 * the following two are 32-bit bitmasks, but {test,clear,set}_bit
601 	 * all expect bit fields to be "unsigned long"
602 	 */
603 	unsigned long p_rcvctrl; /* shadow per-port rcvctrl */
604 	unsigned long p_sendctrl; /* shadow per-port sendctrl */
605 
606 	u32 ibmtu; /* The MTU programmed for this unit */
607 	/*
608 	 * Current max size IB packet (in bytes) including IB headers, that
609 	 * we can send. Changes when ibmtu changes.
610 	 */
611 	u32 ibmaxlen;
612 	/*
613 	 * ibmaxlen at init time, limited by chip and by receive buffer
614 	 * size.  Not changed after init.
615 	 */
616 	u32 init_ibmaxlen;
617 	/* LID programmed for this instance */
618 	u16 lid;
619 	/* list of pkeys programmed; 0 if not set */
620 	u16 pkeys[4];
621 	/* LID mask control */
622 	u8 lmc;
623 	u8 link_width_supported;
624 	u8 link_speed_supported;
625 	u8 link_width_enabled;
626 	u8 link_speed_enabled;
627 	u8 link_width_active;
628 	u8 link_speed_active;
629 	u8 vls_supported;
630 	u8 vls_operational;
631 	/* Rx Polarity inversion (compensate for ~tx on partner) */
632 	u8 rx_pol_inv;
633 
634 	u8 hw_pidx;     /* physical port index */
635 	u8 port;        /* IB port number and index into dd->pports - 1 */
636 
637 	u8 delay_mult;
638 
639 	/* used to override LED behavior */
640 	u8 led_override;  /* Substituted for normal value, if non-zero */
641 	u16 led_override_timeoff; /* delta to next timer event */
642 	u8 led_override_vals[2]; /* Alternates per blink-frame */
643 	u8 led_override_phase; /* Just counts, LSB picks from vals[] */
644 	atomic_t led_override_timer_active;
645 	/* Used to flash LEDs in override mode */
646 	struct timer_list led_override_timer;
647 	struct xmit_wait cong_stats;
648 	struct timer_list symerr_clear_timer;
649 
650 	/* Synchronize access between driver writes and sysfs reads */
651 	spinlock_t cc_shadow_lock
652 		____cacheline_aligned_in_smp;
653 
654 	/* Shadow copy of the congestion control table */
655 	struct cc_table_shadow *ccti_entries_shadow;
656 
657 	/* Shadow copy of the congestion control entries */
658 	struct ib_cc_congestion_setting_attr_shadow *congestion_entries_shadow;
659 
660 	/* List of congestion control table entries */
661 	struct ib_cc_table_entry_shadow *ccti_entries;
662 
663 	/* 16 congestion entries with each entry corresponding to a SL */
664 	struct ib_cc_congestion_entry_shadow *congestion_entries;
665 
666 	/* Maximum number of congestion control entries that the agent expects
667 	 * the manager to send.
668 	 */
669 	u16 cc_supported_table_entries;
670 
671 	/* Total number of congestion control table entries */
672 	u16 total_cct_entry;
673 
674 	/* Bit map identifying service level */
675 	u16 cc_sl_control_map;
676 
677 	/* maximum congestion control table index */
678 	u16 ccti_limit;
679 
680 	/* CA's max number of 64 entry units in the congestion control table */
681 	u8 cc_max_table_entries;
682 };
683 
684 /* Observers. Not to be taken lightly, possibly not to ship. */
685 /*
686  * If a diag read or write is to (bottom <= offset <= top),
687  * the "hoook" is called, allowing, e.g. shadows to be
688  * updated in sync with the driver. struct diag_observer
689  * is the "visible" part.
690  */
691 struct diag_observer;
692 
693 typedef int (*diag_hook) (struct qib_devdata *dd,
694 	const struct diag_observer *op,
695 	u32 offs, u64 *data, u64 mask, int only_32);
696 
697 struct diag_observer {
698 	diag_hook hook;
699 	u32 bottom;
700 	u32 top;
701 };
702 
703 extern int qib_register_observer(struct qib_devdata *dd,
704 	const struct diag_observer *op);
705 
706 /* Only declared here, not defined. Private to diags */
707 struct diag_observer_list_elt;
708 
709 /* device data struct now contains only "general per-device" info.
710  * fields related to a physical IB port are in a qib_pportdata struct,
711  * described above) while fields only used by a particular chip-type are in
712  * a qib_chipdata struct, whose contents are opaque to this file.
713  */
714 struct qib_devdata {
715 	struct qib_ibdev verbs_dev;     /* must be first */
716 	struct list_head list;
717 	/* pointers to related structs for this device */
718 	/* pci access data structure */
719 	struct pci_dev *pcidev;
720 	struct cdev *user_cdev;
721 	struct cdev *diag_cdev;
722 	struct device *user_device;
723 	struct device *diag_device;
724 
725 	/* mem-mapped pointer to base of chip regs */
726 	u64 __iomem *kregbase;
727 	/* end of mem-mapped chip space excluding sendbuf and user regs */
728 	u64 __iomem *kregend;
729 	/* physical address of chip for io_remap, etc. */
730 	resource_size_t physaddr;
731 	/* qib_cfgctxts pointers */
732 	struct qib_ctxtdata **rcd; /* Receive Context Data */
733 
734 	/* qib_pportdata, points to array of (physical) port-specific
735 	 * data structs, indexed by pidx (0..n-1)
736 	 */
737 	struct qib_pportdata *pport;
738 	struct qib_chip_specific *cspec; /* chip-specific */
739 
740 	/* kvirt address of 1st 2k pio buffer */
741 	void __iomem *pio2kbase;
742 	/* kvirt address of 1st 4k pio buffer */
743 	void __iomem *pio4kbase;
744 	/* mem-mapped pointer to base of PIO buffers (if using WC PAT) */
745 	void __iomem *piobase;
746 	/* mem-mapped pointer to base of user chip regs (if using WC PAT) */
747 	u64 __iomem *userbase;
748 	void __iomem *piovl15base; /* base of VL15 buffers, if not WC */
749 	/*
750 	 * points to area where PIOavail registers will be DMA'ed.
751 	 * Has to be on a page of it's own, because the page will be
752 	 * mapped into user program space.  This copy is *ONLY* ever
753 	 * written by DMA, not by the driver!  Need a copy per device
754 	 * when we get to multiple devices
755 	 */
756 	volatile __le64 *pioavailregs_dma; /* DMA'ed by chip */
757 	/* physical address where updates occur */
758 	dma_addr_t pioavailregs_phys;
759 
760 	/* device-specific implementations of functions needed by
761 	 * common code. Contrary to previous consensus, we can't
762 	 * really just point to a device-specific table, because we
763 	 * may need to "bend", e.g. *_f_put_tid
764 	 */
765 	/* fallback to alternate interrupt type if possible */
766 	int (*f_intr_fallback)(struct qib_devdata *);
767 	/* hard reset chip */
768 	int (*f_reset)(struct qib_devdata *);
769 	void (*f_quiet_serdes)(struct qib_pportdata *);
770 	int (*f_bringup_serdes)(struct qib_pportdata *);
771 	int (*f_early_init)(struct qib_devdata *);
772 	void (*f_clear_tids)(struct qib_devdata *, struct qib_ctxtdata *);
773 	void (*f_put_tid)(struct qib_devdata *, u64 __iomem*,
774 				u32, unsigned long);
775 	void (*f_cleanup)(struct qib_devdata *);
776 	void (*f_setextled)(struct qib_pportdata *, u32);
777 	/* fill out chip-specific fields */
778 	int (*f_get_base_info)(struct qib_ctxtdata *, struct qib_base_info *);
779 	/* free irq */
780 	void (*f_free_irq)(struct qib_devdata *);
781 	struct qib_message_header *(*f_get_msgheader)
782 					(struct qib_devdata *, __le32 *);
783 	void (*f_config_ctxts)(struct qib_devdata *);
784 	int (*f_get_ib_cfg)(struct qib_pportdata *, int);
785 	int (*f_set_ib_cfg)(struct qib_pportdata *, int, u32);
786 	int (*f_set_ib_loopback)(struct qib_pportdata *, const char *);
787 	int (*f_get_ib_table)(struct qib_pportdata *, int, void *);
788 	int (*f_set_ib_table)(struct qib_pportdata *, int, void *);
789 	u32 (*f_iblink_state)(u64);
790 	u8 (*f_ibphys_portstate)(u64);
791 	void (*f_xgxs_reset)(struct qib_pportdata *);
792 	/* per chip actions needed for IB Link up/down changes */
793 	int (*f_ib_updown)(struct qib_pportdata *, int, u64);
794 	u32 __iomem *(*f_getsendbuf)(struct qib_pportdata *, u64, u32 *);
795 	/* Read/modify/write of GPIO pins (potentially chip-specific */
796 	int (*f_gpio_mod)(struct qib_devdata *dd, u32 out, u32 dir,
797 		u32 mask);
798 	/* Enable writes to config EEPROM (if supported) */
799 	int (*f_eeprom_wen)(struct qib_devdata *dd, int wen);
800 	/*
801 	 * modify rcvctrl shadow[s] and write to appropriate chip-regs.
802 	 * see above QIB_RCVCTRL_xxx_ENB/DIS for operations.
803 	 * (ctxt == -1) means "all contexts", only meaningful for
804 	 * clearing. Could remove if chip_spec shutdown properly done.
805 	 */
806 	void (*f_rcvctrl)(struct qib_pportdata *, unsigned int op,
807 		int ctxt);
808 	/* Read/modify/write sendctrl appropriately for op and port. */
809 	void (*f_sendctrl)(struct qib_pportdata *, u32 op);
810 	void (*f_set_intr_state)(struct qib_devdata *, u32);
811 	void (*f_set_armlaunch)(struct qib_devdata *, u32);
812 	void (*f_wantpiobuf_intr)(struct qib_devdata *, u32);
813 	int (*f_late_initreg)(struct qib_devdata *);
814 	int (*f_init_sdma_regs)(struct qib_pportdata *);
815 	u16 (*f_sdma_gethead)(struct qib_pportdata *);
816 	int (*f_sdma_busy)(struct qib_pportdata *);
817 	void (*f_sdma_update_tail)(struct qib_pportdata *, u16);
818 	void (*f_sdma_set_desc_cnt)(struct qib_pportdata *, unsigned);
819 	void (*f_sdma_sendctrl)(struct qib_pportdata *, unsigned);
820 	void (*f_sdma_hw_clean_up)(struct qib_pportdata *);
821 	void (*f_sdma_hw_start_up)(struct qib_pportdata *);
822 	void (*f_sdma_init_early)(struct qib_pportdata *);
823 	void (*f_set_cntr_sample)(struct qib_pportdata *, u32, u32);
824 	void (*f_update_usrhead)(struct qib_ctxtdata *, u64, u32, u32, u32);
825 	u32 (*f_hdrqempty)(struct qib_ctxtdata *);
826 	u64 (*f_portcntr)(struct qib_pportdata *, u32);
827 	u32 (*f_read_cntrs)(struct qib_devdata *, loff_t, char **,
828 		u64 **);
829 	u32 (*f_read_portcntrs)(struct qib_devdata *, loff_t, u32,
830 		char **, u64 **);
831 	u32 (*f_setpbc_control)(struct qib_pportdata *, u32, u8, u8);
832 	void (*f_initvl15_bufs)(struct qib_devdata *);
833 	void (*f_init_ctxt)(struct qib_ctxtdata *);
834 	void (*f_txchk_change)(struct qib_devdata *, u32, u32, u32,
835 		struct qib_ctxtdata *);
836 	void (*f_writescratch)(struct qib_devdata *, u32);
837 	int (*f_tempsense_rd)(struct qib_devdata *, int regnum);
838 #ifdef CONFIG_INFINIBAND_QIB_DCA
839 	int (*f_notify_dca)(struct qib_devdata *, unsigned long event);
840 #endif
841 
842 	char *boardname; /* human readable board info */
843 
844 	/* template for writing TIDs  */
845 	u64 tidtemplate;
846 	/* value to write to free TIDs */
847 	u64 tidinvalid;
848 
849 	/* number of registers used for pioavail */
850 	u32 pioavregs;
851 	/* device (not port) flags, basically device capabilities */
852 	u32 flags;
853 	/* last buffer for user use */
854 	u32 lastctxt_piobuf;
855 
856 	/* reset value */
857 	u64 z_int_counter;
858 	/* percpu intcounter */
859 	u64 __percpu *int_counter;
860 
861 	/* pio bufs allocated per ctxt */
862 	u32 pbufsctxt;
863 	/* if remainder on bufs/ctxt, ctxts < extrabuf get 1 extra */
864 	u32 ctxts_extrabuf;
865 	/*
866 	 * number of ctxts configured as max; zero is set to number chip
867 	 * supports, less gives more pio bufs/ctxt, etc.
868 	 */
869 	u32 cfgctxts;
870 	/*
871 	 * number of ctxts available for PSM open
872 	 */
873 	u32 freectxts;
874 
875 	/*
876 	 * hint that we should update pioavailshadow before
877 	 * looking for a PIO buffer
878 	 */
879 	u32 upd_pio_shadow;
880 
881 	/* internal debugging stats */
882 	u32 maxpkts_call;
883 	u32 avgpkts_call;
884 	u64 nopiobufs;
885 
886 	/* PCI Vendor ID (here for NodeInfo) */
887 	u16 vendorid;
888 	/* PCI Device ID (here for NodeInfo) */
889 	u16 deviceid;
890 	/* for write combining settings */
891 	int wc_cookie;
892 	unsigned long wc_base;
893 	unsigned long wc_len;
894 
895 	/* shadow copy of struct page *'s for exp tid pages */
896 	struct page **pageshadow;
897 	/* shadow copy of dma handles for exp tid pages */
898 	dma_addr_t *physshadow;
899 	u64 __iomem *egrtidbase;
900 	spinlock_t sendctrl_lock; /* protect changes to sendctrl shadow */
901 	/* around rcd and (user ctxts) ctxt_cnt use (intr vs free) */
902 	spinlock_t uctxt_lock; /* rcd and user context changes */
903 	/*
904 	 * per unit status, see also portdata statusp
905 	 * mapped readonly into user processes so they can get unit and
906 	 * IB link status cheaply
907 	 */
908 	u64 *devstatusp;
909 	char *freezemsg; /* freeze msg if hw error put chip in freeze */
910 	u32 freezelen; /* max length of freezemsg */
911 	/* timer used to prevent stats overflow, error throttling, etc. */
912 	struct timer_list stats_timer;
913 
914 	/* timer to verify interrupts work, and fallback if possible */
915 	struct timer_list intrchk_timer;
916 	unsigned long ureg_align; /* user register alignment */
917 
918 	/*
919 	 * Protects pioavailshadow, pioavailkernel, pio_need_disarm, and
920 	 * pio_writing.
921 	 */
922 	spinlock_t pioavail_lock;
923 	/*
924 	 * index of last buffer to optimize search for next
925 	 */
926 	u32 last_pio;
927 	/*
928 	 * min kernel pio buffer to optimize search
929 	 */
930 	u32 min_kernel_pio;
931 	/*
932 	 * Shadow copies of registers; size indicates read access size.
933 	 * Most of them are readonly, but some are write-only register,
934 	 * where we manipulate the bits in the shadow copy, and then write
935 	 * the shadow copy to qlogic_ib.
936 	 *
937 	 * We deliberately make most of these 32 bits, since they have
938 	 * restricted range.  For any that we read, we won't to generate 32
939 	 * bit accesses, since Opteron will generate 2 separate 32 bit HT
940 	 * transactions for a 64 bit read, and we want to avoid unnecessary
941 	 * bus transactions.
942 	 */
943 
944 	/* This is the 64 bit group */
945 
946 	unsigned long pioavailshadow[6];
947 	/* bitmap of send buffers available for the kernel to use with PIO. */
948 	unsigned long pioavailkernel[6];
949 	/* bitmap of send buffers which need to be disarmed. */
950 	unsigned long pio_need_disarm[3];
951 	/* bitmap of send buffers which are being written to. */
952 	unsigned long pio_writing[3];
953 	/* kr_revision shadow */
954 	u64 revision;
955 	/* Base GUID for device (from eeprom, network order) */
956 	__be64 base_guid;
957 
958 	/*
959 	 * kr_sendpiobufbase value (chip offset of pio buffers), and the
960 	 * base of the 2KB buffer s(user processes only use 2K)
961 	 */
962 	u64 piobufbase;
963 	u32 pio2k_bufbase;
964 
965 	/* these are the "32 bit" regs */
966 
967 	/* number of GUIDs in the flash for this interface */
968 	u32 nguid;
969 	/*
970 	 * the following two are 32-bit bitmasks, but {test,clear,set}_bit
971 	 * all expect bit fields to be "unsigned long"
972 	 */
973 	unsigned long rcvctrl; /* shadow per device rcvctrl */
974 	unsigned long sendctrl; /* shadow per device sendctrl */
975 
976 	/* value we put in kr_rcvhdrcnt */
977 	u32 rcvhdrcnt;
978 	/* value we put in kr_rcvhdrsize */
979 	u32 rcvhdrsize;
980 	/* value we put in kr_rcvhdrentsize */
981 	u32 rcvhdrentsize;
982 	/* kr_ctxtcnt value */
983 	u32 ctxtcnt;
984 	/* kr_pagealign value */
985 	u32 palign;
986 	/* number of "2KB" PIO buffers */
987 	u32 piobcnt2k;
988 	/* size in bytes of "2KB" PIO buffers */
989 	u32 piosize2k;
990 	/* max usable size in dwords of a "2KB" PIO buffer before going "4KB" */
991 	u32 piosize2kmax_dwords;
992 	/* number of "4KB" PIO buffers */
993 	u32 piobcnt4k;
994 	/* size in bytes of "4KB" PIO buffers */
995 	u32 piosize4k;
996 	/* kr_rcvegrbase value */
997 	u32 rcvegrbase;
998 	/* kr_rcvtidbase value */
999 	u32 rcvtidbase;
1000 	/* kr_rcvtidcnt value */
1001 	u32 rcvtidcnt;
1002 	/* kr_userregbase */
1003 	u32 uregbase;
1004 	/* shadow the control register contents */
1005 	u32 control;
1006 
1007 	/* chip address space used by 4k pio buffers */
1008 	u32 align4k;
1009 	/* size of each rcvegrbuffer */
1010 	u16 rcvegrbufsize;
1011 	/* log2 of above */
1012 	u16 rcvegrbufsize_shift;
1013 	/* localbus width (1, 2,4,8,16,32) from config space  */
1014 	u32 lbus_width;
1015 	/* localbus speed in MHz */
1016 	u32 lbus_speed;
1017 	int unit; /* unit # of this chip */
1018 
1019 	/* start of CHIP_SPEC move to chipspec, but need code changes */
1020 	/* low and high portions of MSI capability/vector */
1021 	u32 msi_lo;
1022 	/* saved after PCIe init for restore after reset */
1023 	u32 msi_hi;
1024 	/* MSI data (vector) saved for restore */
1025 	u16 msi_data;
1026 	/* so we can rewrite it after a chip reset */
1027 	u32 pcibar0;
1028 	/* so we can rewrite it after a chip reset */
1029 	u32 pcibar1;
1030 	u64 rhdrhead_intr_off;
1031 
1032 	/*
1033 	 * ASCII serial number, from flash, large enough for original
1034 	 * all digit strings, and longer QLogic serial number format
1035 	 */
1036 	u8 serial[16];
1037 	/* human readable board version */
1038 	u8 boardversion[96];
1039 	u8 lbus_info[32]; /* human readable localbus info */
1040 	/* chip major rev, from qib_revision */
1041 	u8 majrev;
1042 	/* chip minor rev, from qib_revision */
1043 	u8 minrev;
1044 
1045 	/* Misc small ints */
1046 	/* Number of physical ports available */
1047 	u8 num_pports;
1048 	/* Lowest context number which can be used by user processes */
1049 	u8 first_user_ctxt;
1050 	u8 n_krcv_queues;
1051 	u8 qpn_mask;
1052 	u8 skip_kctxt_mask;
1053 
1054 	u16 rhf_offset; /* offset of RHF within receive header entry */
1055 
1056 	/*
1057 	 * GPIO pins for twsi-connected devices, and device code for eeprom
1058 	 */
1059 	u8 gpio_sda_num;
1060 	u8 gpio_scl_num;
1061 	u8 twsi_eeprom_dev;
1062 	u8 board_atten;
1063 
1064 	/* Support (including locks) for EEPROM logging of errors and time */
1065 	/* control access to actual counters, timer */
1066 	spinlock_t eep_st_lock;
1067 	/* control high-level access to EEPROM */
1068 	struct mutex eep_lock;
1069 	uint64_t traffic_wds;
1070 	struct qib_diag_client *diag_client;
1071 	spinlock_t qib_diag_trans_lock; /* protect diag observer ops */
1072 	struct diag_observer_list_elt *diag_observer_list;
1073 
1074 	u8 psxmitwait_supported;
1075 	/* cycle length of PS* counters in HW (in picoseconds) */
1076 	u16 psxmitwait_check_rate;
1077 	/* high volume overflow errors defered to tasklet */
1078 	struct tasklet_struct error_tasklet;
1079 
1080 	int assigned_node_id; /* NUMA node closest to HCA */
1081 };
1082 
1083 /* hol_state values */
1084 #define QIB_HOL_UP       0
1085 #define QIB_HOL_INIT     1
1086 
1087 #define QIB_SDMA_SENDCTRL_OP_ENABLE    (1U << 0)
1088 #define QIB_SDMA_SENDCTRL_OP_INTENABLE (1U << 1)
1089 #define QIB_SDMA_SENDCTRL_OP_HALT      (1U << 2)
1090 #define QIB_SDMA_SENDCTRL_OP_CLEANUP   (1U << 3)
1091 #define QIB_SDMA_SENDCTRL_OP_DRAIN     (1U << 4)
1092 
1093 /* operation types for f_txchk_change() */
1094 #define TXCHK_CHG_TYPE_DIS1  3
1095 #define TXCHK_CHG_TYPE_ENAB1 2
1096 #define TXCHK_CHG_TYPE_KERN  1
1097 #define TXCHK_CHG_TYPE_USER  0
1098 
1099 #define QIB_CHASE_TIME msecs_to_jiffies(145)
1100 #define QIB_CHASE_DIS_TIME msecs_to_jiffies(160)
1101 
1102 /* Private data for file operations */
1103 struct qib_filedata {
1104 	struct qib_ctxtdata *rcd;
1105 	unsigned subctxt;
1106 	unsigned tidcursor;
1107 	struct qib_user_sdma_queue *pq;
1108 	int rec_cpu_num; /* for cpu affinity; -1 if none */
1109 };
1110 
1111 extern struct list_head qib_dev_list;
1112 extern spinlock_t qib_devs_lock;
1113 extern struct qib_devdata *qib_lookup(int unit);
1114 extern u32 qib_cpulist_count;
1115 extern unsigned long *qib_cpulist;
1116 extern unsigned qib_cc_table_size;
1117 
1118 int qib_init(struct qib_devdata *, int);
1119 int init_chip_wc_pat(struct qib_devdata *dd, u32);
1120 int qib_enable_wc(struct qib_devdata *dd);
1121 void qib_disable_wc(struct qib_devdata *dd);
1122 int qib_count_units(int *npresentp, int *nupp);
1123 int qib_count_active_units(void);
1124 
1125 int qib_cdev_init(int minor, const char *name,
1126 		  const struct file_operations *fops,
1127 		  struct cdev **cdevp, struct device **devp);
1128 void qib_cdev_cleanup(struct cdev **cdevp, struct device **devp);
1129 int qib_dev_init(void);
1130 void qib_dev_cleanup(void);
1131 
1132 int qib_diag_add(struct qib_devdata *);
1133 void qib_diag_remove(struct qib_devdata *);
1134 void qib_handle_e_ibstatuschanged(struct qib_pportdata *, u64);
1135 void qib_sdma_update_tail(struct qib_pportdata *, u16); /* hold sdma_lock */
1136 
1137 int qib_decode_err(struct qib_devdata *dd, char *buf, size_t blen, u64 err);
1138 void qib_bad_intrstatus(struct qib_devdata *);
1139 void qib_handle_urcv(struct qib_devdata *, u64);
1140 
1141 /* clean up any per-chip chip-specific stuff */
1142 void qib_chip_cleanup(struct qib_devdata *);
1143 /* clean up any chip type-specific stuff */
1144 void qib_chip_done(void);
1145 
1146 /* check to see if we have to force ordering for write combining */
1147 int qib_unordered_wc(void);
1148 void qib_pio_copy(void __iomem *to, const void *from, size_t count);
1149 
1150 void qib_disarm_piobufs(struct qib_devdata *, unsigned, unsigned);
1151 int qib_disarm_piobufs_ifneeded(struct qib_ctxtdata *);
1152 void qib_disarm_piobufs_set(struct qib_devdata *, unsigned long *, unsigned);
1153 void qib_cancel_sends(struct qib_pportdata *);
1154 
1155 int qib_create_rcvhdrq(struct qib_devdata *, struct qib_ctxtdata *);
1156 int qib_setup_eagerbufs(struct qib_ctxtdata *);
1157 void qib_set_ctxtcnt(struct qib_devdata *);
1158 int qib_create_ctxts(struct qib_devdata *dd);
1159 struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *, u32, int);
1160 int qib_init_pportdata(struct qib_pportdata *, struct qib_devdata *, u8, u8);
1161 void qib_free_ctxtdata(struct qib_devdata *, struct qib_ctxtdata *);
1162 
1163 u32 qib_kreceive(struct qib_ctxtdata *, u32 *, u32 *);
1164 int qib_reset_device(int);
1165 int qib_wait_linkstate(struct qib_pportdata *, u32, int);
1166 int qib_set_linkstate(struct qib_pportdata *, u8);
1167 int qib_set_mtu(struct qib_pportdata *, u16);
1168 int qib_set_lid(struct qib_pportdata *, u32, u8);
1169 void qib_hol_down(struct qib_pportdata *);
1170 void qib_hol_init(struct qib_pportdata *);
1171 void qib_hol_up(struct qib_pportdata *);
1172 void qib_hol_event(struct timer_list *);
1173 void qib_disable_after_error(struct qib_devdata *);
1174 int qib_set_uevent_bits(struct qib_pportdata *, const int);
1175 
1176 /* for use in system calls, where we want to know device type, etc. */
1177 #define ctxt_fp(fp) \
1178 	(((struct qib_filedata *)(fp)->private_data)->rcd)
1179 #define subctxt_fp(fp) \
1180 	(((struct qib_filedata *)(fp)->private_data)->subctxt)
1181 #define tidcursor_fp(fp) \
1182 	(((struct qib_filedata *)(fp)->private_data)->tidcursor)
1183 #define user_sdma_queue_fp(fp) \
1184 	(((struct qib_filedata *)(fp)->private_data)->pq)
1185 
1186 static inline struct qib_devdata *dd_from_ppd(struct qib_pportdata *ppd)
1187 {
1188 	return ppd->dd;
1189 }
1190 
1191 static inline struct qib_devdata *dd_from_dev(struct qib_ibdev *dev)
1192 {
1193 	return container_of(dev, struct qib_devdata, verbs_dev);
1194 }
1195 
1196 static inline struct qib_devdata *dd_from_ibdev(struct ib_device *ibdev)
1197 {
1198 	return dd_from_dev(to_idev(ibdev));
1199 }
1200 
1201 static inline struct qib_pportdata *ppd_from_ibp(struct qib_ibport *ibp)
1202 {
1203 	return container_of(ibp, struct qib_pportdata, ibport_data);
1204 }
1205 
1206 static inline struct qib_ibport *to_iport(struct ib_device *ibdev, u8 port)
1207 {
1208 	struct qib_devdata *dd = dd_from_ibdev(ibdev);
1209 	unsigned pidx = port - 1; /* IB number port from 1, hdw from 0 */
1210 
1211 	WARN_ON(pidx >= dd->num_pports);
1212 	return &dd->pport[pidx].ibport_data;
1213 }
1214 
1215 /*
1216  * values for dd->flags (_device_ related flags) and
1217  */
1218 #define QIB_HAS_LINK_LATENCY  0x1 /* supports link latency (IB 1.2) */
1219 #define QIB_INITTED           0x2 /* chip and driver up and initted */
1220 #define QIB_DOING_RESET       0x4  /* in the middle of doing chip reset */
1221 #define QIB_PRESENT           0x8  /* chip accesses can be done */
1222 #define QIB_PIO_FLUSH_WC      0x10 /* Needs Write combining flush for PIO */
1223 #define QIB_HAS_THRESH_UPDATE 0x40
1224 #define QIB_HAS_SDMA_TIMEOUT  0x80
1225 #define QIB_USE_SPCL_TRIG     0x100 /* SpecialTrigger launch enabled */
1226 #define QIB_NODMA_RTAIL       0x200 /* rcvhdrtail register DMA enabled */
1227 #define QIB_HAS_INTX          0x800 /* Supports INTx interrupts */
1228 #define QIB_HAS_SEND_DMA      0x1000 /* Supports Send DMA */
1229 #define QIB_HAS_VLSUPP        0x2000 /* Supports multiple VLs; PBC different */
1230 #define QIB_HAS_HDRSUPP       0x4000 /* Supports header suppression */
1231 #define QIB_BADINTR           0x8000 /* severe interrupt problems */
1232 #define QIB_DCA_ENABLED       0x10000 /* Direct Cache Access enabled */
1233 #define QIB_HAS_QSFP          0x20000 /* device (card instance) has QSFP */
1234 
1235 /*
1236  * values for ppd->lflags (_ib_port_ related flags)
1237  */
1238 #define QIBL_LINKV             0x1 /* IB link state valid */
1239 #define QIBL_LINKDOWN          0x8 /* IB link is down */
1240 #define QIBL_LINKINIT          0x10 /* IB link level is up */
1241 #define QIBL_LINKARMED         0x20 /* IB link is ARMED */
1242 #define QIBL_LINKACTIVE        0x40 /* IB link is ACTIVE */
1243 /* leave a gap for more IB-link state */
1244 #define QIBL_IB_AUTONEG_INPROG 0x1000 /* non-IBTA DDR/QDR neg active */
1245 #define QIBL_IB_AUTONEG_FAILED 0x2000 /* non-IBTA DDR/QDR neg failed */
1246 #define QIBL_IB_LINK_DISABLED  0x4000 /* Linkdown-disable forced,
1247 				       * Do not try to bring up */
1248 #define QIBL_IB_FORCE_NOTIFY   0x8000 /* force notify on next ib change */
1249 
1250 /* IB dword length mask in PBC (lower 11 bits); same for all chips */
1251 #define QIB_PBC_LENGTH_MASK                     ((1 << 11) - 1)
1252 
1253 
1254 /* ctxt_flag bit offsets */
1255 		/* waiting for a packet to arrive */
1256 #define QIB_CTXT_WAITING_RCV   2
1257 		/* master has not finished initializing */
1258 #define QIB_CTXT_MASTER_UNINIT 4
1259 		/* waiting for an urgent packet to arrive */
1260 #define QIB_CTXT_WAITING_URG 5
1261 
1262 /* free up any allocated data at closes */
1263 void qib_free_data(struct qib_ctxtdata *dd);
1264 void qib_chg_pioavailkernel(struct qib_devdata *, unsigned, unsigned,
1265 			    u32, struct qib_ctxtdata *);
1266 struct qib_devdata *qib_init_iba7322_funcs(struct pci_dev *,
1267 					   const struct pci_device_id *);
1268 struct qib_devdata *qib_init_iba7220_funcs(struct pci_dev *,
1269 					   const struct pci_device_id *);
1270 struct qib_devdata *qib_init_iba6120_funcs(struct pci_dev *,
1271 					   const struct pci_device_id *);
1272 void qib_free_devdata(struct qib_devdata *);
1273 struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra);
1274 
1275 #define QIB_TWSI_NO_DEV 0xFF
1276 /* Below qib_twsi_ functions must be called with eep_lock held */
1277 int qib_twsi_reset(struct qib_devdata *dd);
1278 int qib_twsi_blk_rd(struct qib_devdata *dd, int dev, int addr, void *buffer,
1279 		    int len);
1280 int qib_twsi_blk_wr(struct qib_devdata *dd, int dev, int addr,
1281 		    const void *buffer, int len);
1282 void qib_get_eeprom_info(struct qib_devdata *);
1283 void qib_dump_lookup_output_queue(struct qib_devdata *);
1284 void qib_force_pio_avail_update(struct qib_devdata *);
1285 void qib_clear_symerror_on_linkup(struct timer_list *t);
1286 
1287 /*
1288  * Set LED override, only the two LSBs have "public" meaning, but
1289  * any non-zero value substitutes them for the Link and LinkTrain
1290  * LED states.
1291  */
1292 #define QIB_LED_PHYS 1 /* Physical (linktraining) GREEN LED */
1293 #define QIB_LED_LOG 2  /* Logical (link) YELLOW LED */
1294 void qib_set_led_override(struct qib_pportdata *ppd, unsigned int val);
1295 
1296 /* send dma routines */
1297 int qib_setup_sdma(struct qib_pportdata *);
1298 void qib_teardown_sdma(struct qib_pportdata *);
1299 void __qib_sdma_intr(struct qib_pportdata *);
1300 void qib_sdma_intr(struct qib_pportdata *);
1301 void qib_user_sdma_send_desc(struct qib_pportdata *dd,
1302 			struct list_head *pktlist);
1303 int qib_sdma_verbs_send(struct qib_pportdata *, struct rvt_sge_state *,
1304 			u32, struct qib_verbs_txreq *);
1305 /* ppd->sdma_lock should be locked before calling this. */
1306 int qib_sdma_make_progress(struct qib_pportdata *dd);
1307 
1308 static inline int qib_sdma_empty(const struct qib_pportdata *ppd)
1309 {
1310 	return ppd->sdma_descq_added == ppd->sdma_descq_removed;
1311 }
1312 
1313 /* must be called under qib_sdma_lock */
1314 static inline u16 qib_sdma_descq_freecnt(const struct qib_pportdata *ppd)
1315 {
1316 	return ppd->sdma_descq_cnt -
1317 		(ppd->sdma_descq_added - ppd->sdma_descq_removed) - 1;
1318 }
1319 
1320 static inline int __qib_sdma_running(struct qib_pportdata *ppd)
1321 {
1322 	return ppd->sdma_state.current_state == qib_sdma_state_s99_running;
1323 }
1324 int qib_sdma_running(struct qib_pportdata *);
1325 void dump_sdma_state(struct qib_pportdata *ppd);
1326 void __qib_sdma_process_event(struct qib_pportdata *, enum qib_sdma_events);
1327 void qib_sdma_process_event(struct qib_pportdata *, enum qib_sdma_events);
1328 
1329 /*
1330  * number of words used for protocol header if not set by qib_userinit();
1331  */
1332 #define QIB_DFLT_RCVHDRSIZE 9
1333 
1334 /*
1335  * We need to be able to handle an IB header of at least 24 dwords.
1336  * We need the rcvhdrq large enough to handle largest IB header, but
1337  * still have room for a 2KB MTU standard IB packet.
1338  * Additionally, some processor/memory controller combinations
1339  * benefit quite strongly from having the DMA'ed data be cacheline
1340  * aligned and a cacheline multiple, so we set the size to 32 dwords
1341  * (2 64-byte primary cachelines for pretty much all processors of
1342  * interest).  The alignment hurts nothing, other than using somewhat
1343  * more memory.
1344  */
1345 #define QIB_RCVHDR_ENTSIZE 32
1346 
1347 int qib_get_user_pages(unsigned long, size_t, struct page **);
1348 void qib_release_user_pages(struct page **, size_t);
1349 int qib_eeprom_read(struct qib_devdata *, u8, void *, int);
1350 int qib_eeprom_write(struct qib_devdata *, u8, const void *, int);
1351 u32 __iomem *qib_getsendbuf_range(struct qib_devdata *, u32 *, u32, u32);
1352 void qib_sendbuf_done(struct qib_devdata *, unsigned);
1353 
1354 static inline void qib_clear_rcvhdrtail(const struct qib_ctxtdata *rcd)
1355 {
1356 	*((u64 *) rcd->rcvhdrtail_kvaddr) = 0ULL;
1357 }
1358 
1359 static inline u32 qib_get_rcvhdrtail(const struct qib_ctxtdata *rcd)
1360 {
1361 	/*
1362 	 * volatile because it's a DMA target from the chip, routine is
1363 	 * inlined, and don't want register caching or reordering.
1364 	 */
1365 	return (u32) le64_to_cpu(
1366 		*((volatile __le64 *)rcd->rcvhdrtail_kvaddr)); /* DMA'ed */
1367 }
1368 
1369 static inline u32 qib_get_hdrqtail(const struct qib_ctxtdata *rcd)
1370 {
1371 	const struct qib_devdata *dd = rcd->dd;
1372 	u32 hdrqtail;
1373 
1374 	if (dd->flags & QIB_NODMA_RTAIL) {
1375 		__le32 *rhf_addr;
1376 		u32 seq;
1377 
1378 		rhf_addr = (__le32 *) rcd->rcvhdrq +
1379 			rcd->head + dd->rhf_offset;
1380 		seq = qib_hdrget_seq(rhf_addr);
1381 		hdrqtail = rcd->head;
1382 		if (seq == rcd->seq_cnt)
1383 			hdrqtail++;
1384 	} else
1385 		hdrqtail = qib_get_rcvhdrtail(rcd);
1386 
1387 	return hdrqtail;
1388 }
1389 
1390 /*
1391  * sysfs interface.
1392  */
1393 
1394 extern const char ib_qib_version[];
1395 
1396 int qib_device_create(struct qib_devdata *);
1397 void qib_device_remove(struct qib_devdata *);
1398 
1399 int qib_create_port_files(struct ib_device *ibdev, u8 port_num,
1400 			  struct kobject *kobj);
1401 int qib_verbs_register_sysfs(struct qib_devdata *);
1402 void qib_verbs_unregister_sysfs(struct qib_devdata *);
1403 /* Hook for sysfs read of QSFP */
1404 extern int qib_qsfp_dump(struct qib_pportdata *ppd, char *buf, int len);
1405 
1406 int __init qib_init_qibfs(void);
1407 int __exit qib_exit_qibfs(void);
1408 
1409 int qibfs_add(struct qib_devdata *);
1410 int qibfs_remove(struct qib_devdata *);
1411 
1412 int qib_pcie_init(struct pci_dev *, const struct pci_device_id *);
1413 int qib_pcie_ddinit(struct qib_devdata *, struct pci_dev *,
1414 		    const struct pci_device_id *);
1415 void qib_pcie_ddcleanup(struct qib_devdata *);
1416 int qib_pcie_params(struct qib_devdata *dd, u32 minw, u32 *nent);
1417 void qib_free_irq(struct qib_devdata *dd);
1418 int qib_reinit_intr(struct qib_devdata *dd);
1419 void qib_pcie_getcmd(struct qib_devdata *, u16 *, u8 *, u8 *);
1420 void qib_pcie_reenable(struct qib_devdata *, u16, u8, u8);
1421 /* interrupts for device */
1422 u64 qib_int_counter(struct qib_devdata *);
1423 /* interrupt for all devices */
1424 u64 qib_sps_ints(void);
1425 
1426 /*
1427  * dma_addr wrappers - all 0's invalid for hw
1428  */
1429 dma_addr_t qib_map_page(struct pci_dev *, struct page *, unsigned long,
1430 			  size_t, int);
1431 const char *qib_get_unit_name(int unit);
1432 const char *qib_get_card_name(struct rvt_dev_info *rdi);
1433 struct pci_dev *qib_get_pci_dev(struct rvt_dev_info *rdi);
1434 
1435 /*
1436  * Flush write combining store buffers (if present) and perform a write
1437  * barrier.
1438  */
1439 static inline void qib_flush_wc(void)
1440 {
1441 #if defined(CONFIG_X86_64)
1442 	asm volatile("sfence" : : : "memory");
1443 #else
1444 	wmb(); /* no reorder around wc flush */
1445 #endif
1446 }
1447 
1448 /* global module parameter variables */
1449 extern unsigned qib_ibmtu;
1450 extern ushort qib_cfgctxts;
1451 extern ushort qib_num_cfg_vls;
1452 extern ushort qib_mini_init; /* If set, do few (ideally 0) writes to chip */
1453 extern unsigned qib_n_krcv_queues;
1454 extern unsigned qib_sdma_fetch_arb;
1455 extern unsigned qib_compat_ddr_negotiate;
1456 extern int qib_special_trigger;
1457 extern unsigned qib_numa_aware;
1458 
1459 extern struct mutex qib_mutex;
1460 
1461 /* Number of seconds before our card status check...  */
1462 #define STATUS_TIMEOUT 60
1463 
1464 #define QIB_DRV_NAME            "ib_qib"
1465 #define QIB_USER_MINOR_BASE     0
1466 #define QIB_TRACE_MINOR         127
1467 #define QIB_DIAGPKT_MINOR       128
1468 #define QIB_DIAG_MINOR_BASE     129
1469 #define QIB_NMINORS             255
1470 
1471 #define PCI_VENDOR_ID_PATHSCALE 0x1fc1
1472 #define PCI_VENDOR_ID_QLOGIC 0x1077
1473 #define PCI_DEVICE_ID_QLOGIC_IB_6120 0x10
1474 #define PCI_DEVICE_ID_QLOGIC_IB_7220 0x7220
1475 #define PCI_DEVICE_ID_QLOGIC_IB_7322 0x7322
1476 
1477 /*
1478  * qib_early_err is used (only!) to print early errors before devdata is
1479  * allocated, or when dd->pcidev may not be valid, and at the tail end of
1480  * cleanup when devdata may have been freed, etc.  qib_dev_porterr is
1481  * the same as qib_dev_err, but is used when the message really needs
1482  * the IB port# to be definitive as to what's happening..
1483  * All of these go to the trace log, and the trace log entry is done
1484  * first to avoid possible serial port delays from printk.
1485  */
1486 #define qib_early_err(dev, fmt, ...) \
1487 	dev_err(dev, fmt, ##__VA_ARGS__)
1488 
1489 #define qib_dev_err(dd, fmt, ...) \
1490 	dev_err(&(dd)->pcidev->dev, "%s: " fmt, \
1491 		qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
1492 
1493 #define qib_dev_warn(dd, fmt, ...) \
1494 	dev_warn(&(dd)->pcidev->dev, "%s: " fmt, \
1495 		qib_get_unit_name((dd)->unit), ##__VA_ARGS__)
1496 
1497 #define qib_dev_porterr(dd, port, fmt, ...) \
1498 	dev_err(&(dd)->pcidev->dev, "%s: IB%u:%u " fmt, \
1499 		qib_get_unit_name((dd)->unit), (dd)->unit, (port), \
1500 		##__VA_ARGS__)
1501 
1502 #define qib_devinfo(pcidev, fmt, ...) \
1503 	dev_info(&(pcidev)->dev, fmt, ##__VA_ARGS__)
1504 
1505 /*
1506  * this is used for formatting hw error messages...
1507  */
1508 struct qib_hwerror_msgs {
1509 	u64 mask;
1510 	const char *msg;
1511 	size_t sz;
1512 };
1513 
1514 #define QLOGIC_IB_HWE_MSG(a, b) { .mask = a, .msg = b }
1515 
1516 /* in qib_intr.c... */
1517 void qib_format_hwerrors(u64 hwerrs,
1518 			 const struct qib_hwerror_msgs *hwerrmsgs,
1519 			 size_t nhwerrmsgs, char *msg, size_t lmsg);
1520 
1521 void qib_stop_send_queue(struct rvt_qp *qp);
1522 void qib_quiesce_qp(struct rvt_qp *qp);
1523 void qib_flush_qp_waiters(struct rvt_qp *qp);
1524 int qib_mtu_to_path_mtu(u32 mtu);
1525 u32 qib_mtu_from_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp, u32 pmtu);
1526 void qib_notify_error_qp(struct rvt_qp *qp);
1527 int qib_get_pmtu_from_attr(struct rvt_dev_info *rdi, struct rvt_qp *qp,
1528 			   struct ib_qp_attr *attr);
1529 
1530 #endif                          /* _QIB_KERNEL_H */
1531