1f931551bSRalph Campbell /*
2e2eed58bSVinit Agnihotri * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved.
3551ace12SMike Marciniszyn * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
4f931551bSRalph Campbell * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
5f931551bSRalph Campbell *
6f931551bSRalph Campbell * This software is available to you under a choice of one of two
7f931551bSRalph Campbell * licenses. You may choose to be licensed under the terms of the GNU
8f931551bSRalph Campbell * General Public License (GPL) Version 2, available from the file
9f931551bSRalph Campbell * COPYING in the main directory of this source tree, or the
10f931551bSRalph Campbell * OpenIB.org BSD license below:
11f931551bSRalph Campbell *
12f931551bSRalph Campbell * Redistribution and use in source and binary forms, with or
13f931551bSRalph Campbell * without modification, are permitted provided that the following
14f931551bSRalph Campbell * conditions are met:
15f931551bSRalph Campbell *
16f931551bSRalph Campbell * - Redistributions of source code must retain the above
17f931551bSRalph Campbell * copyright notice, this list of conditions and the following
18f931551bSRalph Campbell * disclaimer.
19f931551bSRalph Campbell *
20f931551bSRalph Campbell * - Redistributions in binary form must reproduce the above
21f931551bSRalph Campbell * copyright notice, this list of conditions and the following
22f931551bSRalph Campbell * disclaimer in the documentation and/or other materials
23f931551bSRalph Campbell * provided with the distribution.
24f931551bSRalph Campbell *
25f931551bSRalph Campbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26f931551bSRalph Campbell * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27f931551bSRalph Campbell * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28f931551bSRalph Campbell * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29f931551bSRalph Campbell * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30f931551bSRalph Campbell * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31f931551bSRalph Campbell * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32f931551bSRalph Campbell * SOFTWARE.
33f931551bSRalph Campbell */
34f931551bSRalph Campbell
35f931551bSRalph Campbell #include <linux/pci.h>
36f931551bSRalph Campbell #include <linux/netdevice.h>
37f931551bSRalph Campbell #include <linux/vmalloc.h>
38f931551bSRalph Campbell #include <linux/delay.h>
39e4dd23d7SPaul Gortmaker #include <linux/module.h>
407fac3301SMike Marciniszyn #include <linux/printk.h>
418469ba39SMike Marciniszyn #ifdef CONFIG_INFINIBAND_QIB_DCA
428469ba39SMike Marciniszyn #include <linux/dca.h>
438469ba39SMike Marciniszyn #endif
442dc05ab5SDennis Dalessandro #include <rdma/rdma_vt.h>
45f931551bSRalph Campbell
46f931551bSRalph Campbell #include "qib.h"
47f931551bSRalph Campbell #include "qib_common.h"
4836a8f01cSMike Marciniszyn #include "qib_mad.h"
49ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
50ddb88765SMike Marciniszyn #include "qib_debugfs.h"
51ddb88765SMike Marciniszyn #include "qib_verbs.h"
52ddb88765SMike Marciniszyn #endif
53f931551bSRalph Campbell
547fac3301SMike Marciniszyn #undef pr_fmt
557fac3301SMike Marciniszyn #define pr_fmt(fmt) QIB_DRV_NAME ": " fmt
567fac3301SMike Marciniszyn
57f931551bSRalph Campbell /*
58f931551bSRalph Campbell * min buffers we want to have per context, after driver
59f931551bSRalph Campbell */
60f931551bSRalph Campbell #define QIB_MIN_USER_CTXT_BUFCNT 7
61f931551bSRalph Campbell
62f931551bSRalph Campbell #define QLOGIC_IB_R_SOFTWARE_MASK 0xFF
63f931551bSRalph Campbell #define QLOGIC_IB_R_SOFTWARE_SHIFT 24
64f931551bSRalph Campbell #define QLOGIC_IB_R_EMULATOR_MASK (1ULL<<62)
65f931551bSRalph Campbell
66f931551bSRalph Campbell /*
67f931551bSRalph Campbell * Number of ctxts we are configured to use (to allow for more pio
68f931551bSRalph Campbell * buffers per ctxt, etc.) Zero means use chip value.
69f931551bSRalph Campbell */
70f931551bSRalph Campbell ushort qib_cfgctxts;
71f931551bSRalph Campbell module_param_named(cfgctxts, qib_cfgctxts, ushort, S_IRUGO);
72f931551bSRalph Campbell MODULE_PARM_DESC(cfgctxts, "Set max number of contexts to use");
73f931551bSRalph Campbell
74e0f30bacSRamkrishna Vepa unsigned qib_numa_aware;
75e0f30bacSRamkrishna Vepa module_param_named(numa_aware, qib_numa_aware, uint, S_IRUGO);
76e0f30bacSRamkrishna Vepa MODULE_PARM_DESC(numa_aware,
77e0f30bacSRamkrishna Vepa "0 -> PSM allocation close to HCA, 1 -> PSM allocation local to process");
78e0f30bacSRamkrishna Vepa
79f931551bSRalph Campbell /*
80f931551bSRalph Campbell * If set, do not write to any regs if avoidable, hack to allow
81f931551bSRalph Campbell * check for deranged default register values.
82f931551bSRalph Campbell */
83f931551bSRalph Campbell ushort qib_mini_init;
84f931551bSRalph Campbell module_param_named(mini_init, qib_mini_init, ushort, S_IRUGO);
85f931551bSRalph Campbell MODULE_PARM_DESC(mini_init, "If set, do minimal diag init");
86f931551bSRalph Campbell
87f931551bSRalph Campbell unsigned qib_n_krcv_queues;
88f931551bSRalph Campbell module_param_named(krcvqs, qib_n_krcv_queues, uint, S_IRUGO);
89f931551bSRalph Campbell MODULE_PARM_DESC(krcvqs, "number of kernel receive queues per IB port");
90f931551bSRalph Campbell
9136a8f01cSMike Marciniszyn unsigned qib_cc_table_size;
9236a8f01cSMike Marciniszyn module_param_named(cc_table_size, qib_cc_table_size, uint, S_IRUGO);
9336a8f01cSMike Marciniszyn MODULE_PARM_DESC(cc_table_size, "Congestion control table entries 0 (CCA disabled - default), min = 128, max = 1984");
94f931551bSRalph Campbell
954037c92fSKees Cook static void verify_interrupt(struct timer_list *);
96f931551bSRalph Campbell
97059d48fbSMatthew Wilcox DEFINE_XARRAY_FLAGS(qib_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
98f931551bSRalph Campbell u32 qib_cpulist_count;
99f931551bSRalph Campbell unsigned long *qib_cpulist;
100f931551bSRalph Campbell
101f931551bSRalph Campbell /* set number of contexts we'll actually use */
qib_set_ctxtcnt(struct qib_devdata * dd)102f931551bSRalph Campbell void qib_set_ctxtcnt(struct qib_devdata *dd)
103f931551bSRalph Campbell {
1045dbbcb97SMike Marciniszyn if (!qib_cfgctxts) {
1050502f94cSRalph Campbell dd->cfgctxts = dd->first_user_ctxt + num_online_cpus();
1065dbbcb97SMike Marciniszyn if (dd->cfgctxts > dd->ctxtcnt)
1075dbbcb97SMike Marciniszyn dd->cfgctxts = dd->ctxtcnt;
1085dbbcb97SMike Marciniszyn } else if (qib_cfgctxts < dd->num_pports)
109f931551bSRalph Campbell dd->cfgctxts = dd->ctxtcnt;
110f931551bSRalph Campbell else if (qib_cfgctxts <= dd->ctxtcnt)
111f931551bSRalph Campbell dd->cfgctxts = qib_cfgctxts;
112f931551bSRalph Campbell else
113f931551bSRalph Campbell dd->cfgctxts = dd->ctxtcnt;
1146ceaadeeSMitko Haralanov dd->freectxts = (dd->first_user_ctxt > dd->cfgctxts) ? 0 :
1156ceaadeeSMitko Haralanov dd->cfgctxts - dd->first_user_ctxt;
116f931551bSRalph Campbell }
117f931551bSRalph Campbell
118f931551bSRalph Campbell /*
119f931551bSRalph Campbell * Common code for creating the receive context array.
120f931551bSRalph Campbell */
qib_create_ctxts(struct qib_devdata * dd)121f931551bSRalph Campbell int qib_create_ctxts(struct qib_devdata *dd)
122f931551bSRalph Campbell {
123f931551bSRalph Campbell unsigned i;
124e0f30bacSRamkrishna Vepa int local_node_id = pcibus_to_node(dd->pcidev->bus);
125e0f30bacSRamkrishna Vepa
126e0f30bacSRamkrishna Vepa if (local_node_id < 0)
127e0f30bacSRamkrishna Vepa local_node_id = numa_node_id();
128e0f30bacSRamkrishna Vepa dd->assigned_node_id = local_node_id;
129f931551bSRalph Campbell
130f931551bSRalph Campbell /*
131f931551bSRalph Campbell * Allocate full ctxtcnt array, rather than just cfgctxts, because
132f931551bSRalph Campbell * cleanup iterates across all possible ctxts.
133f931551bSRalph Campbell */
134a46a2802SMike Marciniszyn dd->rcd = kcalloc(dd->ctxtcnt, sizeof(*dd->rcd), GFP_KERNEL);
135c40a83b9SLeon Romanovsky if (!dd->rcd)
13606064a10SDennis Dalessandro return -ENOMEM;
137f931551bSRalph Campbell
138f931551bSRalph Campbell /* create (one or more) kctxt */
139f931551bSRalph Campbell for (i = 0; i < dd->first_user_ctxt; ++i) {
140f931551bSRalph Campbell struct qib_pportdata *ppd;
141f931551bSRalph Campbell struct qib_ctxtdata *rcd;
142f931551bSRalph Campbell
143f931551bSRalph Campbell if (dd->skip_kctxt_mask & (1 << i))
144f931551bSRalph Campbell continue;
145f931551bSRalph Campbell
146f931551bSRalph Campbell ppd = dd->pport + (i % dd->num_pports);
147e0f30bacSRamkrishna Vepa
148e0f30bacSRamkrishna Vepa rcd = qib_create_ctxtdata(ppd, i, dd->assigned_node_id);
149f931551bSRalph Campbell if (!rcd) {
1507fac3301SMike Marciniszyn qib_dev_err(dd,
1517fac3301SMike Marciniszyn "Unable to allocate ctxtdata for Kernel ctxt, failing\n");
15206064a10SDennis Dalessandro kfree(dd->rcd);
15306064a10SDennis Dalessandro dd->rcd = NULL;
15406064a10SDennis Dalessandro return -ENOMEM;
155f931551bSRalph Campbell }
156f931551bSRalph Campbell rcd->pkeys[0] = QIB_DEFAULT_P_KEY;
157f931551bSRalph Campbell rcd->seq_cnt = 1;
158f931551bSRalph Campbell }
15906064a10SDennis Dalessandro return 0;
160f931551bSRalph Campbell }
161f931551bSRalph Campbell
162f931551bSRalph Campbell /*
163f931551bSRalph Campbell * Common code for user and kernel context setup.
164f931551bSRalph Campbell */
qib_create_ctxtdata(struct qib_pportdata * ppd,u32 ctxt,int node_id)165e0f30bacSRamkrishna Vepa struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *ppd, u32 ctxt,
166e0f30bacSRamkrishna Vepa int node_id)
167f931551bSRalph Campbell {
168f931551bSRalph Campbell struct qib_devdata *dd = ppd->dd;
169f931551bSRalph Campbell struct qib_ctxtdata *rcd;
170f931551bSRalph Campbell
171e0f30bacSRamkrishna Vepa rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, node_id);
172f931551bSRalph Campbell if (rcd) {
173f931551bSRalph Campbell INIT_LIST_HEAD(&rcd->qp_wait_list);
174e0f30bacSRamkrishna Vepa rcd->node_id = node_id;
175f931551bSRalph Campbell rcd->ppd = ppd;
176f931551bSRalph Campbell rcd->dd = dd;
177f931551bSRalph Campbell rcd->cnt = 1;
178f931551bSRalph Campbell rcd->ctxt = ctxt;
179f931551bSRalph Campbell dd->rcd[ctxt] = rcd;
180ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
181ddb88765SMike Marciniszyn if (ctxt < dd->first_user_ctxt) { /* N/A for PSM contexts */
182ddb88765SMike Marciniszyn rcd->opstats = kzalloc_node(sizeof(*rcd->opstats),
183ddb88765SMike Marciniszyn GFP_KERNEL, node_id);
184ddb88765SMike Marciniszyn if (!rcd->opstats) {
185ddb88765SMike Marciniszyn kfree(rcd);
186ddb88765SMike Marciniszyn qib_dev_err(dd,
187ddb88765SMike Marciniszyn "Unable to allocate per ctxt stats buffer\n");
188ddb88765SMike Marciniszyn return NULL;
189ddb88765SMike Marciniszyn }
190ddb88765SMike Marciniszyn }
191ddb88765SMike Marciniszyn #endif
192f931551bSRalph Campbell dd->f_init_ctxt(rcd);
193f931551bSRalph Campbell
194f931551bSRalph Campbell /*
195f931551bSRalph Campbell * To avoid wasting a lot of memory, we allocate 32KB chunks
196f931551bSRalph Campbell * of physically contiguous memory, advance through it until
197f931551bSRalph Campbell * used up and then allocate more. Of course, we need
198f931551bSRalph Campbell * memory to store those extra pointers, now. 32KB seems to
199f931551bSRalph Campbell * be the most that is "safe" under memory pressure
200f931551bSRalph Campbell * (creating large files and then copying them over
201f931551bSRalph Campbell * NFS while doing lots of MPI jobs). The OOM killer can
202f931551bSRalph Campbell * get invoked, even though we say we can sleep and this can
203f931551bSRalph Campbell * cause significant system problems....
204f931551bSRalph Campbell */
205f931551bSRalph Campbell rcd->rcvegrbuf_size = 0x8000;
206f931551bSRalph Campbell rcd->rcvegrbufs_perchunk =
207f931551bSRalph Campbell rcd->rcvegrbuf_size / dd->rcvegrbufsize;
208f931551bSRalph Campbell rcd->rcvegrbuf_chunks = (rcd->rcvegrcnt +
209f931551bSRalph Campbell rcd->rcvegrbufs_perchunk - 1) /
210f931551bSRalph Campbell rcd->rcvegrbufs_perchunk;
2119e1c0e43SMike Marciniszyn rcd->rcvegrbufs_perchunk_shift =
2129e1c0e43SMike Marciniszyn ilog2(rcd->rcvegrbufs_perchunk);
213f931551bSRalph Campbell }
214f931551bSRalph Campbell return rcd;
215f931551bSRalph Campbell }
216f931551bSRalph Campbell
217f931551bSRalph Campbell /*
218f931551bSRalph Campbell * Common code for initializing the physical port structure.
219f931551bSRalph Campbell */
qib_init_pportdata(struct qib_pportdata * ppd,struct qib_devdata * dd,u8 hw_pidx,u8 port)2207d7632adSMike Marciniszyn int qib_init_pportdata(struct qib_pportdata *ppd, struct qib_devdata *dd,
221f931551bSRalph Campbell u8 hw_pidx, u8 port)
222f931551bSRalph Campbell {
22336a8f01cSMike Marciniszyn int size;
224da12c1f6SMike Marciniszyn
225f931551bSRalph Campbell ppd->dd = dd;
226f931551bSRalph Campbell ppd->hw_pidx = hw_pidx;
227f931551bSRalph Campbell ppd->port = port; /* IB port number, not index */
228f931551bSRalph Campbell
229f931551bSRalph Campbell spin_lock_init(&ppd->sdma_lock);
230f931551bSRalph Campbell spin_lock_init(&ppd->lflags_lock);
2317d7632adSMike Marciniszyn spin_lock_init(&ppd->cc_shadow_lock);
232f931551bSRalph Campbell init_waitqueue_head(&ppd->state_wait);
233f931551bSRalph Campbell
2344037c92fSKees Cook timer_setup(&ppd->symerr_clear_timer, qib_clear_symerror_on_linkup, 0);
235551ace12SMike Marciniszyn
236551ace12SMike Marciniszyn ppd->qib_wq = NULL;
2377d7632adSMike Marciniszyn ppd->ibport_data.pmastats =
2387d7632adSMike Marciniszyn alloc_percpu(struct qib_pma_counters);
2397d7632adSMike Marciniszyn if (!ppd->ibport_data.pmastats)
2407d7632adSMike Marciniszyn return -ENOMEM;
241f24a6d48SHarish Chegondi ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);
242f24a6d48SHarish Chegondi ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);
243f24a6d48SHarish Chegondi ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);
244f24a6d48SHarish Chegondi if (!(ppd->ibport_data.rvp.rc_acks) ||
245f24a6d48SHarish Chegondi !(ppd->ibport_data.rvp.rc_qacks) ||
246f24a6d48SHarish Chegondi !(ppd->ibport_data.rvp.rc_delayed_comp))
247f24a6d48SHarish Chegondi return -ENOMEM;
24836a8f01cSMike Marciniszyn
24936a8f01cSMike Marciniszyn if (qib_cc_table_size < IB_CCT_MIN_ENTRIES)
25036a8f01cSMike Marciniszyn goto bail;
25136a8f01cSMike Marciniszyn
25236a8f01cSMike Marciniszyn ppd->cc_supported_table_entries = min(max_t(int, qib_cc_table_size,
25336a8f01cSMike Marciniszyn IB_CCT_MIN_ENTRIES), IB_CCT_ENTRIES*IB_CC_TABLE_CAP_DEFAULT);
25436a8f01cSMike Marciniszyn
25536a8f01cSMike Marciniszyn ppd->cc_max_table_entries =
25636a8f01cSMike Marciniszyn ppd->cc_supported_table_entries/IB_CCT_ENTRIES;
25736a8f01cSMike Marciniszyn
25836a8f01cSMike Marciniszyn size = IB_CC_TABLE_CAP_DEFAULT * sizeof(struct ib_cc_table_entry)
25936a8f01cSMike Marciniszyn * IB_CCT_ENTRIES;
26036a8f01cSMike Marciniszyn ppd->ccti_entries = kzalloc(size, GFP_KERNEL);
261c40a83b9SLeon Romanovsky if (!ppd->ccti_entries)
26236a8f01cSMike Marciniszyn goto bail;
26336a8f01cSMike Marciniszyn
26436a8f01cSMike Marciniszyn size = IB_CC_CCS_ENTRIES * sizeof(struct ib_cc_congestion_entry);
26536a8f01cSMike Marciniszyn ppd->congestion_entries = kzalloc(size, GFP_KERNEL);
266c40a83b9SLeon Romanovsky if (!ppd->congestion_entries)
26736a8f01cSMike Marciniszyn goto bail_1;
26836a8f01cSMike Marciniszyn
26936a8f01cSMike Marciniszyn size = sizeof(struct cc_table_shadow);
27036a8f01cSMike Marciniszyn ppd->ccti_entries_shadow = kzalloc(size, GFP_KERNEL);
271c40a83b9SLeon Romanovsky if (!ppd->ccti_entries_shadow)
27236a8f01cSMike Marciniszyn goto bail_2;
27336a8f01cSMike Marciniszyn
27436a8f01cSMike Marciniszyn size = sizeof(struct ib_cc_congestion_setting_attr);
27536a8f01cSMike Marciniszyn ppd->congestion_entries_shadow = kzalloc(size, GFP_KERNEL);
276c40a83b9SLeon Romanovsky if (!ppd->congestion_entries_shadow)
27736a8f01cSMike Marciniszyn goto bail_3;
27836a8f01cSMike Marciniszyn
2797d7632adSMike Marciniszyn return 0;
28036a8f01cSMike Marciniszyn
28136a8f01cSMike Marciniszyn bail_3:
28236a8f01cSMike Marciniszyn kfree(ppd->ccti_entries_shadow);
28336a8f01cSMike Marciniszyn ppd->ccti_entries_shadow = NULL;
28436a8f01cSMike Marciniszyn bail_2:
28536a8f01cSMike Marciniszyn kfree(ppd->congestion_entries);
28636a8f01cSMike Marciniszyn ppd->congestion_entries = NULL;
28736a8f01cSMike Marciniszyn bail_1:
28836a8f01cSMike Marciniszyn kfree(ppd->ccti_entries);
28936a8f01cSMike Marciniszyn ppd->ccti_entries = NULL;
29036a8f01cSMike Marciniszyn bail:
29136a8f01cSMike Marciniszyn /* User is intentionally disabling the congestion control agent */
29236a8f01cSMike Marciniszyn if (!qib_cc_table_size)
2937d7632adSMike Marciniszyn return 0;
29436a8f01cSMike Marciniszyn
29536a8f01cSMike Marciniszyn if (qib_cc_table_size < IB_CCT_MIN_ENTRIES) {
29636a8f01cSMike Marciniszyn qib_cc_table_size = 0;
29736a8f01cSMike Marciniszyn qib_dev_err(dd,
29836a8f01cSMike Marciniszyn "Congestion Control table size %d less than minimum %d for port %d\n",
29936a8f01cSMike Marciniszyn qib_cc_table_size, IB_CCT_MIN_ENTRIES, port);
30036a8f01cSMike Marciniszyn }
30136a8f01cSMike Marciniszyn
30236a8f01cSMike Marciniszyn qib_dev_err(dd, "Congestion Control Agent disabled for port %d\n",
30336a8f01cSMike Marciniszyn port);
3047d7632adSMike Marciniszyn return 0;
305f931551bSRalph Campbell }
306f931551bSRalph Campbell
init_pioavailregs(struct qib_devdata * dd)307f931551bSRalph Campbell static int init_pioavailregs(struct qib_devdata *dd)
308f931551bSRalph Campbell {
309f931551bSRalph Campbell int ret, pidx;
310f931551bSRalph Campbell u64 *status_page;
311f931551bSRalph Campbell
312f931551bSRalph Campbell dd->pioavailregs_dma = dma_alloc_coherent(
313f931551bSRalph Campbell &dd->pcidev->dev, PAGE_SIZE, &dd->pioavailregs_phys,
314f931551bSRalph Campbell GFP_KERNEL);
315f931551bSRalph Campbell if (!dd->pioavailregs_dma) {
3167fac3301SMike Marciniszyn qib_dev_err(dd,
3177fac3301SMike Marciniszyn "failed to allocate PIOavail reg area in memory\n");
318f931551bSRalph Campbell ret = -ENOMEM;
319f931551bSRalph Campbell goto done;
320f931551bSRalph Campbell }
321f931551bSRalph Campbell
322f931551bSRalph Campbell /*
323f931551bSRalph Campbell * We really want L2 cache aligned, but for current CPUs of
324f931551bSRalph Campbell * interest, they are the same.
325f931551bSRalph Campbell */
326f931551bSRalph Campbell status_page = (u64 *)
327f931551bSRalph Campbell ((char *) dd->pioavailregs_dma +
328f931551bSRalph Campbell ((2 * L1_CACHE_BYTES +
329f931551bSRalph Campbell dd->pioavregs * sizeof(u64)) & ~L1_CACHE_BYTES));
330f931551bSRalph Campbell /* device status comes first, for backwards compatibility */
331f931551bSRalph Campbell dd->devstatusp = status_page;
332f931551bSRalph Campbell *status_page++ = 0;
333f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
334f931551bSRalph Campbell dd->pport[pidx].statusp = status_page;
335f931551bSRalph Campbell *status_page++ = 0;
336f931551bSRalph Campbell }
337f931551bSRalph Campbell
338f931551bSRalph Campbell /*
339f931551bSRalph Campbell * Setup buffer to hold freeze and other messages, accessible to
340f931551bSRalph Campbell * apps, following statusp. This is per-unit, not per port.
341f931551bSRalph Campbell */
342f931551bSRalph Campbell dd->freezemsg = (char *) status_page;
343f931551bSRalph Campbell *dd->freezemsg = 0;
344f931551bSRalph Campbell /* length of msg buffer is "whatever is left" */
345f931551bSRalph Campbell ret = (char *) status_page - (char *) dd->pioavailregs_dma;
346f931551bSRalph Campbell dd->freezelen = PAGE_SIZE - ret;
347f931551bSRalph Campbell
348f931551bSRalph Campbell ret = 0;
349f931551bSRalph Campbell
350f931551bSRalph Campbell done:
351f931551bSRalph Campbell return ret;
352f931551bSRalph Campbell }
353f931551bSRalph Campbell
354f931551bSRalph Campbell /**
355f931551bSRalph Campbell * init_shadow_tids - allocate the shadow TID array
356f931551bSRalph Campbell * @dd: the qlogic_ib device
357f931551bSRalph Campbell *
358f931551bSRalph Campbell * allocate the shadow TID array, so we can qib_munlock previous
359f931551bSRalph Campbell * entries. It may make more sense to move the pageshadow to the
360f931551bSRalph Campbell * ctxt data structure, so we only allocate memory for ctxts actually
361f931551bSRalph Campbell * in use, since we at 8k per ctxt, now.
362f931551bSRalph Campbell * We don't want failures here to prevent use of the driver/chip,
363f931551bSRalph Campbell * so no return value.
364f931551bSRalph Campbell */
init_shadow_tids(struct qib_devdata * dd)365f931551bSRalph Campbell static void init_shadow_tids(struct qib_devdata *dd)
366f931551bSRalph Campbell {
367f931551bSRalph Campbell struct page **pages;
368f931551bSRalph Campbell dma_addr_t *addrs;
369f931551bSRalph Campbell
370fad953ceSKees Cook pages = vzalloc(array_size(sizeof(struct page *),
371fad953ceSKees Cook dd->cfgctxts * dd->rcvtidcnt));
372c40a83b9SLeon Romanovsky if (!pages)
373f931551bSRalph Campbell goto bail;
374f931551bSRalph Campbell
375fad953ceSKees Cook addrs = vzalloc(array_size(sizeof(dma_addr_t),
376fad953ceSKees Cook dd->cfgctxts * dd->rcvtidcnt));
377c40a83b9SLeon Romanovsky if (!addrs)
378f931551bSRalph Campbell goto bail_free;
379f931551bSRalph Campbell
380f931551bSRalph Campbell dd->pageshadow = pages;
381f931551bSRalph Campbell dd->physshadow = addrs;
382f931551bSRalph Campbell return;
383f931551bSRalph Campbell
384f931551bSRalph Campbell bail_free:
385f931551bSRalph Campbell vfree(pages);
386f931551bSRalph Campbell bail:
387f931551bSRalph Campbell dd->pageshadow = NULL;
388f931551bSRalph Campbell }
389f931551bSRalph Campbell
390f931551bSRalph Campbell /*
391f931551bSRalph Campbell * Do initialization for device that is only needed on
392f931551bSRalph Campbell * first detect, not on resets.
393f931551bSRalph Campbell */
loadtime_init(struct qib_devdata * dd)394f931551bSRalph Campbell static int loadtime_init(struct qib_devdata *dd)
395f931551bSRalph Campbell {
396f931551bSRalph Campbell int ret = 0;
397f931551bSRalph Campbell
398f931551bSRalph Campbell if (((dd->revision >> QLOGIC_IB_R_SOFTWARE_SHIFT) &
399f931551bSRalph Campbell QLOGIC_IB_R_SOFTWARE_MASK) != QIB_CHIP_SWVERSION) {
4007fac3301SMike Marciniszyn qib_dev_err(dd,
401055bb279SColin Ian King "Driver only handles version %d, chip swversion is %d (%llx), failing\n",
402f931551bSRalph Campbell QIB_CHIP_SWVERSION,
403f931551bSRalph Campbell (int)(dd->revision >>
404f931551bSRalph Campbell QLOGIC_IB_R_SOFTWARE_SHIFT) &
405f931551bSRalph Campbell QLOGIC_IB_R_SOFTWARE_MASK,
406f931551bSRalph Campbell (unsigned long long) dd->revision);
407f931551bSRalph Campbell ret = -ENOSYS;
408f931551bSRalph Campbell goto done;
409f931551bSRalph Campbell }
410f931551bSRalph Campbell
411f931551bSRalph Campbell if (dd->revision & QLOGIC_IB_R_EMULATOR_MASK)
412f931551bSRalph Campbell qib_devinfo(dd->pcidev, "%s", dd->boardversion);
413f931551bSRalph Campbell
414f931551bSRalph Campbell spin_lock_init(&dd->pioavail_lock);
415f931551bSRalph Campbell spin_lock_init(&dd->sendctrl_lock);
416f931551bSRalph Campbell spin_lock_init(&dd->uctxt_lock);
417f931551bSRalph Campbell spin_lock_init(&dd->qib_diag_trans_lock);
418f931551bSRalph Campbell spin_lock_init(&dd->eep_st_lock);
419f931551bSRalph Campbell mutex_init(&dd->eep_lock);
420f931551bSRalph Campbell
421f931551bSRalph Campbell if (qib_mini_init)
422f931551bSRalph Campbell goto done;
423f931551bSRalph Campbell
424f931551bSRalph Campbell ret = init_pioavailregs(dd);
425f931551bSRalph Campbell init_shadow_tids(dd);
426f931551bSRalph Campbell
427f931551bSRalph Campbell qib_get_eeprom_info(dd);
428f931551bSRalph Campbell
429f931551bSRalph Campbell /* setup time (don't start yet) to verify we got interrupt */
4304037c92fSKees Cook timer_setup(&dd->intrchk_timer, verify_interrupt, 0);
431f931551bSRalph Campbell done:
432f931551bSRalph Campbell return ret;
433f931551bSRalph Campbell }
434f931551bSRalph Campbell
435f931551bSRalph Campbell /**
436f931551bSRalph Campbell * init_after_reset - re-initialize after a reset
437f931551bSRalph Campbell * @dd: the qlogic_ib device
438f931551bSRalph Campbell *
439f931551bSRalph Campbell * sanity check at least some of the values after reset, and
44025985edcSLucas De Marchi * ensure no receive or transmit (explicitly, in case reset
441f931551bSRalph Campbell * failed
442f931551bSRalph Campbell */
init_after_reset(struct qib_devdata * dd)443f931551bSRalph Campbell static int init_after_reset(struct qib_devdata *dd)
444f931551bSRalph Campbell {
445f931551bSRalph Campbell int i;
446f931551bSRalph Campbell
447f931551bSRalph Campbell /*
448f931551bSRalph Campbell * Ensure chip does no sends or receives, tail updates, or
449f931551bSRalph Campbell * pioavail updates while we re-initialize. This is mostly
450f931551bSRalph Campbell * for the driver data structures, not chip registers.
451f931551bSRalph Campbell */
452f931551bSRalph Campbell for (i = 0; i < dd->num_pports; ++i) {
453f931551bSRalph Campbell /*
454f931551bSRalph Campbell * ctxt == -1 means "all contexts". Only really safe for
455f931551bSRalph Campbell * _dis_abling things, as here.
456f931551bSRalph Campbell */
457f931551bSRalph Campbell dd->f_rcvctrl(dd->pport + i, QIB_RCVCTRL_CTXT_DIS |
458f931551bSRalph Campbell QIB_RCVCTRL_INTRAVAIL_DIS |
459f931551bSRalph Campbell QIB_RCVCTRL_TAILUPD_DIS, -1);
460f931551bSRalph Campbell /* Redundant across ports for some, but no big deal. */
461f931551bSRalph Campbell dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_DIS |
462f931551bSRalph Campbell QIB_SENDCTRL_AVAIL_DIS);
463f931551bSRalph Campbell }
464f931551bSRalph Campbell
465f931551bSRalph Campbell return 0;
466f931551bSRalph Campbell }
467f931551bSRalph Campbell
enable_chip(struct qib_devdata * dd)468f931551bSRalph Campbell static void enable_chip(struct qib_devdata *dd)
469f931551bSRalph Campbell {
470f931551bSRalph Campbell u64 rcvmask;
471f931551bSRalph Campbell int i;
472f931551bSRalph Campbell
473f931551bSRalph Campbell /*
474f931551bSRalph Campbell * Enable PIO send, and update of PIOavail regs to memory.
475f931551bSRalph Campbell */
476f931551bSRalph Campbell for (i = 0; i < dd->num_pports; ++i)
477f931551bSRalph Campbell dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_ENB |
478f931551bSRalph Campbell QIB_SENDCTRL_AVAIL_ENB);
479f931551bSRalph Campbell /*
480f931551bSRalph Campbell * Enable kernel ctxts' receive and receive interrupt.
481f931551bSRalph Campbell * Other ctxts done as user opens and inits them.
482f931551bSRalph Campbell */
483f931551bSRalph Campbell rcvmask = QIB_RCVCTRL_CTXT_ENB | QIB_RCVCTRL_INTRAVAIL_ENB;
484f931551bSRalph Campbell rcvmask |= (dd->flags & QIB_NODMA_RTAIL) ?
485f931551bSRalph Campbell QIB_RCVCTRL_TAILUPD_DIS : QIB_RCVCTRL_TAILUPD_ENB;
486f931551bSRalph Campbell for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {
487f931551bSRalph Campbell struct qib_ctxtdata *rcd = dd->rcd[i];
488f931551bSRalph Campbell
489f931551bSRalph Campbell if (rcd)
490f931551bSRalph Campbell dd->f_rcvctrl(rcd->ppd, rcvmask, i);
491f931551bSRalph Campbell }
492f931551bSRalph Campbell }
493f931551bSRalph Campbell
verify_interrupt(struct timer_list * t)4944037c92fSKees Cook static void verify_interrupt(struct timer_list *t)
495f931551bSRalph Campbell {
4964037c92fSKees Cook struct qib_devdata *dd = from_timer(dd, t, intrchk_timer);
4971ed88dd7SMike Marciniszyn u64 int_counter;
498f931551bSRalph Campbell
499f931551bSRalph Campbell if (!dd)
500f931551bSRalph Campbell return; /* being torn down */
501f931551bSRalph Campbell
502f931551bSRalph Campbell /*
503f931551bSRalph Campbell * If we don't have a lid or any interrupts, let the user know and
504f931551bSRalph Campbell * don't bother checking again.
505f931551bSRalph Campbell */
5061ed88dd7SMike Marciniszyn int_counter = qib_int_counter(dd) - dd->z_int_counter;
5071ed88dd7SMike Marciniszyn if (int_counter == 0) {
508f931551bSRalph Campbell if (!dd->f_intr_fallback(dd))
5097fac3301SMike Marciniszyn dev_err(&dd->pcidev->dev,
5107fac3301SMike Marciniszyn "No interrupts detected, not usable.\n");
511f931551bSRalph Campbell else /* re-arm the timer to see if fallback works */
512f931551bSRalph Campbell mod_timer(&dd->intrchk_timer, jiffies + HZ/2);
513f931551bSRalph Campbell }
514f931551bSRalph Campbell }
515f931551bSRalph Campbell
init_piobuf_state(struct qib_devdata * dd)516f931551bSRalph Campbell static void init_piobuf_state(struct qib_devdata *dd)
517f931551bSRalph Campbell {
518f931551bSRalph Campbell int i, pidx;
519f931551bSRalph Campbell u32 uctxts;
520f931551bSRalph Campbell
521f931551bSRalph Campbell /*
522f931551bSRalph Campbell * Ensure all buffers are free, and fifos empty. Buffers
523f931551bSRalph Campbell * are common, so only do once for port 0.
524f931551bSRalph Campbell *
525f931551bSRalph Campbell * After enable and qib_chg_pioavailkernel so we can safely
526f931551bSRalph Campbell * enable pioavail updates and PIOENABLE. After this, packets
527f931551bSRalph Campbell * are ready and able to go out.
528f931551bSRalph Campbell */
529f931551bSRalph Campbell dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_ALL);
530f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx)
531f931551bSRalph Campbell dd->f_sendctrl(dd->pport + pidx, QIB_SENDCTRL_FLUSH);
532f931551bSRalph Campbell
533f931551bSRalph Campbell /*
534f931551bSRalph Campbell * If not all sendbufs are used, add the one to each of the lower
535f931551bSRalph Campbell * numbered contexts. pbufsctxt and lastctxt_piobuf are
536f931551bSRalph Campbell * calculated in chip-specific code because it may cause some
537f931551bSRalph Campbell * chip-specific adjustments to be made.
538f931551bSRalph Campbell */
539f931551bSRalph Campbell uctxts = dd->cfgctxts - dd->first_user_ctxt;
540f931551bSRalph Campbell dd->ctxts_extrabuf = dd->pbufsctxt ?
541f931551bSRalph Campbell dd->lastctxt_piobuf - (dd->pbufsctxt * uctxts) : 0;
542f931551bSRalph Campbell
543f931551bSRalph Campbell /*
544f931551bSRalph Campbell * Set up the shadow copies of the piobufavail registers,
545f931551bSRalph Campbell * which we compare against the chip registers for now, and
546f931551bSRalph Campbell * the in memory DMA'ed copies of the registers.
547f931551bSRalph Campbell * By now pioavail updates to memory should have occurred, so
548f931551bSRalph Campbell * copy them into our working/shadow registers; this is in
549f931551bSRalph Campbell * case something went wrong with abort, but mostly to get the
550f931551bSRalph Campbell * initial values of the generation bit correct.
551f931551bSRalph Campbell */
552f931551bSRalph Campbell for (i = 0; i < dd->pioavregs; i++) {
553f931551bSRalph Campbell __le64 tmp;
554f931551bSRalph Campbell
555f931551bSRalph Campbell tmp = dd->pioavailregs_dma[i];
556f931551bSRalph Campbell /*
557f931551bSRalph Campbell * Don't need to worry about pioavailkernel here
558f931551bSRalph Campbell * because we will call qib_chg_pioavailkernel() later
559f931551bSRalph Campbell * in initialization, to busy out buffers as needed.
560f931551bSRalph Campbell */
561f931551bSRalph Campbell dd->pioavailshadow[i] = le64_to_cpu(tmp);
562f931551bSRalph Campbell }
563f931551bSRalph Campbell while (i < ARRAY_SIZE(dd->pioavailshadow))
564f931551bSRalph Campbell dd->pioavailshadow[i++] = 0; /* for debugging sanity */
565f931551bSRalph Campbell
566f931551bSRalph Campbell /* after pioavailshadow is setup */
567f931551bSRalph Campbell qib_chg_pioavailkernel(dd, 0, dd->piobcnt2k + dd->piobcnt4k,
568f931551bSRalph Campbell TXCHK_CHG_TYPE_KERN, NULL);
569f931551bSRalph Campbell dd->f_initvl15_bufs(dd);
570f931551bSRalph Campbell }
571f931551bSRalph Campbell
572f931551bSRalph Campbell /**
573551ace12SMike Marciniszyn * qib_create_workqueues - create per port workqueues
574551ace12SMike Marciniszyn * @dd: the qlogic_ib device
575551ace12SMike Marciniszyn */
qib_create_workqueues(struct qib_devdata * dd)576551ace12SMike Marciniszyn static int qib_create_workqueues(struct qib_devdata *dd)
577551ace12SMike Marciniszyn {
578551ace12SMike Marciniszyn int pidx;
579551ace12SMike Marciniszyn struct qib_pportdata *ppd;
580551ace12SMike Marciniszyn
581551ace12SMike Marciniszyn for (pidx = 0; pidx < dd->num_pports; ++pidx) {
582551ace12SMike Marciniszyn ppd = dd->pport + pidx;
583551ace12SMike Marciniszyn if (!ppd->qib_wq) {
584551ace12SMike Marciniszyn char wq_name[8]; /* 3 + 2 + 1 + 1 + 1 */
585da12c1f6SMike Marciniszyn
586551ace12SMike Marciniszyn snprintf(wq_name, sizeof(wq_name), "qib%d_%d",
587551ace12SMike Marciniszyn dd->unit, pidx);
588b59114bbSBhaktipriya Shridhar ppd->qib_wq = alloc_ordered_workqueue(wq_name,
589b59114bbSBhaktipriya Shridhar WQ_MEM_RECLAIM);
590551ace12SMike Marciniszyn if (!ppd->qib_wq)
591551ace12SMike Marciniszyn goto wq_error;
592551ace12SMike Marciniszyn }
593551ace12SMike Marciniszyn }
594551ace12SMike Marciniszyn return 0;
595551ace12SMike Marciniszyn wq_error:
5967fac3301SMike Marciniszyn pr_err("create_singlethread_workqueue failed for port %d\n",
597551ace12SMike Marciniszyn pidx + 1);
598551ace12SMike Marciniszyn for (pidx = 0; pidx < dd->num_pports; ++pidx) {
599551ace12SMike Marciniszyn ppd = dd->pport + pidx;
600551ace12SMike Marciniszyn if (ppd->qib_wq) {
601551ace12SMike Marciniszyn destroy_workqueue(ppd->qib_wq);
602551ace12SMike Marciniszyn ppd->qib_wq = NULL;
603551ace12SMike Marciniszyn }
604551ace12SMike Marciniszyn }
605551ace12SMike Marciniszyn return -ENOMEM;
606551ace12SMike Marciniszyn }
607551ace12SMike Marciniszyn
qib_free_pportdata(struct qib_pportdata * ppd)6087d7632adSMike Marciniszyn static void qib_free_pportdata(struct qib_pportdata *ppd)
6097d7632adSMike Marciniszyn {
6107d7632adSMike Marciniszyn free_percpu(ppd->ibport_data.pmastats);
611f24a6d48SHarish Chegondi free_percpu(ppd->ibport_data.rvp.rc_acks);
612f24a6d48SHarish Chegondi free_percpu(ppd->ibport_data.rvp.rc_qacks);
613f24a6d48SHarish Chegondi free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);
6147d7632adSMike Marciniszyn ppd->ibport_data.pmastats = NULL;
6157d7632adSMike Marciniszyn }
6167d7632adSMike Marciniszyn
617551ace12SMike Marciniszyn /**
618f931551bSRalph Campbell * qib_init - do the actual initialization sequence on the chip
619f931551bSRalph Campbell * @dd: the qlogic_ib device
620f931551bSRalph Campbell * @reinit: reinitializing, so don't allocate new memory
621f931551bSRalph Campbell *
622f931551bSRalph Campbell * Do the actual initialization sequence on the chip. This is done
623f931551bSRalph Campbell * both from the init routine called from the PCI infrastructure, and
624f931551bSRalph Campbell * when we reset the chip, or detect that it was reset internally,
625f931551bSRalph Campbell * or it's administratively re-enabled.
626f931551bSRalph Campbell *
627f931551bSRalph Campbell * Memory allocation here and in called routines is only done in
628f931551bSRalph Campbell * the first case (reinit == 0). We have to be careful, because even
629f931551bSRalph Campbell * without memory allocation, we need to re-write all the chip registers
630f931551bSRalph Campbell * TIDs, etc. after the reset or enable has completed.
631f931551bSRalph Campbell */
qib_init(struct qib_devdata * dd,int reinit)632f931551bSRalph Campbell int qib_init(struct qib_devdata *dd, int reinit)
633f931551bSRalph Campbell {
634f931551bSRalph Campbell int ret = 0, pidx, lastfail = 0;
635f931551bSRalph Campbell u32 portok = 0;
636f931551bSRalph Campbell unsigned i;
637f931551bSRalph Campbell struct qib_ctxtdata *rcd;
638f931551bSRalph Campbell struct qib_pportdata *ppd;
639f931551bSRalph Campbell unsigned long flags;
640f931551bSRalph Campbell
641f931551bSRalph Campbell /* Set linkstate to unknown, so we can watch for a transition. */
642f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
643f931551bSRalph Campbell ppd = dd->pport + pidx;
644f931551bSRalph Campbell spin_lock_irqsave(&ppd->lflags_lock, flags);
645f931551bSRalph Campbell ppd->lflags &= ~(QIBL_LINKACTIVE | QIBL_LINKARMED |
646f931551bSRalph Campbell QIBL_LINKDOWN | QIBL_LINKINIT |
647f931551bSRalph Campbell QIBL_LINKV);
648f931551bSRalph Campbell spin_unlock_irqrestore(&ppd->lflags_lock, flags);
649f931551bSRalph Campbell }
650f931551bSRalph Campbell
651f931551bSRalph Campbell if (reinit)
652f931551bSRalph Campbell ret = init_after_reset(dd);
653f931551bSRalph Campbell else
654f931551bSRalph Campbell ret = loadtime_init(dd);
655f931551bSRalph Campbell if (ret)
656f931551bSRalph Campbell goto done;
657f931551bSRalph Campbell
658f931551bSRalph Campbell /* Bypass most chip-init, to get to device creation */
659f931551bSRalph Campbell if (qib_mini_init)
660f931551bSRalph Campbell return 0;
661f931551bSRalph Campbell
662f931551bSRalph Campbell ret = dd->f_late_initreg(dd);
663f931551bSRalph Campbell if (ret)
664f931551bSRalph Campbell goto done;
665f931551bSRalph Campbell
666f931551bSRalph Campbell /* dd->rcd can be NULL if early init failed */
667f931551bSRalph Campbell for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {
668f931551bSRalph Campbell /*
669f931551bSRalph Campbell * Set up the (kernel) rcvhdr queue and egr TIDs. If doing
670f931551bSRalph Campbell * re-init, the simplest way to handle this is to free
671f931551bSRalph Campbell * existing, and re-allocate.
672f931551bSRalph Campbell * Need to re-create rest of ctxt 0 ctxtdata as well.
673f931551bSRalph Campbell */
674f931551bSRalph Campbell rcd = dd->rcd[i];
675f931551bSRalph Campbell if (!rcd)
676f931551bSRalph Campbell continue;
677f931551bSRalph Campbell
678f931551bSRalph Campbell lastfail = qib_create_rcvhdrq(dd, rcd);
679f931551bSRalph Campbell if (!lastfail)
680f931551bSRalph Campbell lastfail = qib_setup_eagerbufs(rcd);
681666fe24bSArushi if (lastfail)
6827fac3301SMike Marciniszyn qib_dev_err(dd,
6837fac3301SMike Marciniszyn "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");
684f931551bSRalph Campbell }
685f931551bSRalph Campbell
686f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
687f931551bSRalph Campbell int mtu;
688da12c1f6SMike Marciniszyn
689f931551bSRalph Campbell if (lastfail)
690f931551bSRalph Campbell ret = lastfail;
691f931551bSRalph Campbell ppd = dd->pport + pidx;
692f931551bSRalph Campbell mtu = ib_mtu_enum_to_int(qib_ibmtu);
693f931551bSRalph Campbell if (mtu == -1) {
694f931551bSRalph Campbell mtu = QIB_DEFAULT_MTU;
695f931551bSRalph Campbell qib_ibmtu = 0; /* don't leave invalid value */
696f931551bSRalph Campbell }
697f931551bSRalph Campbell /* set max we can ever have for this driver load */
698f931551bSRalph Campbell ppd->init_ibmaxlen = min(mtu > 2048 ?
699f931551bSRalph Campbell dd->piosize4k : dd->piosize2k,
700f931551bSRalph Campbell dd->rcvegrbufsize +
701f931551bSRalph Campbell (dd->rcvhdrentsize << 2));
702f931551bSRalph Campbell /*
703f931551bSRalph Campbell * Have to initialize ibmaxlen, but this will normally
704f931551bSRalph Campbell * change immediately in qib_set_mtu().
705f931551bSRalph Campbell */
706f931551bSRalph Campbell ppd->ibmaxlen = ppd->init_ibmaxlen;
707f931551bSRalph Campbell qib_set_mtu(ppd, mtu);
708f931551bSRalph Campbell
709f931551bSRalph Campbell spin_lock_irqsave(&ppd->lflags_lock, flags);
710f931551bSRalph Campbell ppd->lflags |= QIBL_IB_LINK_DISABLED;
711f931551bSRalph Campbell spin_unlock_irqrestore(&ppd->lflags_lock, flags);
712f931551bSRalph Campbell
713f931551bSRalph Campbell lastfail = dd->f_bringup_serdes(ppd);
714f931551bSRalph Campbell if (lastfail) {
715f931551bSRalph Campbell qib_devinfo(dd->pcidev,
716f931551bSRalph Campbell "Failed to bringup IB port %u\n", ppd->port);
717f931551bSRalph Campbell lastfail = -ENETDOWN;
718f931551bSRalph Campbell continue;
719f931551bSRalph Campbell }
720f931551bSRalph Campbell
721f931551bSRalph Campbell portok++;
722f931551bSRalph Campbell }
723f931551bSRalph Campbell
724f931551bSRalph Campbell if (!portok) {
725f931551bSRalph Campbell /* none of the ports initialized */
726f931551bSRalph Campbell if (!ret && lastfail)
727f931551bSRalph Campbell ret = lastfail;
728f931551bSRalph Campbell else if (!ret)
729f931551bSRalph Campbell ret = -ENETDOWN;
730f931551bSRalph Campbell /* but continue on, so we can debug cause */
731f931551bSRalph Campbell }
732f931551bSRalph Campbell
733f931551bSRalph Campbell enable_chip(dd);
734f931551bSRalph Campbell
735f931551bSRalph Campbell init_piobuf_state(dd);
736f931551bSRalph Campbell
737f931551bSRalph Campbell done:
738f931551bSRalph Campbell if (!ret) {
739f931551bSRalph Campbell /* chip is OK for user apps; mark it as initialized */
740f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
741f931551bSRalph Campbell ppd = dd->pport + pidx;
742f931551bSRalph Campbell /*
743f931551bSRalph Campbell * Set status even if port serdes is not initialized
744f931551bSRalph Campbell * so that diags will work.
745f931551bSRalph Campbell */
746f931551bSRalph Campbell *ppd->statusp |= QIB_STATUS_CHIP_PRESENT |
747f931551bSRalph Campbell QIB_STATUS_INITTED;
748f931551bSRalph Campbell if (!ppd->link_speed_enabled)
749f931551bSRalph Campbell continue;
750f931551bSRalph Campbell if (dd->flags & QIB_HAS_SEND_DMA)
751f931551bSRalph Campbell ret = qib_setup_sdma(ppd);
7524037c92fSKees Cook timer_setup(&ppd->hol_timer, qib_hol_event, 0);
753f931551bSRalph Campbell ppd->hol_state = QIB_HOL_UP;
754f931551bSRalph Campbell }
755f931551bSRalph Campbell
756f931551bSRalph Campbell /* now we can enable all interrupts from the chip */
757f931551bSRalph Campbell dd->f_set_intr_state(dd, 1);
758f931551bSRalph Campbell
759f931551bSRalph Campbell /*
760f931551bSRalph Campbell * Setup to verify we get an interrupt, and fallback
761f931551bSRalph Campbell * to an alternate if necessary and possible.
762f931551bSRalph Campbell */
763f931551bSRalph Campbell mod_timer(&dd->intrchk_timer, jiffies + HZ/2);
764f931551bSRalph Campbell /* start stats retrieval timer */
765f931551bSRalph Campbell mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);
766f931551bSRalph Campbell }
767f931551bSRalph Campbell
768f931551bSRalph Campbell /* if ret is non-zero, we probably should do some cleanup here... */
769f931551bSRalph Campbell return ret;
770f931551bSRalph Campbell }
771f931551bSRalph Campbell
772f931551bSRalph Campbell /*
773f931551bSRalph Campbell * These next two routines are placeholders in case we don't have per-arch
774f931551bSRalph Campbell * code for controlling write combining. If explicit control of write
775f931551bSRalph Campbell * combining is not available, performance will probably be awful.
776f931551bSRalph Campbell */
777f931551bSRalph Campbell
qib_enable_wc(struct qib_devdata * dd)778f931551bSRalph Campbell int __attribute__((weak)) qib_enable_wc(struct qib_devdata *dd)
779f931551bSRalph Campbell {
780f931551bSRalph Campbell return -EOPNOTSUPP;
781f931551bSRalph Campbell }
782f931551bSRalph Campbell
qib_disable_wc(struct qib_devdata * dd)783f931551bSRalph Campbell void __attribute__((weak)) qib_disable_wc(struct qib_devdata *dd)
784f931551bSRalph Campbell {
785f931551bSRalph Campbell }
786f931551bSRalph Campbell
qib_lookup(int unit)787f931551bSRalph Campbell struct qib_devdata *qib_lookup(int unit)
788f931551bSRalph Campbell {
789059d48fbSMatthew Wilcox return xa_load(&qib_dev_table, unit);
790f931551bSRalph Campbell }
791f931551bSRalph Campbell
792f931551bSRalph Campbell /*
793f931551bSRalph Campbell * Stop the timers during unit shutdown, or after an error late
794f931551bSRalph Campbell * in initialization.
795f931551bSRalph Campbell */
qib_stop_timers(struct qib_devdata * dd)796f931551bSRalph Campbell static void qib_stop_timers(struct qib_devdata *dd)
797f931551bSRalph Campbell {
798f931551bSRalph Campbell struct qib_pportdata *ppd;
799f931551bSRalph Campbell int pidx;
800f931551bSRalph Campbell
8014037c92fSKees Cook if (dd->stats_timer.function)
802f931551bSRalph Campbell del_timer_sync(&dd->stats_timer);
8034037c92fSKees Cook if (dd->intrchk_timer.function)
804f931551bSRalph Campbell del_timer_sync(&dd->intrchk_timer);
805f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
806f931551bSRalph Campbell ppd = dd->pport + pidx;
8074037c92fSKees Cook if (ppd->hol_timer.function)
808f931551bSRalph Campbell del_timer_sync(&ppd->hol_timer);
8094037c92fSKees Cook if (ppd->led_override_timer.function) {
810f931551bSRalph Campbell del_timer_sync(&ppd->led_override_timer);
811f931551bSRalph Campbell atomic_set(&ppd->led_override_timer_active, 0);
812f931551bSRalph Campbell }
8134037c92fSKees Cook if (ppd->symerr_clear_timer.function)
814f931551bSRalph Campbell del_timer_sync(&ppd->symerr_clear_timer);
815f931551bSRalph Campbell }
816f931551bSRalph Campbell }
817f931551bSRalph Campbell
818f931551bSRalph Campbell /**
819f931551bSRalph Campbell * qib_shutdown_device - shut down a device
820f931551bSRalph Campbell * @dd: the qlogic_ib device
821f931551bSRalph Campbell *
822f931551bSRalph Campbell * This is called to make the device quiet when we are about to
823f931551bSRalph Campbell * unload the driver, and also when the device is administratively
824f931551bSRalph Campbell * disabled. It does not free any data structures.
825f931551bSRalph Campbell * Everything it does has to be setup again by qib_init(dd, 1)
826f931551bSRalph Campbell */
qib_shutdown_device(struct qib_devdata * dd)827f931551bSRalph Campbell static void qib_shutdown_device(struct qib_devdata *dd)
828f931551bSRalph Campbell {
829f931551bSRalph Campbell struct qib_pportdata *ppd;
830f931551bSRalph Campbell unsigned pidx;
831f931551bSRalph Campbell
8328d3e7113SAlex Estrin if (dd->flags & QIB_SHUTDOWN)
8338d3e7113SAlex Estrin return;
8348d3e7113SAlex Estrin dd->flags |= QIB_SHUTDOWN;
8358d3e7113SAlex Estrin
836f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
837f931551bSRalph Campbell ppd = dd->pport + pidx;
838f931551bSRalph Campbell
839f931551bSRalph Campbell spin_lock_irq(&ppd->lflags_lock);
840f931551bSRalph Campbell ppd->lflags &= ~(QIBL_LINKDOWN | QIBL_LINKINIT |
841f931551bSRalph Campbell QIBL_LINKARMED | QIBL_LINKACTIVE |
842f931551bSRalph Campbell QIBL_LINKV);
843f931551bSRalph Campbell spin_unlock_irq(&ppd->lflags_lock);
844f931551bSRalph Campbell *ppd->statusp &= ~(QIB_STATUS_IB_CONF | QIB_STATUS_IB_READY);
845f931551bSRalph Campbell }
846f931551bSRalph Campbell dd->flags &= ~QIB_INITTED;
847f931551bSRalph Campbell
848f931551bSRalph Campbell /* mask interrupts, but not errors */
849f931551bSRalph Campbell dd->f_set_intr_state(dd, 0);
850f931551bSRalph Campbell
851f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
852f931551bSRalph Campbell ppd = dd->pport + pidx;
853f931551bSRalph Campbell dd->f_rcvctrl(ppd, QIB_RCVCTRL_TAILUPD_DIS |
854f931551bSRalph Campbell QIB_RCVCTRL_CTXT_DIS |
855f931551bSRalph Campbell QIB_RCVCTRL_INTRAVAIL_DIS |
856f931551bSRalph Campbell QIB_RCVCTRL_PKEY_ENB, -1);
857f931551bSRalph Campbell /*
858f931551bSRalph Campbell * Gracefully stop all sends allowing any in progress to
859f931551bSRalph Campbell * trickle out first.
860f931551bSRalph Campbell */
861f931551bSRalph Campbell dd->f_sendctrl(ppd, QIB_SENDCTRL_CLEAR);
862f931551bSRalph Campbell }
863f931551bSRalph Campbell
864f931551bSRalph Campbell /*
865f931551bSRalph Campbell * Enough for anything that's going to trickle out to have actually
866f931551bSRalph Campbell * done so.
867f931551bSRalph Campbell */
868f931551bSRalph Campbell udelay(20);
869f931551bSRalph Campbell
870f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
871f931551bSRalph Campbell ppd = dd->pport + pidx;
872f931551bSRalph Campbell dd->f_setextled(ppd, 0); /* make sure LEDs are off */
873f931551bSRalph Campbell
874f931551bSRalph Campbell if (dd->flags & QIB_HAS_SEND_DMA)
875f931551bSRalph Campbell qib_teardown_sdma(ppd);
876f931551bSRalph Campbell
877f931551bSRalph Campbell dd->f_sendctrl(ppd, QIB_SENDCTRL_AVAIL_DIS |
878f931551bSRalph Campbell QIB_SENDCTRL_SEND_DIS);
879f931551bSRalph Campbell /*
880f931551bSRalph Campbell * Clear SerdesEnable.
881f931551bSRalph Campbell * We can't count on interrupts since we are stopping.
882f931551bSRalph Campbell */
883f931551bSRalph Campbell dd->f_quiet_serdes(ppd);
884551ace12SMike Marciniszyn
885551ace12SMike Marciniszyn if (ppd->qib_wq) {
886551ace12SMike Marciniszyn destroy_workqueue(ppd->qib_wq);
887551ace12SMike Marciniszyn ppd->qib_wq = NULL;
888551ace12SMike Marciniszyn }
8897d7632adSMike Marciniszyn qib_free_pportdata(ppd);
890f931551bSRalph Campbell }
891f931551bSRalph Campbell
892f931551bSRalph Campbell }
893f931551bSRalph Campbell
894f931551bSRalph Campbell /**
895f931551bSRalph Campbell * qib_free_ctxtdata - free a context's allocated data
896f931551bSRalph Campbell * @dd: the qlogic_ib device
897f931551bSRalph Campbell * @rcd: the ctxtdata structure
898f931551bSRalph Campbell *
899f931551bSRalph Campbell * free up any allocated data for a context
900f931551bSRalph Campbell * This should not touch anything that would affect a simultaneous
901f931551bSRalph Campbell * re-allocation of context data, because it is called after qib_mutex
902f931551bSRalph Campbell * is released (and can be called from reinit as well).
903f931551bSRalph Campbell * It should never change any chip state, or global driver state.
904f931551bSRalph Campbell */
qib_free_ctxtdata(struct qib_devdata * dd,struct qib_ctxtdata * rcd)905f931551bSRalph Campbell void qib_free_ctxtdata(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
906f931551bSRalph Campbell {
907f931551bSRalph Campbell if (!rcd)
908f931551bSRalph Campbell return;
909f931551bSRalph Campbell
910f931551bSRalph Campbell if (rcd->rcvhdrq) {
911f931551bSRalph Campbell dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,
912f931551bSRalph Campbell rcd->rcvhdrq, rcd->rcvhdrq_phys);
913f931551bSRalph Campbell rcd->rcvhdrq = NULL;
914f931551bSRalph Campbell if (rcd->rcvhdrtail_kvaddr) {
915f931551bSRalph Campbell dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
916f931551bSRalph Campbell rcd->rcvhdrtail_kvaddr,
917f931551bSRalph Campbell rcd->rcvhdrqtailaddr_phys);
918f931551bSRalph Campbell rcd->rcvhdrtail_kvaddr = NULL;
919f931551bSRalph Campbell }
920f931551bSRalph Campbell }
921f931551bSRalph Campbell if (rcd->rcvegrbuf) {
922f931551bSRalph Campbell unsigned e;
923f931551bSRalph Campbell
924f931551bSRalph Campbell for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {
925f931551bSRalph Campbell void *base = rcd->rcvegrbuf[e];
926f931551bSRalph Campbell size_t size = rcd->rcvegrbuf_size;
927f931551bSRalph Campbell
928f931551bSRalph Campbell dma_free_coherent(&dd->pcidev->dev, size,
929f931551bSRalph Campbell base, rcd->rcvegrbuf_phys[e]);
930f931551bSRalph Campbell }
931f931551bSRalph Campbell kfree(rcd->rcvegrbuf);
932f931551bSRalph Campbell rcd->rcvegrbuf = NULL;
933f931551bSRalph Campbell kfree(rcd->rcvegrbuf_phys);
934f931551bSRalph Campbell rcd->rcvegrbuf_phys = NULL;
935f931551bSRalph Campbell rcd->rcvegrbuf_chunks = 0;
936f931551bSRalph Campbell }
937f931551bSRalph Campbell
938f931551bSRalph Campbell kfree(rcd->tid_pg_list);
939f931551bSRalph Campbell vfree(rcd->user_event_mask);
940f931551bSRalph Campbell vfree(rcd->subctxt_uregbase);
941f931551bSRalph Campbell vfree(rcd->subctxt_rcvegrbuf);
942f931551bSRalph Campbell vfree(rcd->subctxt_rcvhdr_base);
943ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
944ddb88765SMike Marciniszyn kfree(rcd->opstats);
945ddb88765SMike Marciniszyn rcd->opstats = NULL;
946ddb88765SMike Marciniszyn #endif
947f931551bSRalph Campbell kfree(rcd);
948f931551bSRalph Campbell }
949f931551bSRalph Campbell
950f931551bSRalph Campbell /*
951f931551bSRalph Campbell * Perform a PIO buffer bandwidth write test, to verify proper system
952f931551bSRalph Campbell * configuration. Even when all the setup calls work, occasionally
953f931551bSRalph Campbell * BIOS or other issues can prevent write combining from working, or
954f931551bSRalph Campbell * can cause other bandwidth problems to the chip.
955f931551bSRalph Campbell *
956f931551bSRalph Campbell * This test simply writes the same buffer over and over again, and
957f931551bSRalph Campbell * measures close to the peak bandwidth to the chip (not testing
958f931551bSRalph Campbell * data bandwidth to the wire). On chips that use an address-based
959f931551bSRalph Campbell * trigger to send packets to the wire, this is easy. On chips that
960f931551bSRalph Campbell * use a count to trigger, we want to make sure that the packet doesn't
961f931551bSRalph Campbell * go out on the wire, or trigger flow control checks.
962f931551bSRalph Campbell */
qib_verify_pioperf(struct qib_devdata * dd)963f931551bSRalph Campbell static void qib_verify_pioperf(struct qib_devdata *dd)
964f931551bSRalph Campbell {
965f931551bSRalph Campbell u32 pbnum, cnt, lcnt;
966f931551bSRalph Campbell u32 __iomem *piobuf;
967f931551bSRalph Campbell u32 *addr;
968f931551bSRalph Campbell u64 msecs, emsecs;
969f931551bSRalph Campbell
970f931551bSRalph Campbell piobuf = dd->f_getsendbuf(dd->pport, 0ULL, &pbnum);
971f931551bSRalph Campbell if (!piobuf) {
972f931551bSRalph Campbell qib_devinfo(dd->pcidev,
973f931551bSRalph Campbell "No PIObufs for checking perf, skipping\n");
974f931551bSRalph Campbell return;
975f931551bSRalph Campbell }
976f931551bSRalph Campbell
977f931551bSRalph Campbell /*
978f931551bSRalph Campbell * Enough to give us a reasonable test, less than piobuf size, and
979f931551bSRalph Campbell * likely multiple of store buffer length.
980f931551bSRalph Campbell */
981f931551bSRalph Campbell cnt = 1024;
982f931551bSRalph Campbell
983f931551bSRalph Campbell addr = vmalloc(cnt);
984c40a83b9SLeon Romanovsky if (!addr)
985f931551bSRalph Campbell goto done;
986f931551bSRalph Campbell
987f931551bSRalph Campbell preempt_disable(); /* we want reasonably accurate elapsed time */
988f931551bSRalph Campbell msecs = 1 + jiffies_to_msecs(jiffies);
989f931551bSRalph Campbell for (lcnt = 0; lcnt < 10000U; lcnt++) {
990f931551bSRalph Campbell /* wait until we cross msec boundary */
991f931551bSRalph Campbell if (jiffies_to_msecs(jiffies) >= msecs)
992f931551bSRalph Campbell break;
993f931551bSRalph Campbell udelay(1);
994f931551bSRalph Campbell }
995f931551bSRalph Campbell
996f931551bSRalph Campbell dd->f_set_armlaunch(dd, 0);
997f931551bSRalph Campbell
998f931551bSRalph Campbell /*
999f931551bSRalph Campbell * length 0, no dwords actually sent
1000f931551bSRalph Campbell */
1001f931551bSRalph Campbell writeq(0, piobuf);
1002f931551bSRalph Campbell qib_flush_wc();
1003f931551bSRalph Campbell
1004f931551bSRalph Campbell /*
1005f931551bSRalph Campbell * This is only roughly accurate, since even with preempt we
1006f931551bSRalph Campbell * still take interrupts that could take a while. Running for
1007f931551bSRalph Campbell * >= 5 msec seems to get us "close enough" to accurate values.
1008f931551bSRalph Campbell */
1009f931551bSRalph Campbell msecs = jiffies_to_msecs(jiffies);
1010f931551bSRalph Campbell for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) {
1011f931551bSRalph Campbell qib_pio_copy(piobuf + 64, addr, cnt >> 2);
1012f931551bSRalph Campbell emsecs = jiffies_to_msecs(jiffies) - msecs;
1013f931551bSRalph Campbell }
1014f931551bSRalph Campbell
1015f931551bSRalph Campbell /* 1 GiB/sec, slightly over IB SDR line rate */
1016f931551bSRalph Campbell if (lcnt < (emsecs * 1024U))
1017f931551bSRalph Campbell qib_dev_err(dd,
10187fac3301SMike Marciniszyn "Performance problem: bandwidth to PIO buffers is only %u MiB/sec\n",
1019f931551bSRalph Campbell lcnt / (u32) emsecs);
1020f931551bSRalph Campbell
1021f931551bSRalph Campbell preempt_enable();
1022f931551bSRalph Campbell
1023f931551bSRalph Campbell vfree(addr);
1024f931551bSRalph Campbell
1025f931551bSRalph Campbell done:
1026f931551bSRalph Campbell /* disarm piobuf, so it's available again */
1027f931551bSRalph Campbell dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(pbnum));
1028f931551bSRalph Campbell qib_sendbuf_done(dd, pbnum);
1029f931551bSRalph Campbell dd->f_set_armlaunch(dd, 1);
1030f931551bSRalph Campbell }
1031f931551bSRalph Campbell
qib_free_devdata(struct qib_devdata * dd)1032f931551bSRalph Campbell void qib_free_devdata(struct qib_devdata *dd)
1033f931551bSRalph Campbell {
1034f931551bSRalph Campbell unsigned long flags;
1035f931551bSRalph Campbell
1036059d48fbSMatthew Wilcox xa_lock_irqsave(&qib_dev_table, flags);
1037059d48fbSMatthew Wilcox __xa_erase(&qib_dev_table, dd->unit);
1038059d48fbSMatthew Wilcox xa_unlock_irqrestore(&qib_dev_table, flags);
1039f931551bSRalph Campbell
1040ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
1041ddb88765SMike Marciniszyn qib_dbg_ibdev_exit(&dd->verbs_dev);
1042ddb88765SMike Marciniszyn #endif
10431ed88dd7SMike Marciniszyn free_percpu(dd->int_counter);
1044ea0e4ce3SJubin John rvt_dealloc_device(&dd->verbs_dev.rdi);
1045f931551bSRalph Campbell }
1046f931551bSRalph Campbell
qib_int_counter(struct qib_devdata * dd)10471ed88dd7SMike Marciniszyn u64 qib_int_counter(struct qib_devdata *dd)
10481ed88dd7SMike Marciniszyn {
10491ed88dd7SMike Marciniszyn int cpu;
10501ed88dd7SMike Marciniszyn u64 int_counter = 0;
10511ed88dd7SMike Marciniszyn
10521ed88dd7SMike Marciniszyn for_each_possible_cpu(cpu)
10531ed88dd7SMike Marciniszyn int_counter += *per_cpu_ptr(dd->int_counter, cpu);
10541ed88dd7SMike Marciniszyn return int_counter;
10551ed88dd7SMike Marciniszyn }
10561ed88dd7SMike Marciniszyn
qib_sps_ints(void)10571ed88dd7SMike Marciniszyn u64 qib_sps_ints(void)
10581ed88dd7SMike Marciniszyn {
1059059d48fbSMatthew Wilcox unsigned long index, flags;
10601ed88dd7SMike Marciniszyn struct qib_devdata *dd;
10611ed88dd7SMike Marciniszyn u64 sps_ints = 0;
10621ed88dd7SMike Marciniszyn
1063059d48fbSMatthew Wilcox xa_lock_irqsave(&qib_dev_table, flags);
1064059d48fbSMatthew Wilcox xa_for_each(&qib_dev_table, index, dd) {
10651ed88dd7SMike Marciniszyn sps_ints += qib_int_counter(dd);
10661ed88dd7SMike Marciniszyn }
1067059d48fbSMatthew Wilcox xa_unlock_irqrestore(&qib_dev_table, flags);
10681ed88dd7SMike Marciniszyn return sps_ints;
10691ed88dd7SMike Marciniszyn }
10701ed88dd7SMike Marciniszyn
1071f931551bSRalph Campbell /*
1072f931551bSRalph Campbell * Allocate our primary per-unit data structure. Must be done via verbs
1073f931551bSRalph Campbell * allocator, because the verbs cleanup process both does cleanup and
1074f931551bSRalph Campbell * free of the data structure.
1075f931551bSRalph Campbell * "extra" is for chip-specific data.
1076f931551bSRalph Campbell */
qib_alloc_devdata(struct pci_dev * pdev,size_t extra)1077f931551bSRalph Campbell struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)
1078f931551bSRalph Campbell {
1079f931551bSRalph Campbell struct qib_devdata *dd;
10805df1673fSDennis Dalessandro int ret, nports;
1081f931551bSRalph Campbell
10825df1673fSDennis Dalessandro /* extra is * number of ports */
10835df1673fSDennis Dalessandro nports = extra / sizeof(struct qib_pportdata);
10845df1673fSDennis Dalessandro dd = (struct qib_devdata *)rvt_alloc_device(sizeof(*dd) + extra,
10855df1673fSDennis Dalessandro nports);
1086f8b6c47aSMike Marciniszyn if (!dd)
1087f8b6c47aSMike Marciniszyn return ERR_PTR(-ENOMEM);
1088f931551bSRalph Campbell
1089059d48fbSMatthew Wilcox ret = xa_alloc_irq(&qib_dev_table, &dd->unit, dd, xa_limit_32b,
1090059d48fbSMatthew Wilcox GFP_KERNEL);
1091f931551bSRalph Campbell if (ret < 0) {
1092f931551bSRalph Campbell qib_early_err(&pdev->dev,
1093f931551bSRalph Campbell "Could not allocate unit ID: error %d\n", -ret);
1094f931551bSRalph Campbell goto bail;
1095f931551bSRalph Campbell }
10965084c8ffSMichael J. Ruhl rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s%d", "qib", dd->unit);
10975084c8ffSMichael J. Ruhl
10981ed88dd7SMike Marciniszyn dd->int_counter = alloc_percpu(u64);
10991ed88dd7SMike Marciniszyn if (!dd->int_counter) {
11001ed88dd7SMike Marciniszyn ret = -ENOMEM;
11011ed88dd7SMike Marciniszyn qib_early_err(&pdev->dev,
11021ed88dd7SMike Marciniszyn "Could not allocate per-cpu int_counter\n");
11031ed88dd7SMike Marciniszyn goto bail;
11041ed88dd7SMike Marciniszyn }
1105f931551bSRalph Campbell
1106f931551bSRalph Campbell if (!qib_cpulist_count) {
1107f931551bSRalph Campbell u32 count = num_online_cpus();
1108da12c1f6SMike Marciniszyn
1109b577ea54SChristophe JAILLET qib_cpulist = bitmap_zalloc(count, GFP_KERNEL);
1110f931551bSRalph Campbell if (qib_cpulist)
1111f931551bSRalph Campbell qib_cpulist_count = count;
1112f931551bSRalph Campbell }
1113f8b6c47aSMike Marciniszyn #ifdef CONFIG_DEBUG_FS
1114f8b6c47aSMike Marciniszyn qib_dbg_ibdev_init(&dd->verbs_dev);
1115f8b6c47aSMike Marciniszyn #endif
1116f931551bSRalph Campbell return dd;
1117f8b6c47aSMike Marciniszyn bail:
1118f8b6c47aSMike Marciniszyn if (!list_empty(&dd->list))
1119f8b6c47aSMike Marciniszyn list_del_init(&dd->list);
1120ea0e4ce3SJubin John rvt_dealloc_device(&dd->verbs_dev.rdi);
1121a46a2802SMike Marciniszyn return ERR_PTR(ret);
1122f931551bSRalph Campbell }
1123f931551bSRalph Campbell
1124f931551bSRalph Campbell /*
1125f931551bSRalph Campbell * Called from freeze mode handlers, and from PCI error
1126f931551bSRalph Campbell * reporting code. Should be paranoid about state of
1127f931551bSRalph Campbell * system and data structures.
1128f931551bSRalph Campbell */
qib_disable_after_error(struct qib_devdata * dd)1129f931551bSRalph Campbell void qib_disable_after_error(struct qib_devdata *dd)
1130f931551bSRalph Campbell {
1131f931551bSRalph Campbell if (dd->flags & QIB_INITTED) {
1132f931551bSRalph Campbell u32 pidx;
1133f931551bSRalph Campbell
1134f931551bSRalph Campbell dd->flags &= ~QIB_INITTED;
1135f931551bSRalph Campbell if (dd->pport)
1136f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1137f931551bSRalph Campbell struct qib_pportdata *ppd;
1138f931551bSRalph Campbell
1139f931551bSRalph Campbell ppd = dd->pport + pidx;
1140f931551bSRalph Campbell if (dd->flags & QIB_PRESENT) {
1141f931551bSRalph Campbell qib_set_linkstate(ppd,
1142f931551bSRalph Campbell QIB_IB_LINKDOWN_DISABLE);
1143f931551bSRalph Campbell dd->f_setextled(ppd, 0);
1144f931551bSRalph Campbell }
1145f931551bSRalph Campbell *ppd->statusp &= ~QIB_STATUS_IB_READY;
1146f931551bSRalph Campbell }
1147f931551bSRalph Campbell }
1148f931551bSRalph Campbell
1149f931551bSRalph Campbell /*
1150f931551bSRalph Campbell * Mark as having had an error for driver, and also
1151f931551bSRalph Campbell * for /sys and status word mapped to user programs.
1152f931551bSRalph Campbell * This marks unit as not usable, until reset.
1153f931551bSRalph Campbell */
1154f931551bSRalph Campbell if (dd->devstatusp)
1155f931551bSRalph Campbell *dd->devstatusp |= QIB_STATUS_HWERROR;
1156f931551bSRalph Campbell }
1157f931551bSRalph Campbell
11581e6d9abeSGreg Kroah-Hartman static void qib_remove_one(struct pci_dev *);
11591e6d9abeSGreg Kroah-Hartman static int qib_init_one(struct pci_dev *, const struct pci_device_id *);
11608d3e7113SAlex Estrin static void qib_shutdown_one(struct pci_dev *);
1161f931551bSRalph Campbell
1162e2eed58bSVinit Agnihotri #define DRIVER_LOAD_MSG "Intel " QIB_DRV_NAME " loaded: "
1163f931551bSRalph Campbell #define PFX QIB_DRV_NAME ": "
1164f931551bSRalph Campbell
11659baa3c34SBenoit Taine static const struct pci_device_id qib_pci_tbl[] = {
1166f931551bSRalph Campbell { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_QLOGIC_IB_6120) },
1167f931551bSRalph Campbell { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7220) },
1168f931551bSRalph Campbell { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7322) },
1169f931551bSRalph Campbell { 0, }
1170f931551bSRalph Campbell };
1171f931551bSRalph Campbell
1172f931551bSRalph Campbell MODULE_DEVICE_TABLE(pci, qib_pci_tbl);
1173f931551bSRalph Campbell
1174bea25e82SPaul Bolle static struct pci_driver qib_driver = {
1175f931551bSRalph Campbell .name = QIB_DRV_NAME,
1176f931551bSRalph Campbell .probe = qib_init_one,
11771e6d9abeSGreg Kroah-Hartman .remove = qib_remove_one,
11788d3e7113SAlex Estrin .shutdown = qib_shutdown_one,
1179f931551bSRalph Campbell .id_table = qib_pci_tbl,
1180f931551bSRalph Campbell .err_handler = &qib_pci_err_handler,
1181f931551bSRalph Campbell };
1182f931551bSRalph Campbell
11838469ba39SMike Marciniszyn #ifdef CONFIG_INFINIBAND_QIB_DCA
11848469ba39SMike Marciniszyn
11858469ba39SMike Marciniszyn static int qib_notify_dca(struct notifier_block *, unsigned long, void *);
11868469ba39SMike Marciniszyn static struct notifier_block dca_notifier = {
11878469ba39SMike Marciniszyn .notifier_call = qib_notify_dca,
11888469ba39SMike Marciniszyn .next = NULL,
11898469ba39SMike Marciniszyn .priority = 0
11908469ba39SMike Marciniszyn };
11918469ba39SMike Marciniszyn
qib_notify_dca_device(struct device * device,void * data)11928469ba39SMike Marciniszyn static int qib_notify_dca_device(struct device *device, void *data)
11938469ba39SMike Marciniszyn {
11948469ba39SMike Marciniszyn struct qib_devdata *dd = dev_get_drvdata(device);
11958469ba39SMike Marciniszyn unsigned long event = *(unsigned long *)data;
11968469ba39SMike Marciniszyn
11978469ba39SMike Marciniszyn return dd->f_notify_dca(dd, event);
11988469ba39SMike Marciniszyn }
11998469ba39SMike Marciniszyn
qib_notify_dca(struct notifier_block * nb,unsigned long event,void * p)12008469ba39SMike Marciniszyn static int qib_notify_dca(struct notifier_block *nb, unsigned long event,
12018469ba39SMike Marciniszyn void *p)
12028469ba39SMike Marciniszyn {
12038469ba39SMike Marciniszyn int rval;
12048469ba39SMike Marciniszyn
12058469ba39SMike Marciniszyn rval = driver_for_each_device(&qib_driver.driver, NULL,
12068469ba39SMike Marciniszyn &event, qib_notify_dca_device);
12078469ba39SMike Marciniszyn return rval ? NOTIFY_BAD : NOTIFY_DONE;
12088469ba39SMike Marciniszyn }
12098469ba39SMike Marciniszyn
12108469ba39SMike Marciniszyn #endif
12118469ba39SMike Marciniszyn
1212f931551bSRalph Campbell /*
1213f931551bSRalph Campbell * Do all the generic driver unit- and chip-independent memory
1214f931551bSRalph Campbell * allocation and initialization.
1215f931551bSRalph Campbell */
qib_ib_init(void)12160a66d2bdSVinit Agnihotri static int __init qib_ib_init(void)
1217f931551bSRalph Campbell {
1218f931551bSRalph Campbell int ret;
1219f931551bSRalph Campbell
1220f931551bSRalph Campbell ret = qib_dev_init();
1221f931551bSRalph Campbell if (ret)
1222f931551bSRalph Campbell goto bail;
1223f931551bSRalph Campbell
1224f931551bSRalph Campbell /*
1225f931551bSRalph Campbell * These must be called before the driver is registered with
1226f931551bSRalph Campbell * the PCI subsystem.
1227f931551bSRalph Campbell */
12288469ba39SMike Marciniszyn #ifdef CONFIG_INFINIBAND_QIB_DCA
12298469ba39SMike Marciniszyn dca_register_notify(&dca_notifier);
12308469ba39SMike Marciniszyn #endif
1231ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
1232ddb88765SMike Marciniszyn qib_dbg_init();
1233ddb88765SMike Marciniszyn #endif
1234f931551bSRalph Campbell ret = pci_register_driver(&qib_driver);
1235f931551bSRalph Campbell if (ret < 0) {
12367fac3301SMike Marciniszyn pr_err("Unable to register driver: error %d\n", -ret);
123785caafe3SMike Marciniszyn goto bail_dev;
1238f931551bSRalph Campbell }
1239f931551bSRalph Campbell
1240f931551bSRalph Campbell /* not fatal if it doesn't work */
1241f931551bSRalph Campbell if (qib_init_qibfs())
12427fac3301SMike Marciniszyn pr_err("Unable to register ipathfs\n");
1243f931551bSRalph Campbell goto bail; /* all OK */
1244f931551bSRalph Campbell
124585caafe3SMike Marciniszyn bail_dev:
12468469ba39SMike Marciniszyn #ifdef CONFIG_INFINIBAND_QIB_DCA
12478469ba39SMike Marciniszyn dca_unregister_notify(&dca_notifier);
12488469ba39SMike Marciniszyn #endif
1249ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
1250ddb88765SMike Marciniszyn qib_dbg_exit();
1251ddb88765SMike Marciniszyn #endif
1252f931551bSRalph Campbell qib_dev_cleanup();
1253f931551bSRalph Campbell bail:
1254f931551bSRalph Campbell return ret;
1255f931551bSRalph Campbell }
1256f931551bSRalph Campbell
12570a66d2bdSVinit Agnihotri module_init(qib_ib_init);
1258f931551bSRalph Campbell
1259f931551bSRalph Campbell /*
1260f931551bSRalph Campbell * Do the non-unit driver cleanup, memory free, etc. at unload.
1261f931551bSRalph Campbell */
qib_ib_cleanup(void)12620a66d2bdSVinit Agnihotri static void __exit qib_ib_cleanup(void)
1263f931551bSRalph Campbell {
1264f931551bSRalph Campbell int ret;
1265f931551bSRalph Campbell
1266f931551bSRalph Campbell ret = qib_exit_qibfs();
1267f931551bSRalph Campbell if (ret)
12687fac3301SMike Marciniszyn pr_err(
12697fac3301SMike Marciniszyn "Unable to cleanup counter filesystem: error %d\n",
12707fac3301SMike Marciniszyn -ret);
1271f931551bSRalph Campbell
12728469ba39SMike Marciniszyn #ifdef CONFIG_INFINIBAND_QIB_DCA
12738469ba39SMike Marciniszyn dca_unregister_notify(&dca_notifier);
12748469ba39SMike Marciniszyn #endif
1275f931551bSRalph Campbell pci_unregister_driver(&qib_driver);
1276ddb88765SMike Marciniszyn #ifdef CONFIG_DEBUG_FS
1277ddb88765SMike Marciniszyn qib_dbg_exit();
1278ddb88765SMike Marciniszyn #endif
1279f931551bSRalph Campbell
1280f931551bSRalph Campbell qib_cpulist_count = 0;
1281b577ea54SChristophe JAILLET bitmap_free(qib_cpulist);
1282f931551bSRalph Campbell
1283059d48fbSMatthew Wilcox WARN_ON(!xa_empty(&qib_dev_table));
1284f931551bSRalph Campbell qib_dev_cleanup();
1285f931551bSRalph Campbell }
1286f931551bSRalph Campbell
12870a66d2bdSVinit Agnihotri module_exit(qib_ib_cleanup);
1288f931551bSRalph Campbell
1289f931551bSRalph Campbell /* this can only be called after a successful initialization */
cleanup_device_data(struct qib_devdata * dd)1290f931551bSRalph Campbell static void cleanup_device_data(struct qib_devdata *dd)
1291f931551bSRalph Campbell {
1292f931551bSRalph Campbell int ctxt;
1293f931551bSRalph Campbell int pidx;
1294f931551bSRalph Campbell struct qib_ctxtdata **tmp;
1295f931551bSRalph Campbell unsigned long flags;
1296f931551bSRalph Campbell
1297f931551bSRalph Campbell /* users can't do anything more with chip */
129836a8f01cSMike Marciniszyn for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1299f931551bSRalph Campbell if (dd->pport[pidx].statusp)
1300f931551bSRalph Campbell *dd->pport[pidx].statusp &= ~QIB_STATUS_CHIP_PRESENT;
1301f931551bSRalph Campbell
130236a8f01cSMike Marciniszyn spin_lock(&dd->pport[pidx].cc_shadow_lock);
130336a8f01cSMike Marciniszyn
130436a8f01cSMike Marciniszyn kfree(dd->pport[pidx].congestion_entries);
130536a8f01cSMike Marciniszyn dd->pport[pidx].congestion_entries = NULL;
130636a8f01cSMike Marciniszyn kfree(dd->pport[pidx].ccti_entries);
130736a8f01cSMike Marciniszyn dd->pport[pidx].ccti_entries = NULL;
130836a8f01cSMike Marciniszyn kfree(dd->pport[pidx].ccti_entries_shadow);
130936a8f01cSMike Marciniszyn dd->pport[pidx].ccti_entries_shadow = NULL;
131036a8f01cSMike Marciniszyn kfree(dd->pport[pidx].congestion_entries_shadow);
131136a8f01cSMike Marciniszyn dd->pport[pidx].congestion_entries_shadow = NULL;
131236a8f01cSMike Marciniszyn
131336a8f01cSMike Marciniszyn spin_unlock(&dd->pport[pidx].cc_shadow_lock);
131436a8f01cSMike Marciniszyn }
131536a8f01cSMike Marciniszyn
1316f931551bSRalph Campbell qib_disable_wc(dd);
1317f931551bSRalph Campbell
1318f931551bSRalph Campbell if (dd->pioavailregs_dma) {
1319f931551bSRalph Campbell dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
1320f931551bSRalph Campbell (void *) dd->pioavailregs_dma,
1321f931551bSRalph Campbell dd->pioavailregs_phys);
1322f931551bSRalph Campbell dd->pioavailregs_dma = NULL;
1323f931551bSRalph Campbell }
1324f931551bSRalph Campbell
1325f931551bSRalph Campbell if (dd->pageshadow) {
1326f931551bSRalph Campbell struct page **tmpp = dd->pageshadow;
1327f931551bSRalph Campbell dma_addr_t *tmpd = dd->physshadow;
1328308c813bSMike Marciniszyn int i;
1329f931551bSRalph Campbell
1330f931551bSRalph Campbell for (ctxt = 0; ctxt < dd->cfgctxts; ctxt++) {
1331f931551bSRalph Campbell int ctxt_tidbase = ctxt * dd->rcvtidcnt;
1332f931551bSRalph Campbell int maxtid = ctxt_tidbase + dd->rcvtidcnt;
1333f931551bSRalph Campbell
1334f931551bSRalph Campbell for (i = ctxt_tidbase; i < maxtid; i++) {
1335f931551bSRalph Campbell if (!tmpp[i])
1336f931551bSRalph Campbell continue;
13373f69f4e0SChristophe JAILLET dma_unmap_page(&dd->pcidev->dev, tmpd[i],
13383f69f4e0SChristophe JAILLET PAGE_SIZE, DMA_FROM_DEVICE);
1339f931551bSRalph Campbell qib_release_user_pages(&tmpp[i], 1);
1340f931551bSRalph Campbell tmpp[i] = NULL;
1341f931551bSRalph Campbell }
1342f931551bSRalph Campbell }
1343f931551bSRalph Campbell
1344f931551bSRalph Campbell dd->pageshadow = NULL;
1345f931551bSRalph Campbell vfree(tmpp);
1346308c813bSMike Marciniszyn dd->physshadow = NULL;
1347308c813bSMike Marciniszyn vfree(tmpd);
1348f931551bSRalph Campbell }
1349f931551bSRalph Campbell
1350f931551bSRalph Campbell /*
1351f931551bSRalph Campbell * Free any resources still in use (usually just kernel contexts)
1352f931551bSRalph Campbell * at unload; we do for ctxtcnt, because that's what we allocate.
1353f931551bSRalph Campbell * We acquire lock to be really paranoid that rcd isn't being
1354f931551bSRalph Campbell * accessed from some interrupt-related code (that should not happen,
1355f931551bSRalph Campbell * but best to be sure).
1356f931551bSRalph Campbell */
1357f931551bSRalph Campbell spin_lock_irqsave(&dd->uctxt_lock, flags);
1358f931551bSRalph Campbell tmp = dd->rcd;
1359f931551bSRalph Campbell dd->rcd = NULL;
1360f931551bSRalph Campbell spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1361f931551bSRalph Campbell for (ctxt = 0; tmp && ctxt < dd->ctxtcnt; ctxt++) {
1362f931551bSRalph Campbell struct qib_ctxtdata *rcd = tmp[ctxt];
1363f931551bSRalph Campbell
1364f931551bSRalph Campbell tmp[ctxt] = NULL; /* debugging paranoia */
1365f931551bSRalph Campbell qib_free_ctxtdata(dd, rcd);
1366f931551bSRalph Campbell }
1367f931551bSRalph Campbell kfree(tmp);
1368f931551bSRalph Campbell }
1369f931551bSRalph Campbell
1370f931551bSRalph Campbell /*
1371f931551bSRalph Campbell * Clean up on unit shutdown, or error during unit load after
1372f931551bSRalph Campbell * successful initialization.
1373f931551bSRalph Campbell */
qib_postinit_cleanup(struct qib_devdata * dd)1374f931551bSRalph Campbell static void qib_postinit_cleanup(struct qib_devdata *dd)
1375f931551bSRalph Campbell {
1376f931551bSRalph Campbell /*
1377f931551bSRalph Campbell * Clean up chip-specific stuff.
1378f931551bSRalph Campbell * We check for NULL here, because it's outside
1379f931551bSRalph Campbell * the kregbase check, and we need to call it
1380f931551bSRalph Campbell * after the free_irq. Thus it's possible that
1381f931551bSRalph Campbell * the function pointers were never initialized.
1382f931551bSRalph Campbell */
1383f931551bSRalph Campbell if (dd->f_cleanup)
1384f931551bSRalph Campbell dd->f_cleanup(dd);
1385f931551bSRalph Campbell
1386f931551bSRalph Campbell qib_pcie_ddcleanup(dd);
1387f931551bSRalph Campbell
1388f931551bSRalph Campbell cleanup_device_data(dd);
1389f931551bSRalph Campbell
1390f931551bSRalph Campbell qib_free_devdata(dd);
1391f931551bSRalph Campbell }
1392f931551bSRalph Campbell
qib_init_one(struct pci_dev * pdev,const struct pci_device_id * ent)13931e6d9abeSGreg Kroah-Hartman static int qib_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1394f931551bSRalph Campbell {
1395f931551bSRalph Campbell int ret, j, pidx, initfail;
1396f931551bSRalph Campbell struct qib_devdata *dd = NULL;
1397f931551bSRalph Campbell
1398f931551bSRalph Campbell ret = qib_pcie_init(pdev, ent);
1399f931551bSRalph Campbell if (ret)
1400f931551bSRalph Campbell goto bail;
1401f931551bSRalph Campbell
1402f931551bSRalph Campbell /*
1403f931551bSRalph Campbell * Do device-specific initialiation, function table setup, dd
1404f931551bSRalph Campbell * allocation, etc.
1405f931551bSRalph Campbell */
1406f931551bSRalph Campbell switch (ent->device) {
1407f931551bSRalph Campbell case PCI_DEVICE_ID_QLOGIC_IB_6120:
14087e3a1f4aSRalph Campbell #ifdef CONFIG_PCI_MSI
1409f931551bSRalph Campbell dd = qib_init_iba6120_funcs(pdev, ent);
14107e3a1f4aSRalph Campbell #else
14117fac3301SMike Marciniszyn qib_early_err(&pdev->dev,
1412e2eed58bSVinit Agnihotri "Intel PCIE device 0x%x cannot work if CONFIG_PCI_MSI is not enabled\n",
14137e3a1f4aSRalph Campbell ent->device);
14149e43e010SRalph Campbell dd = ERR_PTR(-ENODEV);
14157e3a1f4aSRalph Campbell #endif
1416f931551bSRalph Campbell break;
1417f931551bSRalph Campbell
1418f931551bSRalph Campbell case PCI_DEVICE_ID_QLOGIC_IB_7220:
1419f931551bSRalph Campbell dd = qib_init_iba7220_funcs(pdev, ent);
1420f931551bSRalph Campbell break;
1421f931551bSRalph Campbell
1422f931551bSRalph Campbell case PCI_DEVICE_ID_QLOGIC_IB_7322:
1423f931551bSRalph Campbell dd = qib_init_iba7322_funcs(pdev, ent);
1424f931551bSRalph Campbell break;
1425f931551bSRalph Campbell
1426f931551bSRalph Campbell default:
14277fac3301SMike Marciniszyn qib_early_err(&pdev->dev,
1428e2eed58bSVinit Agnihotri "Failing on unknown Intel deviceid 0x%x\n",
14297fac3301SMike Marciniszyn ent->device);
1430f931551bSRalph Campbell ret = -ENODEV;
1431f931551bSRalph Campbell }
1432f931551bSRalph Campbell
1433f931551bSRalph Campbell if (IS_ERR(dd))
1434f931551bSRalph Campbell ret = PTR_ERR(dd);
1435f931551bSRalph Campbell if (ret)
1436f931551bSRalph Campbell goto bail; /* error already printed */
1437f931551bSRalph Campbell
1438551ace12SMike Marciniszyn ret = qib_create_workqueues(dd);
1439551ace12SMike Marciniszyn if (ret)
1440551ace12SMike Marciniszyn goto bail;
1441551ace12SMike Marciniszyn
1442f931551bSRalph Campbell /* do the generic initialization */
1443f931551bSRalph Campbell initfail = qib_init(dd, 0);
1444f931551bSRalph Campbell
1445f931551bSRalph Campbell ret = qib_register_ib_device(dd);
1446f931551bSRalph Campbell
1447f931551bSRalph Campbell /*
1448f931551bSRalph Campbell * Now ready for use. this should be cleared whenever we
1449f931551bSRalph Campbell * detect a reset, or initiate one. If earlier failure,
1450f931551bSRalph Campbell * we still create devices, so diags, etc. can be used
1451f931551bSRalph Campbell * to determine cause of problem.
1452f931551bSRalph Campbell */
1453f931551bSRalph Campbell if (!qib_mini_init && !initfail && !ret)
1454f931551bSRalph Campbell dd->flags |= QIB_INITTED;
1455f931551bSRalph Campbell
1456f931551bSRalph Campbell j = qib_device_create(dd);
1457f931551bSRalph Campbell if (j)
1458f931551bSRalph Campbell qib_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
1459f931551bSRalph Campbell j = qibfs_add(dd);
1460f931551bSRalph Campbell if (j)
1461f931551bSRalph Campbell qib_dev_err(dd, "Failed filesystem setup for counters: %d\n",
1462f931551bSRalph Campbell -j);
1463f931551bSRalph Campbell
1464f931551bSRalph Campbell if (qib_mini_init || initfail || ret) {
1465f931551bSRalph Campbell qib_stop_timers(dd);
1466f0626710STejun Heo flush_workqueue(ib_wq);
1467f931551bSRalph Campbell for (pidx = 0; pidx < dd->num_pports; ++pidx)
1468f931551bSRalph Campbell dd->f_quiet_serdes(dd->pport + pidx);
1469756a33b8SRalph Campbell if (qib_mini_init)
1470756a33b8SRalph Campbell goto bail;
1471756a33b8SRalph Campbell if (!j) {
1472756a33b8SRalph Campbell (void) qibfs_remove(dd);
1473756a33b8SRalph Campbell qib_device_remove(dd);
1474756a33b8SRalph Campbell }
1475756a33b8SRalph Campbell if (!ret)
1476756a33b8SRalph Campbell qib_unregister_ib_device(dd);
1477756a33b8SRalph Campbell qib_postinit_cleanup(dd);
1478f931551bSRalph Campbell if (initfail)
1479f931551bSRalph Campbell ret = initfail;
1480f931551bSRalph Campbell goto bail;
1481f931551bSRalph Campbell }
1482f931551bSRalph Campbell
1483f931551bSRalph Campbell ret = qib_enable_wc(dd);
1484f931551bSRalph Campbell if (ret) {
14857fac3301SMike Marciniszyn qib_dev_err(dd,
14867fac3301SMike Marciniszyn "Write combining not enabled (err %d): performance may be poor\n",
1487f931551bSRalph Campbell -ret);
1488f931551bSRalph Campbell ret = 0;
1489f931551bSRalph Campbell }
1490f931551bSRalph Campbell
1491f931551bSRalph Campbell qib_verify_pioperf(dd);
1492f931551bSRalph Campbell bail:
1493f931551bSRalph Campbell return ret;
1494f931551bSRalph Campbell }
1495f931551bSRalph Campbell
qib_remove_one(struct pci_dev * pdev)14961e6d9abeSGreg Kroah-Hartman static void qib_remove_one(struct pci_dev *pdev)
1497f931551bSRalph Campbell {
1498f931551bSRalph Campbell struct qib_devdata *dd = pci_get_drvdata(pdev);
1499f931551bSRalph Campbell int ret;
1500f931551bSRalph Campbell
1501f931551bSRalph Campbell /* unregister from IB core */
1502f931551bSRalph Campbell qib_unregister_ib_device(dd);
1503f931551bSRalph Campbell
1504f931551bSRalph Campbell /*
1505f931551bSRalph Campbell * Disable the IB link, disable interrupts on the device,
1506f931551bSRalph Campbell * clear dma engines, etc.
1507f931551bSRalph Campbell */
1508f931551bSRalph Campbell if (!qib_mini_init)
1509f931551bSRalph Campbell qib_shutdown_device(dd);
1510f931551bSRalph Campbell
1511f931551bSRalph Campbell qib_stop_timers(dd);
1512f931551bSRalph Campbell
1513f0626710STejun Heo /* wait until all of our (qsfp) queue_work() calls complete */
1514f0626710STejun Heo flush_workqueue(ib_wq);
1515f931551bSRalph Campbell
1516f931551bSRalph Campbell ret = qibfs_remove(dd);
1517f931551bSRalph Campbell if (ret)
1518f931551bSRalph Campbell qib_dev_err(dd, "Failed counters filesystem cleanup: %d\n",
1519f931551bSRalph Campbell -ret);
1520f931551bSRalph Campbell
1521f931551bSRalph Campbell qib_device_remove(dd);
1522f931551bSRalph Campbell
1523f931551bSRalph Campbell qib_postinit_cleanup(dd);
1524f931551bSRalph Campbell }
1525f931551bSRalph Campbell
qib_shutdown_one(struct pci_dev * pdev)15268d3e7113SAlex Estrin static void qib_shutdown_one(struct pci_dev *pdev)
15278d3e7113SAlex Estrin {
15288d3e7113SAlex Estrin struct qib_devdata *dd = pci_get_drvdata(pdev);
15298d3e7113SAlex Estrin
15308d3e7113SAlex Estrin qib_shutdown_device(dd);
15318d3e7113SAlex Estrin }
15328d3e7113SAlex Estrin
1533f931551bSRalph Campbell /**
1534f931551bSRalph Campbell * qib_create_rcvhdrq - create a receive header queue
1535f931551bSRalph Campbell * @dd: the qlogic_ib device
1536f931551bSRalph Campbell * @rcd: the context data
1537f931551bSRalph Campbell *
1538f931551bSRalph Campbell * This must be contiguous memory (from an i/o perspective), and must be
1539f931551bSRalph Campbell * DMA'able (which means for some systems, it will go through an IOMMU,
1540f931551bSRalph Campbell * or be forced into a low address range).
1541f931551bSRalph Campbell */
qib_create_rcvhdrq(struct qib_devdata * dd,struct qib_ctxtdata * rcd)1542f931551bSRalph Campbell int qib_create_rcvhdrq(struct qib_devdata *dd, struct qib_ctxtdata *rcd)
1543f931551bSRalph Campbell {
1544f931551bSRalph Campbell unsigned amt;
1545e0f30bacSRamkrishna Vepa int old_node_id;
1546f931551bSRalph Campbell
1547f931551bSRalph Campbell if (!rcd->rcvhdrq) {
1548f931551bSRalph Campbell dma_addr_t phys_hdrqtail;
1549f931551bSRalph Campbell
1550f931551bSRalph Campbell amt = ALIGN(dd->rcvhdrcnt * dd->rcvhdrentsize *
1551f931551bSRalph Campbell sizeof(u32), PAGE_SIZE);
1552e0f30bacSRamkrishna Vepa
1553e0f30bacSRamkrishna Vepa old_node_id = dev_to_node(&dd->pcidev->dev);
1554e0f30bacSRamkrishna Vepa set_dev_node(&dd->pcidev->dev, rcd->node_id);
1555*2fce26a1SChristoph Hellwig rcd->rcvhdrq = dma_alloc_coherent(&dd->pcidev->dev, amt,
1556*2fce26a1SChristoph Hellwig &rcd->rcvhdrq_phys, GFP_KERNEL);
1557e0f30bacSRamkrishna Vepa set_dev_node(&dd->pcidev->dev, old_node_id);
1558f931551bSRalph Campbell
1559f931551bSRalph Campbell if (!rcd->rcvhdrq) {
15607fac3301SMike Marciniszyn qib_dev_err(dd,
15617fac3301SMike Marciniszyn "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
1562f931551bSRalph Campbell amt, rcd->ctxt);
1563f931551bSRalph Campbell goto bail;
1564f931551bSRalph Campbell }
1565f931551bSRalph Campbell
1566f931551bSRalph Campbell if (rcd->ctxt >= dd->first_user_ctxt) {
1567f931551bSRalph Campbell rcd->user_event_mask = vmalloc_user(PAGE_SIZE);
1568f931551bSRalph Campbell if (!rcd->user_event_mask)
1569f931551bSRalph Campbell goto bail_free_hdrq;
1570f931551bSRalph Campbell }
1571f931551bSRalph Campbell
1572f931551bSRalph Campbell if (!(dd->flags & QIB_NODMA_RTAIL)) {
1573e0f30bacSRamkrishna Vepa set_dev_node(&dd->pcidev->dev, rcd->node_id);
1574f931551bSRalph Campbell rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(
1575f931551bSRalph Campbell &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,
1576*2fce26a1SChristoph Hellwig GFP_KERNEL);
1577e0f30bacSRamkrishna Vepa set_dev_node(&dd->pcidev->dev, old_node_id);
1578f931551bSRalph Campbell if (!rcd->rcvhdrtail_kvaddr)
1579f931551bSRalph Campbell goto bail_free;
1580f931551bSRalph Campbell rcd->rcvhdrqtailaddr_phys = phys_hdrqtail;
1581f931551bSRalph Campbell }
1582f931551bSRalph Campbell
1583f931551bSRalph Campbell rcd->rcvhdrq_size = amt;
1584f931551bSRalph Campbell }
1585f931551bSRalph Campbell
1586f931551bSRalph Campbell /* clear for security and sanity on each use */
1587f931551bSRalph Campbell memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);
1588f931551bSRalph Campbell if (rcd->rcvhdrtail_kvaddr)
1589f931551bSRalph Campbell memset(rcd->rcvhdrtail_kvaddr, 0, PAGE_SIZE);
1590f931551bSRalph Campbell return 0;
1591f931551bSRalph Campbell
1592f931551bSRalph Campbell bail_free:
15937fac3301SMike Marciniszyn qib_dev_err(dd,
15947fac3301SMike Marciniszyn "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
15957fac3301SMike Marciniszyn rcd->ctxt);
1596f931551bSRalph Campbell vfree(rcd->user_event_mask);
1597f931551bSRalph Campbell rcd->user_event_mask = NULL;
1598f931551bSRalph Campbell bail_free_hdrq:
1599f931551bSRalph Campbell dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
1600f931551bSRalph Campbell rcd->rcvhdrq_phys);
1601f931551bSRalph Campbell rcd->rcvhdrq = NULL;
1602f931551bSRalph Campbell bail:
1603f931551bSRalph Campbell return -ENOMEM;
1604f931551bSRalph Campbell }
1605f931551bSRalph Campbell
1606f931551bSRalph Campbell /**
1607ae360f41SLeon Romanovsky * qib_setup_eagerbufs - allocate eager buffers, both kernel and user contexts.
1608f931551bSRalph Campbell * @rcd: the context we are setting up.
1609f931551bSRalph Campbell *
1610f931551bSRalph Campbell * Allocate the eager TID buffers and program them into hip.
1611f931551bSRalph Campbell * They are no longer completely contiguous, we do multiple allocation
1612f931551bSRalph Campbell * calls. Otherwise we get the OOM code involved, by asking for too
1613f931551bSRalph Campbell * much per call, with disastrous results on some kernels.
1614f931551bSRalph Campbell */
qib_setup_eagerbufs(struct qib_ctxtdata * rcd)1615f931551bSRalph Campbell int qib_setup_eagerbufs(struct qib_ctxtdata *rcd)
1616f931551bSRalph Campbell {
1617f931551bSRalph Campbell struct qib_devdata *dd = rcd->dd;
1618f931551bSRalph Campbell unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;
1619f931551bSRalph Campbell size_t size;
1620e0f30bacSRamkrishna Vepa int old_node_id;
1621f931551bSRalph Campbell
1622f931551bSRalph Campbell egrcnt = rcd->rcvegrcnt;
1623f931551bSRalph Campbell egroff = rcd->rcvegr_tid_base;
1624f931551bSRalph Campbell egrsize = dd->rcvegrbufsize;
1625f931551bSRalph Campbell
1626f931551bSRalph Campbell chunk = rcd->rcvegrbuf_chunks;
1627f931551bSRalph Campbell egrperchunk = rcd->rcvegrbufs_perchunk;
1628f931551bSRalph Campbell size = rcd->rcvegrbuf_size;
1629f931551bSRalph Campbell if (!rcd->rcvegrbuf) {
1630f931551bSRalph Campbell rcd->rcvegrbuf =
1631590b5b7dSKees Cook kcalloc_node(chunk, sizeof(rcd->rcvegrbuf[0]),
1632e0f30bacSRamkrishna Vepa GFP_KERNEL, rcd->node_id);
1633f931551bSRalph Campbell if (!rcd->rcvegrbuf)
1634f931551bSRalph Campbell goto bail;
1635f931551bSRalph Campbell }
1636f931551bSRalph Campbell if (!rcd->rcvegrbuf_phys) {
1637f931551bSRalph Campbell rcd->rcvegrbuf_phys =
16387d502071SJohannes Thumshirn kmalloc_array_node(chunk,
16397d502071SJohannes Thumshirn sizeof(rcd->rcvegrbuf_phys[0]),
1640e0f30bacSRamkrishna Vepa GFP_KERNEL, rcd->node_id);
1641f931551bSRalph Campbell if (!rcd->rcvegrbuf_phys)
1642f931551bSRalph Campbell goto bail_rcvegrbuf;
1643f931551bSRalph Campbell }
1644f931551bSRalph Campbell for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {
1645f931551bSRalph Campbell if (rcd->rcvegrbuf[e])
1646f931551bSRalph Campbell continue;
1647e0f30bacSRamkrishna Vepa
1648e0f30bacSRamkrishna Vepa old_node_id = dev_to_node(&dd->pcidev->dev);
1649e0f30bacSRamkrishna Vepa set_dev_node(&dd->pcidev->dev, rcd->node_id);
1650f931551bSRalph Campbell rcd->rcvegrbuf[e] =
1651f931551bSRalph Campbell dma_alloc_coherent(&dd->pcidev->dev, size,
1652f931551bSRalph Campbell &rcd->rcvegrbuf_phys[e],
1653*2fce26a1SChristoph Hellwig GFP_KERNEL);
1654e0f30bacSRamkrishna Vepa set_dev_node(&dd->pcidev->dev, old_node_id);
1655f931551bSRalph Campbell if (!rcd->rcvegrbuf[e])
1656f931551bSRalph Campbell goto bail_rcvegrbuf_phys;
1657f931551bSRalph Campbell }
1658f931551bSRalph Campbell
1659f931551bSRalph Campbell rcd->rcvegr_phys = rcd->rcvegrbuf_phys[0];
1660f931551bSRalph Campbell
1661f931551bSRalph Campbell for (e = chunk = 0; chunk < rcd->rcvegrbuf_chunks; chunk++) {
1662f931551bSRalph Campbell dma_addr_t pa = rcd->rcvegrbuf_phys[chunk];
1663f931551bSRalph Campbell unsigned i;
1664f931551bSRalph Campbell
16655df4223aSRalph Campbell /* clear for security and sanity on each use */
16665df4223aSRalph Campbell memset(rcd->rcvegrbuf[chunk], 0, size);
16675df4223aSRalph Campbell
1668f931551bSRalph Campbell for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {
1669f931551bSRalph Campbell dd->f_put_tid(dd, e + egroff +
1670f931551bSRalph Campbell (u64 __iomem *)
1671f931551bSRalph Campbell ((char __iomem *)
1672f931551bSRalph Campbell dd->kregbase +
1673f931551bSRalph Campbell dd->rcvegrbase),
1674f931551bSRalph Campbell RCVHQ_RCV_TYPE_EAGER, pa);
1675f931551bSRalph Campbell pa += egrsize;
1676f931551bSRalph Campbell }
1677f931551bSRalph Campbell cond_resched(); /* don't hog the cpu */
1678f931551bSRalph Campbell }
1679f931551bSRalph Campbell
1680f931551bSRalph Campbell return 0;
1681f931551bSRalph Campbell
1682f931551bSRalph Campbell bail_rcvegrbuf_phys:
1683f931551bSRalph Campbell for (e = 0; e < rcd->rcvegrbuf_chunks && rcd->rcvegrbuf[e]; e++)
1684f931551bSRalph Campbell dma_free_coherent(&dd->pcidev->dev, size,
1685f931551bSRalph Campbell rcd->rcvegrbuf[e], rcd->rcvegrbuf_phys[e]);
1686f931551bSRalph Campbell kfree(rcd->rcvegrbuf_phys);
1687f931551bSRalph Campbell rcd->rcvegrbuf_phys = NULL;
1688f931551bSRalph Campbell bail_rcvegrbuf:
1689f931551bSRalph Campbell kfree(rcd->rcvegrbuf);
1690f931551bSRalph Campbell rcd->rcvegrbuf = NULL;
1691f931551bSRalph Campbell bail:
1692f931551bSRalph Campbell return -ENOMEM;
1693f931551bSRalph Campbell }
1694f931551bSRalph Campbell
1695fce24a9dSDave Olson /*
1696fce24a9dSDave Olson * Note: Changes to this routine should be mirrored
1697fce24a9dSDave Olson * for the diagnostics routine qib_remap_ioaddr32().
1698fce24a9dSDave Olson * There is also related code for VL15 buffers in qib_init_7322_variables().
1699fce24a9dSDave Olson * The teardown code that unmaps is in qib_pcie_ddcleanup()
1700fce24a9dSDave Olson */
init_chip_wc_pat(struct qib_devdata * dd,u32 vl15buflen)1701f931551bSRalph Campbell int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen)
1702f931551bSRalph Campbell {
1703f931551bSRalph Campbell u64 __iomem *qib_kregbase = NULL;
1704f931551bSRalph Campbell void __iomem *qib_piobase = NULL;
1705f931551bSRalph Campbell u64 __iomem *qib_userbase = NULL;
1706f931551bSRalph Campbell u64 qib_kreglen;
1707f931551bSRalph Campbell u64 qib_pio2koffset = dd->piobufbase & 0xffffffff;
1708f931551bSRalph Campbell u64 qib_pio4koffset = dd->piobufbase >> 32;
1709f931551bSRalph Campbell u64 qib_pio2klen = dd->piobcnt2k * dd->palign;
1710f931551bSRalph Campbell u64 qib_pio4klen = dd->piobcnt4k * dd->align4k;
1711f931551bSRalph Campbell u64 qib_physaddr = dd->physaddr;
1712f931551bSRalph Campbell u64 qib_piolen;
1713f931551bSRalph Campbell u64 qib_userlen = 0;
1714f931551bSRalph Campbell
1715f931551bSRalph Campbell /*
1716f931551bSRalph Campbell * Free the old mapping because the kernel will try to reuse the
1717f931551bSRalph Campbell * old mapping and not create a new mapping with the
1718f931551bSRalph Campbell * write combining attribute.
1719f931551bSRalph Campbell */
1720f931551bSRalph Campbell iounmap(dd->kregbase);
1721f931551bSRalph Campbell dd->kregbase = NULL;
1722f931551bSRalph Campbell
1723f931551bSRalph Campbell /*
1724f931551bSRalph Campbell * Assumes chip address space looks like:
1725f931551bSRalph Campbell * - kregs + sregs + cregs + uregs (in any order)
1726f931551bSRalph Campbell * - piobufs (2K and 4K bufs in either order)
1727f931551bSRalph Campbell * or:
1728f931551bSRalph Campbell * - kregs + sregs + cregs (in any order)
1729f931551bSRalph Campbell * - piobufs (2K and 4K bufs in either order)
1730f931551bSRalph Campbell * - uregs
1731f931551bSRalph Campbell */
1732f931551bSRalph Campbell if (dd->piobcnt4k == 0) {
1733f931551bSRalph Campbell qib_kreglen = qib_pio2koffset;
1734f931551bSRalph Campbell qib_piolen = qib_pio2klen;
1735f931551bSRalph Campbell } else if (qib_pio2koffset < qib_pio4koffset) {
1736f931551bSRalph Campbell qib_kreglen = qib_pio2koffset;
1737f931551bSRalph Campbell qib_piolen = qib_pio4koffset + qib_pio4klen - qib_kreglen;
1738f931551bSRalph Campbell } else {
1739f931551bSRalph Campbell qib_kreglen = qib_pio4koffset;
1740f931551bSRalph Campbell qib_piolen = qib_pio2koffset + qib_pio2klen - qib_kreglen;
1741f931551bSRalph Campbell }
1742f931551bSRalph Campbell qib_piolen += vl15buflen;
1743f931551bSRalph Campbell /* Map just the configured ports (not all hw ports) */
1744f931551bSRalph Campbell if (dd->uregbase > qib_kreglen)
1745f931551bSRalph Campbell qib_userlen = dd->ureg_align * dd->cfgctxts;
1746f931551bSRalph Campbell
1747f931551bSRalph Campbell /* Sanity checks passed, now create the new mappings */
17484bdc0d67SChristoph Hellwig qib_kregbase = ioremap(qib_physaddr, qib_kreglen);
1749f931551bSRalph Campbell if (!qib_kregbase)
1750f931551bSRalph Campbell goto bail;
1751f931551bSRalph Campbell
1752f931551bSRalph Campbell qib_piobase = ioremap_wc(qib_physaddr + qib_kreglen, qib_piolen);
1753f931551bSRalph Campbell if (!qib_piobase)
1754f931551bSRalph Campbell goto bail_kregbase;
1755f931551bSRalph Campbell
1756f931551bSRalph Campbell if (qib_userlen) {
17574bdc0d67SChristoph Hellwig qib_userbase = ioremap(qib_physaddr + dd->uregbase,
1758f931551bSRalph Campbell qib_userlen);
1759f931551bSRalph Campbell if (!qib_userbase)
1760f931551bSRalph Campbell goto bail_piobase;
1761f931551bSRalph Campbell }
1762f931551bSRalph Campbell
1763f931551bSRalph Campbell dd->kregbase = qib_kregbase;
1764f931551bSRalph Campbell dd->kregend = (u64 __iomem *)
1765f931551bSRalph Campbell ((char __iomem *) qib_kregbase + qib_kreglen);
1766f931551bSRalph Campbell dd->piobase = qib_piobase;
1767f931551bSRalph Campbell dd->pio2kbase = (void __iomem *)
1768f931551bSRalph Campbell (((char __iomem *) dd->piobase) +
1769f931551bSRalph Campbell qib_pio2koffset - qib_kreglen);
1770f931551bSRalph Campbell if (dd->piobcnt4k)
1771f931551bSRalph Campbell dd->pio4kbase = (void __iomem *)
1772f931551bSRalph Campbell (((char __iomem *) dd->piobase) +
1773f931551bSRalph Campbell qib_pio4koffset - qib_kreglen);
1774f931551bSRalph Campbell if (qib_userlen)
1775f931551bSRalph Campbell /* ureg will now be accessed relative to dd->userbase */
1776f931551bSRalph Campbell dd->userbase = qib_userbase;
1777f931551bSRalph Campbell return 0;
1778f931551bSRalph Campbell
1779f931551bSRalph Campbell bail_piobase:
1780f931551bSRalph Campbell iounmap(qib_piobase);
1781f931551bSRalph Campbell bail_kregbase:
1782f931551bSRalph Campbell iounmap(qib_kregbase);
1783f931551bSRalph Campbell bail:
1784f931551bSRalph Campbell return -ENOMEM;
1785f931551bSRalph Campbell }
1786