1145eba1aSCai Huoqing // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2f48ad614SDennis Dalessandro /*
319d8b90aSGrzegorz Andrejczuk * Copyright(c) 2015 - 2020 Intel Corporation.
4ddf65f28SScott Breyer * Copyright(c) 2021 Cornelis Networks.
5f48ad614SDennis Dalessandro */
6f48ad614SDennis Dalessandro
7f48ad614SDennis Dalessandro #include <linux/pci.h>
8f48ad614SDennis Dalessandro #include <linux/netdevice.h>
9f48ad614SDennis Dalessandro #include <linux/vmalloc.h>
10f48ad614SDennis Dalessandro #include <linux/delay.h>
1103b92789SMatthew Wilcox #include <linux/xarray.h>
12f48ad614SDennis Dalessandro #include <linux/module.h>
13f48ad614SDennis Dalessandro #include <linux/printk.h>
14f48ad614SDennis Dalessandro #include <linux/hrtimer.h>
158737ce95SMichael J. Ruhl #include <linux/bitmap.h>
1698fa15f3SAnshuman Khandual #include <linux/numa.h>
17f48ad614SDennis Dalessandro #include <rdma/rdma_vt.h>
18f48ad614SDennis Dalessandro
19f48ad614SDennis Dalessandro #include "hfi.h"
20f48ad614SDennis Dalessandro #include "device.h"
21f48ad614SDennis Dalessandro #include "common.h"
22f48ad614SDennis Dalessandro #include "trace.h"
23f48ad614SDennis Dalessandro #include "mad.h"
24f48ad614SDennis Dalessandro #include "sdma.h"
25f48ad614SDennis Dalessandro #include "debugfs.h"
26f48ad614SDennis Dalessandro #include "verbs.h"
27f48ad614SDennis Dalessandro #include "aspm.h"
284197344bSDennis Dalessandro #include "affinity.h"
29d4829ea6SVishwanathapura, Niranjana #include "vnic.h"
30fe4e74eeSMichael J. Ruhl #include "exp_rcv.h"
314730f4a6SGrzegorz Andrejczuk #include "netdev.h"
32f48ad614SDennis Dalessandro
33f48ad614SDennis Dalessandro #undef pr_fmt
34f48ad614SDennis Dalessandro #define pr_fmt(fmt) DRIVER_NAME ": " fmt
35f48ad614SDennis Dalessandro
36f48ad614SDennis Dalessandro /*
37f48ad614SDennis Dalessandro * min buffers we want to have per context, after driver
38f48ad614SDennis Dalessandro */
39f48ad614SDennis Dalessandro #define HFI1_MIN_USER_CTXT_BUFCNT 7
40f48ad614SDennis Dalessandro
41f48ad614SDennis Dalessandro #define HFI1_MIN_EAGER_BUFFER_SIZE (4 * 1024) /* 4KB */
42f48ad614SDennis Dalessandro #define HFI1_MAX_EAGER_BUFFER_SIZE (256 * 1024) /* 256KB */
43f48ad614SDennis Dalessandro
4457f97e96SMichael J. Ruhl #define NUM_IB_PORTS 1
4557f97e96SMichael J. Ruhl
46f48ad614SDennis Dalessandro /*
47f48ad614SDennis Dalessandro * Number of user receive contexts we are configured to use (to allow for more
48f48ad614SDennis Dalessandro * pio buffers per ctxt, etc.) Zero means use one user context per CPU.
49f48ad614SDennis Dalessandro */
50f48ad614SDennis Dalessandro int num_user_contexts = -1;
515da9e742SMichael J. Ruhl module_param_named(num_user_contexts, num_user_contexts, int, 0444);
52f48ad614SDennis Dalessandro MODULE_PARM_DESC(
535da9e742SMichael J. Ruhl num_user_contexts, "Set max number of user contexts to use (default: -1 will use the real (non-HT) CPU count)");
54f48ad614SDennis Dalessandro
55f48ad614SDennis Dalessandro uint krcvqs[RXE_NUM_DATA_VL];
56f48ad614SDennis Dalessandro int krcvqsset;
57f48ad614SDennis Dalessandro module_param_array(krcvqs, uint, &krcvqsset, S_IRUGO);
58f48ad614SDennis Dalessandro MODULE_PARM_DESC(krcvqs, "Array of the number of non-control kernel receive queues by VL");
59f48ad614SDennis Dalessandro
60f48ad614SDennis Dalessandro /* computed based on above array */
61429b6a72SHarish Chegondi unsigned long n_krcvqs;
62f48ad614SDennis Dalessandro
63f48ad614SDennis Dalessandro static unsigned hfi1_rcvarr_split = 25;
64f48ad614SDennis Dalessandro module_param_named(rcvarr_split, hfi1_rcvarr_split, uint, S_IRUGO);
65f48ad614SDennis Dalessandro MODULE_PARM_DESC(rcvarr_split, "Percent of context's RcvArray entries used for Eager buffers");
66f48ad614SDennis Dalessandro
679746fa43STymoteusz Kielan static uint eager_buffer_size = (8 << 20); /* 8MB */
68f48ad614SDennis Dalessandro module_param(eager_buffer_size, uint, S_IRUGO);
699746fa43STymoteusz Kielan MODULE_PARM_DESC(eager_buffer_size, "Size of the eager buffers, default: 8MB");
70f48ad614SDennis Dalessandro
71f48ad614SDennis Dalessandro static uint rcvhdrcnt = 2048; /* 2x the max eager buffer count */
72f48ad614SDennis Dalessandro module_param_named(rcvhdrcnt, rcvhdrcnt, uint, S_IRUGO);
73f48ad614SDennis Dalessandro MODULE_PARM_DESC(rcvhdrcnt, "Receive header queue count (default 2048)");
74f48ad614SDennis Dalessandro
75f48ad614SDennis Dalessandro static uint hfi1_hdrq_entsize = 32;
76d9a6ce68SMike Marciniszyn module_param_named(hdrq_entsize, hfi1_hdrq_entsize, uint, 0444);
77d9a6ce68SMike Marciniszyn MODULE_PARM_DESC(hdrq_entsize, "Size of header queue entries: 2 - 8B, 16 - 64B, 32 - 128B (default)");
78f48ad614SDennis Dalessandro
79f48ad614SDennis Dalessandro unsigned int user_credit_return_threshold = 33; /* default is 33% */
80f48ad614SDennis Dalessandro module_param(user_credit_return_threshold, uint, S_IRUGO);
81f48ad614SDennis Dalessandro MODULE_PARM_DESC(user_credit_return_threshold, "Credit return threshold for user send contexts, return when unreturned credits passes this many blocks (in percent of allocated blocks, 0 is off)");
82f48ad614SDennis Dalessandro
8303b92789SMatthew Wilcox DEFINE_XARRAY_FLAGS(hfi1_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
84f48ad614SDennis Dalessandro
hfi1_create_kctxt(struct hfi1_devdata * dd,struct hfi1_pportdata * ppd)85f2a3bc00SMichael J. Ruhl static int hfi1_create_kctxt(struct hfi1_devdata *dd,
86f2a3bc00SMichael J. Ruhl struct hfi1_pportdata *ppd)
87f48ad614SDennis Dalessandro {
88f2a3bc00SMichael J. Ruhl struct hfi1_ctxtdata *rcd;
89f48ad614SDennis Dalessandro int ret;
90f48ad614SDennis Dalessandro
91f48ad614SDennis Dalessandro /* Control context has to be always 0 */
92f48ad614SDennis Dalessandro BUILD_BUG_ON(HFI1_CTRL_CTXT != 0);
93f48ad614SDennis Dalessandro
94f2a3bc00SMichael J. Ruhl ret = hfi1_create_ctxtdata(ppd, dd->node, &rcd);
95f2a3bc00SMichael J. Ruhl if (ret < 0) {
96f2a3bc00SMichael J. Ruhl dd_dev_err(dd, "Kernel receive context allocation failed\n");
97f2a3bc00SMichael J. Ruhl return ret;
98f48ad614SDennis Dalessandro }
99f2a3bc00SMichael J. Ruhl
100f48ad614SDennis Dalessandro /*
101f2a3bc00SMichael J. Ruhl * Set up the kernel context flags here and now because they use
102f2a3bc00SMichael J. Ruhl * default values for all receive side memories. User contexts will
103f2a3bc00SMichael J. Ruhl * be handled as they are created.
104f48ad614SDennis Dalessandro */
105f48ad614SDennis Dalessandro rcd->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
106f48ad614SDennis Dalessandro HFI1_CAP_KGET(NODROP_RHQ_FULL) |
107f48ad614SDennis Dalessandro HFI1_CAP_KGET(NODROP_EGR_FULL) |
108f48ad614SDennis Dalessandro HFI1_CAP_KGET(DMA_RTAIL);
109f48ad614SDennis Dalessandro
110f48ad614SDennis Dalessandro /* Control context must use DMA_RTAIL */
111f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT)
112f48ad614SDennis Dalessandro rcd->flags |= HFI1_CAP_DMA_RTAIL;
11301c7fc50SMike Marciniszyn rcd->fast_handler = get_dma_rtail_setting(rcd) ?
11401c7fc50SMike Marciniszyn handle_receive_interrupt_dma_rtail :
11501c7fc50SMike Marciniszyn handle_receive_interrupt_nodma_rtail;
11601c7fc50SMike Marciniszyn
1172fb3b5aeSMike Marciniszyn hfi1_set_seq_cnt(rcd, 1);
118f48ad614SDennis Dalessandro
119f48ad614SDennis Dalessandro rcd->sc = sc_alloc(dd, SC_ACK, rcd->rcvhdrqentsize, dd->node);
120f48ad614SDennis Dalessandro if (!rcd->sc) {
121f2a3bc00SMichael J. Ruhl dd_dev_err(dd, "Kernel send context allocation failed\n");
122f2a3bc00SMichael J. Ruhl return -ENOMEM;
123f48ad614SDennis Dalessandro }
1249b60d2cbSMichael J. Ruhl hfi1_init_ctxt(rcd->sc);
125f2a3bc00SMichael J. Ruhl
126f2a3bc00SMichael J. Ruhl return 0;
127f48ad614SDennis Dalessandro }
128f48ad614SDennis Dalessandro
129f48ad614SDennis Dalessandro /*
130f2a3bc00SMichael J. Ruhl * Create the receive context array and one or more kernel contexts
131f48ad614SDennis Dalessandro */
hfi1_create_kctxts(struct hfi1_devdata * dd)132f2a3bc00SMichael J. Ruhl int hfi1_create_kctxts(struct hfi1_devdata *dd)
133f2a3bc00SMichael J. Ruhl {
134f2a3bc00SMichael J. Ruhl u16 i;
135f2a3bc00SMichael J. Ruhl int ret;
136f2a3bc00SMichael J. Ruhl
137953a9cebSKamenee Arumugam dd->rcd = kcalloc_node(dd->num_rcv_contexts, sizeof(*dd->rcd),
138f2a3bc00SMichael J. Ruhl GFP_KERNEL, dd->node);
139f2a3bc00SMichael J. Ruhl if (!dd->rcd)
140f2a3bc00SMichael J. Ruhl return -ENOMEM;
141f2a3bc00SMichael J. Ruhl
142f2a3bc00SMichael J. Ruhl for (i = 0; i < dd->first_dyn_alloc_ctxt; ++i) {
143f2a3bc00SMichael J. Ruhl ret = hfi1_create_kctxt(dd, dd->pport);
144f2a3bc00SMichael J. Ruhl if (ret)
145f2a3bc00SMichael J. Ruhl goto bail;
146f2a3bc00SMichael J. Ruhl }
147f48ad614SDennis Dalessandro
148f48ad614SDennis Dalessandro return 0;
149f2a3bc00SMichael J. Ruhl bail:
150f683c80cSMichael J. Ruhl for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i)
151d295dbebSMichael J. Ruhl hfi1_free_ctxt(dd->rcd[i]);
152f683c80cSMichael J. Ruhl
153f683c80cSMichael J. Ruhl /* All the contexts should be freed, free the array */
154f48ad614SDennis Dalessandro kfree(dd->rcd);
155f48ad614SDennis Dalessandro dd->rcd = NULL;
156f48ad614SDennis Dalessandro return ret;
157f48ad614SDennis Dalessandro }
158f48ad614SDennis Dalessandro
159f48ad614SDennis Dalessandro /*
160d295dbebSMichael J. Ruhl * Helper routines for the receive context reference count (rcd and uctxt).
161f683c80cSMichael J. Ruhl */
hfi1_rcd_init(struct hfi1_ctxtdata * rcd)162f683c80cSMichael J. Ruhl static void hfi1_rcd_init(struct hfi1_ctxtdata *rcd)
163f683c80cSMichael J. Ruhl {
164f683c80cSMichael J. Ruhl kref_init(&rcd->kref);
165f683c80cSMichael J. Ruhl }
166f683c80cSMichael J. Ruhl
167f2a3bc00SMichael J. Ruhl /**
168f2a3bc00SMichael J. Ruhl * hfi1_rcd_free - When reference is zero clean up.
169f2a3bc00SMichael J. Ruhl * @kref: pointer to an initialized rcd data structure
170f2a3bc00SMichael J. Ruhl *
171f2a3bc00SMichael J. Ruhl */
hfi1_rcd_free(struct kref * kref)172f683c80cSMichael J. Ruhl static void hfi1_rcd_free(struct kref *kref)
173f683c80cSMichael J. Ruhl {
174d295dbebSMichael J. Ruhl unsigned long flags;
175f683c80cSMichael J. Ruhl struct hfi1_ctxtdata *rcd =
176f683c80cSMichael J. Ruhl container_of(kref, struct hfi1_ctxtdata, kref);
177f683c80cSMichael J. Ruhl
178d295dbebSMichael J. Ruhl spin_lock_irqsave(&rcd->dd->uctxt_lock, flags);
179d295dbebSMichael J. Ruhl rcd->dd->rcd[rcd->ctxt] = NULL;
180d295dbebSMichael J. Ruhl spin_unlock_irqrestore(&rcd->dd->uctxt_lock, flags);
181d295dbebSMichael J. Ruhl
182bc5add09SMichael J. Ruhl hfi1_free_ctxtdata(rcd->dd, rcd);
183bc5add09SMichael J. Ruhl
184f683c80cSMichael J. Ruhl kfree(rcd);
185f683c80cSMichael J. Ruhl }
186f683c80cSMichael J. Ruhl
187f2a3bc00SMichael J. Ruhl /**
188f2a3bc00SMichael J. Ruhl * hfi1_rcd_put - decrement reference for rcd
189f2a3bc00SMichael J. Ruhl * @rcd: pointer to an initialized rcd data structure
190f2a3bc00SMichael J. Ruhl *
191f2a3bc00SMichael J. Ruhl * Use this to put a reference after the init.
192f2a3bc00SMichael J. Ruhl */
hfi1_rcd_put(struct hfi1_ctxtdata * rcd)193f683c80cSMichael J. Ruhl int hfi1_rcd_put(struct hfi1_ctxtdata *rcd)
194f683c80cSMichael J. Ruhl {
195f683c80cSMichael J. Ruhl if (rcd)
196f683c80cSMichael J. Ruhl return kref_put(&rcd->kref, hfi1_rcd_free);
197f683c80cSMichael J. Ruhl
198f683c80cSMichael J. Ruhl return 0;
199f683c80cSMichael J. Ruhl }
200f683c80cSMichael J. Ruhl
201f2a3bc00SMichael J. Ruhl /**
202f2a3bc00SMichael J. Ruhl * hfi1_rcd_get - increment reference for rcd
203f2a3bc00SMichael J. Ruhl * @rcd: pointer to an initialized rcd data structure
204f2a3bc00SMichael J. Ruhl *
205f2a3bc00SMichael J. Ruhl * Use this to get a reference after the init.
206bc5add09SMichael J. Ruhl *
207bc5add09SMichael J. Ruhl * Return : reflect kref_get_unless_zero(), which returns non-zero on
208bc5add09SMichael J. Ruhl * increment, otherwise 0.
209f2a3bc00SMichael J. Ruhl */
hfi1_rcd_get(struct hfi1_ctxtdata * rcd)210bc5add09SMichael J. Ruhl int hfi1_rcd_get(struct hfi1_ctxtdata *rcd)
211f683c80cSMichael J. Ruhl {
212bc5add09SMichael J. Ruhl return kref_get_unless_zero(&rcd->kref);
213f683c80cSMichael J. Ruhl }
214f683c80cSMichael J. Ruhl
215f2a3bc00SMichael J. Ruhl /**
216f2a3bc00SMichael J. Ruhl * allocate_rcd_index - allocate an rcd index from the rcd array
217f2a3bc00SMichael J. Ruhl * @dd: pointer to a valid devdata structure
218f2a3bc00SMichael J. Ruhl * @rcd: rcd data structure to assign
219f2a3bc00SMichael J. Ruhl * @index: pointer to index that is allocated
220f2a3bc00SMichael J. Ruhl *
221f2a3bc00SMichael J. Ruhl * Find an empty index in the rcd array, and assign the given rcd to it.
222f2a3bc00SMichael J. Ruhl * If the array is full, we are EBUSY.
223f2a3bc00SMichael J. Ruhl *
224f2a3bc00SMichael J. Ruhl */
allocate_rcd_index(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd,u16 * index)225d295dbebSMichael J. Ruhl static int allocate_rcd_index(struct hfi1_devdata *dd,
226f2a3bc00SMichael J. Ruhl struct hfi1_ctxtdata *rcd, u16 *index)
227f2a3bc00SMichael J. Ruhl {
228f2a3bc00SMichael J. Ruhl unsigned long flags;
229f2a3bc00SMichael J. Ruhl u16 ctxt;
230f2a3bc00SMichael J. Ruhl
231f2a3bc00SMichael J. Ruhl spin_lock_irqsave(&dd->uctxt_lock, flags);
232f2a3bc00SMichael J. Ruhl for (ctxt = 0; ctxt < dd->num_rcv_contexts; ctxt++)
233f2a3bc00SMichael J. Ruhl if (!dd->rcd[ctxt])
234f2a3bc00SMichael J. Ruhl break;
235f2a3bc00SMichael J. Ruhl
236f2a3bc00SMichael J. Ruhl if (ctxt < dd->num_rcv_contexts) {
237f2a3bc00SMichael J. Ruhl rcd->ctxt = ctxt;
238f2a3bc00SMichael J. Ruhl dd->rcd[ctxt] = rcd;
239f2a3bc00SMichael J. Ruhl hfi1_rcd_init(rcd);
240f2a3bc00SMichael J. Ruhl }
241f2a3bc00SMichael J. Ruhl spin_unlock_irqrestore(&dd->uctxt_lock, flags);
242f2a3bc00SMichael J. Ruhl
243f2a3bc00SMichael J. Ruhl if (ctxt >= dd->num_rcv_contexts)
244f2a3bc00SMichael J. Ruhl return -EBUSY;
245f2a3bc00SMichael J. Ruhl
246f2a3bc00SMichael J. Ruhl *index = ctxt;
247f2a3bc00SMichael J. Ruhl
248f2a3bc00SMichael J. Ruhl return 0;
249f2a3bc00SMichael J. Ruhl }
250f2a3bc00SMichael J. Ruhl
251d295dbebSMichael J. Ruhl /**
252d59075adSMichael J. Ruhl * hfi1_rcd_get_by_index_safe - validate the ctxt index before accessing the
253d59075adSMichael J. Ruhl * array
254d59075adSMichael J. Ruhl * @dd: pointer to a valid devdata structure
255d59075adSMichael J. Ruhl * @ctxt: the index of an possilbe rcd
256d59075adSMichael J. Ruhl *
257d59075adSMichael J. Ruhl * This is a wrapper for hfi1_rcd_get_by_index() to validate that the given
258d59075adSMichael J. Ruhl * ctxt index is valid.
259d59075adSMichael J. Ruhl *
260d59075adSMichael J. Ruhl * The caller is responsible for making the _put().
261d59075adSMichael J. Ruhl *
262d59075adSMichael J. Ruhl */
hfi1_rcd_get_by_index_safe(struct hfi1_devdata * dd,u16 ctxt)263d59075adSMichael J. Ruhl struct hfi1_ctxtdata *hfi1_rcd_get_by_index_safe(struct hfi1_devdata *dd,
264d59075adSMichael J. Ruhl u16 ctxt)
265d59075adSMichael J. Ruhl {
266d59075adSMichael J. Ruhl if (ctxt < dd->num_rcv_contexts)
267d59075adSMichael J. Ruhl return hfi1_rcd_get_by_index(dd, ctxt);
268d59075adSMichael J. Ruhl
269d59075adSMichael J. Ruhl return NULL;
270d59075adSMichael J. Ruhl }
271d59075adSMichael J. Ruhl
272d59075adSMichael J. Ruhl /**
273bf194997SLeon Romanovsky * hfi1_rcd_get_by_index - get by index
274d295dbebSMichael J. Ruhl * @dd: pointer to a valid devdata structure
275d295dbebSMichael J. Ruhl * @ctxt: the index of an possilbe rcd
276d295dbebSMichael J. Ruhl *
277d295dbebSMichael J. Ruhl * We need to protect access to the rcd array. If access is needed to
278d295dbebSMichael J. Ruhl * one or more index, get the protecting spinlock and then increment the
279d295dbebSMichael J. Ruhl * kref.
280d295dbebSMichael J. Ruhl *
281d295dbebSMichael J. Ruhl * The caller is responsible for making the _put().
282d295dbebSMichael J. Ruhl *
283d295dbebSMichael J. Ruhl */
hfi1_rcd_get_by_index(struct hfi1_devdata * dd,u16 ctxt)284d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *hfi1_rcd_get_by_index(struct hfi1_devdata *dd, u16 ctxt)
285d295dbebSMichael J. Ruhl {
286d295dbebSMichael J. Ruhl unsigned long flags;
287d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd = NULL;
288d295dbebSMichael J. Ruhl
289d295dbebSMichael J. Ruhl spin_lock_irqsave(&dd->uctxt_lock, flags);
290d295dbebSMichael J. Ruhl if (dd->rcd[ctxt]) {
291d295dbebSMichael J. Ruhl rcd = dd->rcd[ctxt];
292bc5add09SMichael J. Ruhl if (!hfi1_rcd_get(rcd))
293bc5add09SMichael J. Ruhl rcd = NULL;
294d295dbebSMichael J. Ruhl }
295d295dbebSMichael J. Ruhl spin_unlock_irqrestore(&dd->uctxt_lock, flags);
296d295dbebSMichael J. Ruhl
297d295dbebSMichael J. Ruhl return rcd;
298d295dbebSMichael J. Ruhl }
299d295dbebSMichael J. Ruhl
300f683c80cSMichael J. Ruhl /*
301d295dbebSMichael J. Ruhl * Common code for user and kernel context create and setup.
302d295dbebSMichael J. Ruhl * NOTE: the initial kref is done here (hf1_rcd_init()).
303f48ad614SDennis Dalessandro */
hfi1_create_ctxtdata(struct hfi1_pportdata * ppd,int numa,struct hfi1_ctxtdata ** context)304f2a3bc00SMichael J. Ruhl int hfi1_create_ctxtdata(struct hfi1_pportdata *ppd, int numa,
305f2a3bc00SMichael J. Ruhl struct hfi1_ctxtdata **context)
306f48ad614SDennis Dalessandro {
307f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd;
308f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd;
309f48ad614SDennis Dalessandro unsigned kctxt_ngroups = 0;
310f48ad614SDennis Dalessandro u32 base;
311f48ad614SDennis Dalessandro
312f48ad614SDennis Dalessandro if (dd->rcv_entries.nctxt_extra >
3132280740fSVishwanathapura, Niranjana dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt)
314f48ad614SDennis Dalessandro kctxt_ngroups = (dd->rcv_entries.nctxt_extra -
3152280740fSVishwanathapura, Niranjana (dd->num_rcv_contexts - dd->first_dyn_alloc_ctxt));
3164dfe7cceSJianxin Xiong rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, numa);
317f48ad614SDennis Dalessandro if (rcd) {
318f48ad614SDennis Dalessandro u32 rcvtids, max_entries;
319f2a3bc00SMichael J. Ruhl u16 ctxt;
320f2a3bc00SMichael J. Ruhl int ret;
321f48ad614SDennis Dalessandro
322f2a3bc00SMichael J. Ruhl ret = allocate_rcd_index(dd, rcd, &ctxt);
323f2a3bc00SMichael J. Ruhl if (ret) {
324f2a3bc00SMichael J. Ruhl *context = NULL;
325f2a3bc00SMichael J. Ruhl kfree(rcd);
326f2a3bc00SMichael J. Ruhl return ret;
327f2a3bc00SMichael J. Ruhl }
328f2a3bc00SMichael J. Ruhl
329f48ad614SDennis Dalessandro INIT_LIST_HEAD(&rcd->qp_wait_list);
330c8314811SMike Marciniszyn hfi1_exp_tid_group_init(rcd);
331f48ad614SDennis Dalessandro rcd->ppd = ppd;
332f48ad614SDennis Dalessandro rcd->dd = dd;
333f48ad614SDennis Dalessandro rcd->numa_id = numa;
334f48ad614SDennis Dalessandro rcd->rcv_array_groups = dd->rcv_entries.ngroups;
335b0ba3c18SMike Marciniszyn rcd->rhf_rcv_function_map = normal_rhf_rcv_functions;
336f6a3cfecSMike Marciniszyn rcd->slow_handler = handle_receive_interrupt;
337f6a3cfecSMike Marciniszyn rcd->do_interrupt = rcd->slow_handler;
3380bae02d5SGrzegorz Andrejczuk rcd->msix_intr = CCE_NUM_MSIX_VECTORS;
339f48ad614SDennis Dalessandro
340ed71e86aSKaike Wan mutex_init(&rcd->exp_mutex);
34137356e78SKaike Wan spin_lock_init(&rcd->exp_lock);
34237356e78SKaike Wan INIT_LIST_HEAD(&rcd->flow_queue.queue_head);
343838b6fd2SKaike Wan INIT_LIST_HEAD(&rcd->rarr_queue.queue_head);
344f48ad614SDennis Dalessandro
345*d2590edcSDean Luick hfi1_cdbg(PROC, "setting up context %u", rcd->ctxt);
346d295dbebSMichael J. Ruhl
347f48ad614SDennis Dalessandro /*
348f48ad614SDennis Dalessandro * Calculate the context's RcvArray entry starting point.
349f48ad614SDennis Dalessandro * We do this here because we have to take into account all
350f48ad614SDennis Dalessandro * the RcvArray entries that previous context would have
3512280740fSVishwanathapura, Niranjana * taken and we have to account for any extra groups assigned
3522280740fSVishwanathapura, Niranjana * to the static (kernel) or dynamic (vnic/user) contexts.
353f48ad614SDennis Dalessandro */
3542280740fSVishwanathapura, Niranjana if (ctxt < dd->first_dyn_alloc_ctxt) {
355f48ad614SDennis Dalessandro if (ctxt < kctxt_ngroups) {
356f48ad614SDennis Dalessandro base = ctxt * (dd->rcv_entries.ngroups + 1);
357f48ad614SDennis Dalessandro rcd->rcv_array_groups++;
358ee495adaSDennis Dalessandro } else {
359f48ad614SDennis Dalessandro base = kctxt_ngroups +
360f48ad614SDennis Dalessandro (ctxt * dd->rcv_entries.ngroups);
361ee495adaSDennis Dalessandro }
362f48ad614SDennis Dalessandro } else {
3632280740fSVishwanathapura, Niranjana u16 ct = ctxt - dd->first_dyn_alloc_ctxt;
364f48ad614SDennis Dalessandro
365f48ad614SDennis Dalessandro base = ((dd->n_krcv_queues * dd->rcv_entries.ngroups) +
366f48ad614SDennis Dalessandro kctxt_ngroups);
367f48ad614SDennis Dalessandro if (ct < dd->rcv_entries.nctxt_extra) {
368f48ad614SDennis Dalessandro base += ct * (dd->rcv_entries.ngroups + 1);
369f48ad614SDennis Dalessandro rcd->rcv_array_groups++;
370ee495adaSDennis Dalessandro } else {
371f48ad614SDennis Dalessandro base += dd->rcv_entries.nctxt_extra +
372f48ad614SDennis Dalessandro (ct * dd->rcv_entries.ngroups);
373f48ad614SDennis Dalessandro }
374ee495adaSDennis Dalessandro }
375f48ad614SDennis Dalessandro rcd->eager_base = base * dd->rcv_entries.group_size;
376f48ad614SDennis Dalessandro
377f48ad614SDennis Dalessandro rcd->rcvhdrq_cnt = rcvhdrcnt;
378f48ad614SDennis Dalessandro rcd->rcvhdrqentsize = hfi1_hdrq_entsize;
37940442b30SMike Marciniszyn rcd->rhf_offset =
38040442b30SMike Marciniszyn rcd->rcvhdrqentsize - sizeof(u64) / sizeof(u32);
381f48ad614SDennis Dalessandro /*
382f48ad614SDennis Dalessandro * Simple Eager buffer allocation: we have already pre-allocated
383f48ad614SDennis Dalessandro * the number of RcvArray entry groups. Each ctxtdata structure
384f48ad614SDennis Dalessandro * holds the number of groups for that context.
385f48ad614SDennis Dalessandro *
386f48ad614SDennis Dalessandro * To follow CSR requirements and maintain cacheline alignment,
387f48ad614SDennis Dalessandro * make sure all sizes and bases are multiples of group_size.
388f48ad614SDennis Dalessandro *
389f48ad614SDennis Dalessandro * The expected entry count is what is left after assigning
390f48ad614SDennis Dalessandro * eager.
391f48ad614SDennis Dalessandro */
392f48ad614SDennis Dalessandro max_entries = rcd->rcv_array_groups *
393f48ad614SDennis Dalessandro dd->rcv_entries.group_size;
394f48ad614SDennis Dalessandro rcvtids = ((max_entries * hfi1_rcvarr_split) / 100);
395f48ad614SDennis Dalessandro rcd->egrbufs.count = round_down(rcvtids,
396f48ad614SDennis Dalessandro dd->rcv_entries.group_size);
397f48ad614SDennis Dalessandro if (rcd->egrbufs.count > MAX_EAGER_ENTRIES) {
398f48ad614SDennis Dalessandro dd_dev_err(dd, "ctxt%u: requested too many RcvArray entries.\n",
399f48ad614SDennis Dalessandro rcd->ctxt);
400f48ad614SDennis Dalessandro rcd->egrbufs.count = MAX_EAGER_ENTRIES;
401f48ad614SDennis Dalessandro }
402f48ad614SDennis Dalessandro hfi1_cdbg(PROC,
403*d2590edcSDean Luick "ctxt%u: max Eager buffer RcvArray entries: %u",
404f48ad614SDennis Dalessandro rcd->ctxt, rcd->egrbufs.count);
405f48ad614SDennis Dalessandro
406f48ad614SDennis Dalessandro /*
407f48ad614SDennis Dalessandro * Allocate array that will hold the eager buffer accounting
408f48ad614SDennis Dalessandro * data.
409f48ad614SDennis Dalessandro * This will allocate the maximum possible buffer count based
410f48ad614SDennis Dalessandro * on the value of the RcvArray split parameter.
411f48ad614SDennis Dalessandro * The resulting value will be rounded down to the closest
412f48ad614SDennis Dalessandro * multiple of dd->rcv_entries.group_size.
413f48ad614SDennis Dalessandro */
414953a9cebSKamenee Arumugam rcd->egrbufs.buffers =
415953a9cebSKamenee Arumugam kcalloc_node(rcd->egrbufs.count,
416953a9cebSKamenee Arumugam sizeof(*rcd->egrbufs.buffers),
417b448bf9aSSebastian Sanchez GFP_KERNEL, numa);
418f48ad614SDennis Dalessandro if (!rcd->egrbufs.buffers)
419f48ad614SDennis Dalessandro goto bail;
420953a9cebSKamenee Arumugam rcd->egrbufs.rcvtids =
421953a9cebSKamenee Arumugam kcalloc_node(rcd->egrbufs.count,
422f48ad614SDennis Dalessandro sizeof(*rcd->egrbufs.rcvtids),
423b448bf9aSSebastian Sanchez GFP_KERNEL, numa);
424f48ad614SDennis Dalessandro if (!rcd->egrbufs.rcvtids)
425f48ad614SDennis Dalessandro goto bail;
426f48ad614SDennis Dalessandro rcd->egrbufs.size = eager_buffer_size;
427f48ad614SDennis Dalessandro /*
428f48ad614SDennis Dalessandro * The size of the buffers programmed into the RcvArray
429f48ad614SDennis Dalessandro * entries needs to be big enough to handle the highest
430f48ad614SDennis Dalessandro * MTU supported.
431f48ad614SDennis Dalessandro */
432f48ad614SDennis Dalessandro if (rcd->egrbufs.size < hfi1_max_mtu) {
433f48ad614SDennis Dalessandro rcd->egrbufs.size = __roundup_pow_of_two(hfi1_max_mtu);
434f48ad614SDennis Dalessandro hfi1_cdbg(PROC,
435*d2590edcSDean Luick "ctxt%u: eager bufs size too small. Adjusting to %u",
436f48ad614SDennis Dalessandro rcd->ctxt, rcd->egrbufs.size);
437f48ad614SDennis Dalessandro }
438f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size = HFI1_MAX_EAGER_BUFFER_SIZE;
439f48ad614SDennis Dalessandro
4402280740fSVishwanathapura, Niranjana /* Applicable only for statically created kernel contexts */
4412280740fSVishwanathapura, Niranjana if (ctxt < dd->first_dyn_alloc_ctxt) {
442b448bf9aSSebastian Sanchez rcd->opstats = kzalloc_node(sizeof(*rcd->opstats),
443b448bf9aSSebastian Sanchez GFP_KERNEL, numa);
444f48ad614SDennis Dalessandro if (!rcd->opstats)
445f48ad614SDennis Dalessandro goto bail;
44637356e78SKaike Wan
44737356e78SKaike Wan /* Initialize TID flow generations for the context */
44837356e78SKaike Wan hfi1_kern_init_ctxt_generations(rcd);
449f48ad614SDennis Dalessandro }
450f683c80cSMichael J. Ruhl
451f2a3bc00SMichael J. Ruhl *context = rcd;
452f2a3bc00SMichael J. Ruhl return 0;
453f48ad614SDennis Dalessandro }
454f2a3bc00SMichael J. Ruhl
455f48ad614SDennis Dalessandro bail:
456f2a3bc00SMichael J. Ruhl *context = NULL;
457d295dbebSMichael J. Ruhl hfi1_free_ctxt(rcd);
458f2a3bc00SMichael J. Ruhl return -ENOMEM;
459f2a3bc00SMichael J. Ruhl }
460f2a3bc00SMichael J. Ruhl
461f2a3bc00SMichael J. Ruhl /**
462bf194997SLeon Romanovsky * hfi1_free_ctxt - free context
463f2a3bc00SMichael J. Ruhl * @rcd: pointer to an initialized rcd data structure
464f2a3bc00SMichael J. Ruhl *
465d295dbebSMichael J. Ruhl * This wrapper is the free function that matches hfi1_create_ctxtdata().
466d295dbebSMichael J. Ruhl * When a context is done being used (kernel or user), this function is called
4671b8ba6e4SMiaoqian Lin * for the "final" put to match the kref init from hfi1_create_ctxtdata().
468d295dbebSMichael J. Ruhl * Other users of the context do a get/put sequence to make sure that the
469d295dbebSMichael J. Ruhl * structure isn't removed while in use.
470f2a3bc00SMichael J. Ruhl */
hfi1_free_ctxt(struct hfi1_ctxtdata * rcd)471d295dbebSMichael J. Ruhl void hfi1_free_ctxt(struct hfi1_ctxtdata *rcd)
472f2a3bc00SMichael J. Ruhl {
473f2a3bc00SMichael J. Ruhl hfi1_rcd_put(rcd);
474f2a3bc00SMichael J. Ruhl }
475f48ad614SDennis Dalessandro
476f48ad614SDennis Dalessandro /*
477f48ad614SDennis Dalessandro * Select the largest ccti value over all SLs to determine the intra-
478f48ad614SDennis Dalessandro * packet gap for the link.
479f48ad614SDennis Dalessandro *
480f48ad614SDennis Dalessandro * called with cca_timer_lock held (to protect access to cca_timer
481f48ad614SDennis Dalessandro * array), and rcu_read_lock() (to protect access to cc_state).
482f48ad614SDennis Dalessandro */
set_link_ipg(struct hfi1_pportdata * ppd)483f48ad614SDennis Dalessandro void set_link_ipg(struct hfi1_pportdata *ppd)
484f48ad614SDennis Dalessandro {
485f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd;
486f48ad614SDennis Dalessandro struct cc_state *cc_state;
487f48ad614SDennis Dalessandro int i;
488f48ad614SDennis Dalessandro u16 cce, ccti_limit, max_ccti = 0;
489f48ad614SDennis Dalessandro u16 shift, mult;
490f48ad614SDennis Dalessandro u64 src;
491f48ad614SDennis Dalessandro u32 current_egress_rate; /* Mbits /sec */
492f93e91a0SDennis Dalessandro u64 max_pkt_time;
493f48ad614SDennis Dalessandro /*
494f48ad614SDennis Dalessandro * max_pkt_time is the maximum packet egress time in units
495f48ad614SDennis Dalessandro * of the fabric clock period 1/(805 MHz).
496f48ad614SDennis Dalessandro */
497f48ad614SDennis Dalessandro
498f48ad614SDennis Dalessandro cc_state = get_cc_state(ppd);
499f48ad614SDennis Dalessandro
500f48ad614SDennis Dalessandro if (!cc_state)
501f48ad614SDennis Dalessandro /*
502f48ad614SDennis Dalessandro * This should _never_ happen - rcu_read_lock() is held,
503f48ad614SDennis Dalessandro * and set_link_ipg() should not be called if cc_state
504f48ad614SDennis Dalessandro * is NULL.
505f48ad614SDennis Dalessandro */
506f48ad614SDennis Dalessandro return;
507f48ad614SDennis Dalessandro
508f48ad614SDennis Dalessandro for (i = 0; i < OPA_MAX_SLS; i++) {
509f48ad614SDennis Dalessandro u16 ccti = ppd->cca_timer[i].ccti;
510f48ad614SDennis Dalessandro
511f48ad614SDennis Dalessandro if (ccti > max_ccti)
512f48ad614SDennis Dalessandro max_ccti = ccti;
513f48ad614SDennis Dalessandro }
514f48ad614SDennis Dalessandro
515f48ad614SDennis Dalessandro ccti_limit = cc_state->cct.ccti_limit;
516f48ad614SDennis Dalessandro if (max_ccti > ccti_limit)
517f48ad614SDennis Dalessandro max_ccti = ccti_limit;
518f48ad614SDennis Dalessandro
519f48ad614SDennis Dalessandro cce = cc_state->cct.entries[max_ccti].entry;
520f48ad614SDennis Dalessandro shift = (cce & 0xc000) >> 14;
521f48ad614SDennis Dalessandro mult = (cce & 0x3fff);
522f48ad614SDennis Dalessandro
523f48ad614SDennis Dalessandro current_egress_rate = active_egress_rate(ppd);
524f48ad614SDennis Dalessandro
525f48ad614SDennis Dalessandro max_pkt_time = egress_cycles(ppd->ibmaxlen, current_egress_rate);
526f48ad614SDennis Dalessandro
527f48ad614SDennis Dalessandro src = (max_pkt_time >> shift) * mult;
528f48ad614SDennis Dalessandro
529f48ad614SDennis Dalessandro src &= SEND_STATIC_RATE_CONTROL_CSR_SRC_RELOAD_SMASK;
530f48ad614SDennis Dalessandro src <<= SEND_STATIC_RATE_CONTROL_CSR_SRC_RELOAD_SHIFT;
531f48ad614SDennis Dalessandro
532f48ad614SDennis Dalessandro write_csr(dd, SEND_STATIC_RATE_CONTROL, src);
533f48ad614SDennis Dalessandro }
534f48ad614SDennis Dalessandro
cca_timer_fn(struct hrtimer * t)535f48ad614SDennis Dalessandro static enum hrtimer_restart cca_timer_fn(struct hrtimer *t)
536f48ad614SDennis Dalessandro {
537f48ad614SDennis Dalessandro struct cca_timer *cca_timer;
538f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
539f48ad614SDennis Dalessandro int sl;
540f48ad614SDennis Dalessandro u16 ccti_timer, ccti_min;
541f48ad614SDennis Dalessandro struct cc_state *cc_state;
542f48ad614SDennis Dalessandro unsigned long flags;
543f48ad614SDennis Dalessandro enum hrtimer_restart ret = HRTIMER_NORESTART;
544f48ad614SDennis Dalessandro
545f48ad614SDennis Dalessandro cca_timer = container_of(t, struct cca_timer, hrtimer);
546f48ad614SDennis Dalessandro ppd = cca_timer->ppd;
547f48ad614SDennis Dalessandro sl = cca_timer->sl;
548f48ad614SDennis Dalessandro
549f48ad614SDennis Dalessandro rcu_read_lock();
550f48ad614SDennis Dalessandro
551f48ad614SDennis Dalessandro cc_state = get_cc_state(ppd);
552f48ad614SDennis Dalessandro
553f48ad614SDennis Dalessandro if (!cc_state) {
554f48ad614SDennis Dalessandro rcu_read_unlock();
555f48ad614SDennis Dalessandro return HRTIMER_NORESTART;
556f48ad614SDennis Dalessandro }
557f48ad614SDennis Dalessandro
558f48ad614SDennis Dalessandro /*
559f48ad614SDennis Dalessandro * 1) decrement ccti for SL
560f48ad614SDennis Dalessandro * 2) calculate IPG for link (set_link_ipg())
561f48ad614SDennis Dalessandro * 3) restart timer, unless ccti is at min value
562f48ad614SDennis Dalessandro */
563f48ad614SDennis Dalessandro
564f48ad614SDennis Dalessandro ccti_min = cc_state->cong_setting.entries[sl].ccti_min;
565f48ad614SDennis Dalessandro ccti_timer = cc_state->cong_setting.entries[sl].ccti_timer;
566f48ad614SDennis Dalessandro
567f48ad614SDennis Dalessandro spin_lock_irqsave(&ppd->cca_timer_lock, flags);
568f48ad614SDennis Dalessandro
569f48ad614SDennis Dalessandro if (cca_timer->ccti > ccti_min) {
570f48ad614SDennis Dalessandro cca_timer->ccti--;
571f48ad614SDennis Dalessandro set_link_ipg(ppd);
572f48ad614SDennis Dalessandro }
573f48ad614SDennis Dalessandro
574f48ad614SDennis Dalessandro if (cca_timer->ccti > ccti_min) {
575f48ad614SDennis Dalessandro unsigned long nsec = 1024 * ccti_timer;
576f48ad614SDennis Dalessandro /* ccti_timer is in units of 1.024 usec */
577f48ad614SDennis Dalessandro hrtimer_forward_now(t, ns_to_ktime(nsec));
578f48ad614SDennis Dalessandro ret = HRTIMER_RESTART;
579f48ad614SDennis Dalessandro }
580f48ad614SDennis Dalessandro
581f48ad614SDennis Dalessandro spin_unlock_irqrestore(&ppd->cca_timer_lock, flags);
582f48ad614SDennis Dalessandro rcu_read_unlock();
583f48ad614SDennis Dalessandro return ret;
584f48ad614SDennis Dalessandro }
585f48ad614SDennis Dalessandro
586f48ad614SDennis Dalessandro /*
587f48ad614SDennis Dalessandro * Common code for initializing the physical port structure.
588f48ad614SDennis Dalessandro */
hfi1_init_pportdata(struct pci_dev * pdev,struct hfi1_pportdata * ppd,struct hfi1_devdata * dd,u8 hw_pidx,u32 port)589f48ad614SDennis Dalessandro void hfi1_init_pportdata(struct pci_dev *pdev, struct hfi1_pportdata *ppd,
5901fb7f897SMark Bloch struct hfi1_devdata *dd, u8 hw_pidx, u32 port)
591f48ad614SDennis Dalessandro {
5928adf71faSJianxin Xiong int i;
593f48ad614SDennis Dalessandro uint default_pkey_idx;
5948adf71faSJianxin Xiong struct cc_state *cc_state;
595f48ad614SDennis Dalessandro
596f48ad614SDennis Dalessandro ppd->dd = dd;
597f48ad614SDennis Dalessandro ppd->hw_pidx = hw_pidx;
598f48ad614SDennis Dalessandro ppd->port = port; /* IB port number, not index */
59907190076SKamenee Arumugam ppd->prev_link_width = LINK_WIDTH_DEFAULT;
60007190076SKamenee Arumugam /*
60107190076SKamenee Arumugam * There are C_VL_COUNT number of PortVLXmitWait counters.
60207190076SKamenee Arumugam * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
60307190076SKamenee Arumugam */
60407190076SKamenee Arumugam for (i = 0; i < C_VL_COUNT + 1; i++) {
60507190076SKamenee Arumugam ppd->port_vl_xmit_wait_last[i] = 0;
60607190076SKamenee Arumugam ppd->vl_xmit_flit_cnt[i] = 0;
60707190076SKamenee Arumugam }
608f48ad614SDennis Dalessandro
609f48ad614SDennis Dalessandro default_pkey_idx = 1;
610f48ad614SDennis Dalessandro
611f48ad614SDennis Dalessandro ppd->pkeys[default_pkey_idx] = DEFAULT_P_KEY;
61253526500SNeel Desai ppd->part_enforce |= HFI1_PART_ENFORCE_IN;
61362004871SMike Marciniszyn ppd->pkeys[0] = 0x8001;
614f48ad614SDennis Dalessandro
615f48ad614SDennis Dalessandro INIT_WORK(&ppd->link_vc_work, handle_verify_cap);
616f48ad614SDennis Dalessandro INIT_WORK(&ppd->link_up_work, handle_link_up);
617f48ad614SDennis Dalessandro INIT_WORK(&ppd->link_down_work, handle_link_down);
618f48ad614SDennis Dalessandro INIT_WORK(&ppd->freeze_work, handle_freeze);
619f48ad614SDennis Dalessandro INIT_WORK(&ppd->link_downgrade_work, handle_link_downgrade);
620f48ad614SDennis Dalessandro INIT_WORK(&ppd->sma_message_work, handle_sma_message);
621f48ad614SDennis Dalessandro INIT_WORK(&ppd->link_bounce_work, handle_link_bounce);
622673b975fSDean Luick INIT_DELAYED_WORK(&ppd->start_link_work, handle_start_link);
623f48ad614SDennis Dalessandro INIT_WORK(&ppd->linkstate_active_work, receive_interrupt_work);
624f48ad614SDennis Dalessandro INIT_WORK(&ppd->qsfp_info.qsfp_work, qsfp_event);
625f48ad614SDennis Dalessandro
626f48ad614SDennis Dalessandro mutex_init(&ppd->hls_lock);
627f48ad614SDennis Dalessandro spin_lock_init(&ppd->qsfp_info.qsfp_lock);
628f48ad614SDennis Dalessandro
629f48ad614SDennis Dalessandro ppd->qsfp_info.ppd = ppd;
630f48ad614SDennis Dalessandro ppd->sm_trap_qp = 0x0;
631f48ad614SDennis Dalessandro ppd->sa_qp = 0x1;
632f48ad614SDennis Dalessandro
633f48ad614SDennis Dalessandro ppd->hfi1_wq = NULL;
634f48ad614SDennis Dalessandro
635f48ad614SDennis Dalessandro spin_lock_init(&ppd->cca_timer_lock);
636f48ad614SDennis Dalessandro
637f48ad614SDennis Dalessandro for (i = 0; i < OPA_MAX_SLS; i++) {
638f48ad614SDennis Dalessandro hrtimer_init(&ppd->cca_timer[i].hrtimer, CLOCK_MONOTONIC,
639f48ad614SDennis Dalessandro HRTIMER_MODE_REL);
640f48ad614SDennis Dalessandro ppd->cca_timer[i].ppd = ppd;
641f48ad614SDennis Dalessandro ppd->cca_timer[i].sl = i;
642f48ad614SDennis Dalessandro ppd->cca_timer[i].ccti = 0;
643f48ad614SDennis Dalessandro ppd->cca_timer[i].hrtimer.function = cca_timer_fn;
644f48ad614SDennis Dalessandro }
645f48ad614SDennis Dalessandro
646f48ad614SDennis Dalessandro ppd->cc_max_table_entries = IB_CC_TABLE_CAP_DEFAULT;
647f48ad614SDennis Dalessandro
648f48ad614SDennis Dalessandro spin_lock_init(&ppd->cc_state_lock);
649f48ad614SDennis Dalessandro spin_lock_init(&ppd->cc_log_lock);
6508adf71faSJianxin Xiong cc_state = kzalloc(sizeof(*cc_state), GFP_KERNEL);
6518adf71faSJianxin Xiong RCU_INIT_POINTER(ppd->cc_state, cc_state);
6528adf71faSJianxin Xiong if (!cc_state)
653f48ad614SDennis Dalessandro goto bail;
654f48ad614SDennis Dalessandro return;
655f48ad614SDennis Dalessandro
656f48ad614SDennis Dalessandro bail:
65757f97e96SMichael J. Ruhl dd_dev_err(dd, "Congestion Control Agent disabled for port %d\n", port);
658f48ad614SDennis Dalessandro }
659f48ad614SDennis Dalessandro
660f48ad614SDennis Dalessandro /*
661f48ad614SDennis Dalessandro * Do initialization for device that is only needed on
662f48ad614SDennis Dalessandro * first detect, not on resets.
663f48ad614SDennis Dalessandro */
loadtime_init(struct hfi1_devdata * dd)664f48ad614SDennis Dalessandro static int loadtime_init(struct hfi1_devdata *dd)
665f48ad614SDennis Dalessandro {
666f48ad614SDennis Dalessandro return 0;
667f48ad614SDennis Dalessandro }
668f48ad614SDennis Dalessandro
669f48ad614SDennis Dalessandro /**
670f48ad614SDennis Dalessandro * init_after_reset - re-initialize after a reset
671f48ad614SDennis Dalessandro * @dd: the hfi1_ib device
672f48ad614SDennis Dalessandro *
673f48ad614SDennis Dalessandro * sanity check at least some of the values after reset, and
674f48ad614SDennis Dalessandro * ensure no receive or transmit (explicitly, in case reset
675f48ad614SDennis Dalessandro * failed
676f48ad614SDennis Dalessandro */
init_after_reset(struct hfi1_devdata * dd)677f48ad614SDennis Dalessandro static int init_after_reset(struct hfi1_devdata *dd)
678f48ad614SDennis Dalessandro {
679f48ad614SDennis Dalessandro int i;
680d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd;
681f48ad614SDennis Dalessandro /*
682f48ad614SDennis Dalessandro * Ensure chip does no sends or receives, tail updates, or
683f48ad614SDennis Dalessandro * pioavail updates while we re-initialize. This is mostly
684f48ad614SDennis Dalessandro * for the driver data structures, not chip registers.
685f48ad614SDennis Dalessandro */
686d295dbebSMichael J. Ruhl for (i = 0; i < dd->num_rcv_contexts; i++) {
687d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
688f48ad614SDennis Dalessandro hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
689f48ad614SDennis Dalessandro HFI1_RCVCTRL_INTRAVAIL_DIS |
690d295dbebSMichael J. Ruhl HFI1_RCVCTRL_TAILUPD_DIS, rcd);
691d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd);
692d295dbebSMichael J. Ruhl }
693f48ad614SDennis Dalessandro pio_send_control(dd, PSC_GLOBAL_DISABLE);
694f48ad614SDennis Dalessandro for (i = 0; i < dd->num_send_contexts; i++)
695f48ad614SDennis Dalessandro sc_disable(dd->send_contexts[i].sc);
696f48ad614SDennis Dalessandro
697f48ad614SDennis Dalessandro return 0;
698f48ad614SDennis Dalessandro }
699f48ad614SDennis Dalessandro
enable_chip(struct hfi1_devdata * dd)700f48ad614SDennis Dalessandro static void enable_chip(struct hfi1_devdata *dd)
701f48ad614SDennis Dalessandro {
702d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd;
703f48ad614SDennis Dalessandro u32 rcvmask;
704e6f7622dSMichael J. Ruhl u16 i;
705f48ad614SDennis Dalessandro
706f48ad614SDennis Dalessandro /* enable PIO send */
707f48ad614SDennis Dalessandro pio_send_control(dd, PSC_GLOBAL_ENABLE);
708f48ad614SDennis Dalessandro
709f48ad614SDennis Dalessandro /*
710f48ad614SDennis Dalessandro * Enable kernel ctxts' receive and receive interrupt.
711f48ad614SDennis Dalessandro * Other ctxts done as user opens and initializes them.
712f48ad614SDennis Dalessandro */
7132280740fSVishwanathapura, Niranjana for (i = 0; i < dd->first_dyn_alloc_ctxt; ++i) {
714d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
715d295dbebSMichael J. Ruhl if (!rcd)
716d295dbebSMichael J. Ruhl continue;
717f48ad614SDennis Dalessandro rcvmask = HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB;
718d295dbebSMichael J. Ruhl rcvmask |= HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
719f48ad614SDennis Dalessandro HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS;
720d295dbebSMichael J. Ruhl if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
721f48ad614SDennis Dalessandro rcvmask |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
722d295dbebSMichael J. Ruhl if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_RHQ_FULL))
723f48ad614SDennis Dalessandro rcvmask |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
724d295dbebSMichael J. Ruhl if (HFI1_CAP_KGET_MASK(rcd->flags, NODROP_EGR_FULL))
725f48ad614SDennis Dalessandro rcvmask |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
72637356e78SKaike Wan if (HFI1_CAP_IS_KSET(TID_RDMA))
72737356e78SKaike Wan rcvmask |= HFI1_RCVCTRL_TIDFLOW_ENB;
728d295dbebSMichael J. Ruhl hfi1_rcvctrl(dd, rcvmask, rcd);
729d295dbebSMichael J. Ruhl sc_enable(rcd->sc);
730d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd);
731f48ad614SDennis Dalessandro }
732f48ad614SDennis Dalessandro }
733f48ad614SDennis Dalessandro
734f48ad614SDennis Dalessandro /**
735f48ad614SDennis Dalessandro * create_workqueues - create per port workqueues
736f48ad614SDennis Dalessandro * @dd: the hfi1_ib device
737f48ad614SDennis Dalessandro */
create_workqueues(struct hfi1_devdata * dd)738f48ad614SDennis Dalessandro static int create_workqueues(struct hfi1_devdata *dd)
739f48ad614SDennis Dalessandro {
740f48ad614SDennis Dalessandro int pidx;
741f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
742f48ad614SDennis Dalessandro
743f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
744f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
745f48ad614SDennis Dalessandro if (!ppd->hfi1_wq) {
746f48ad614SDennis Dalessandro ppd->hfi1_wq =
747f48ad614SDennis Dalessandro alloc_workqueue(
748f48ad614SDennis Dalessandro "hfi%d_%d",
7494c4b1996SMike Marciniszyn WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE |
7504c4b1996SMike Marciniszyn WQ_MEM_RECLAIM,
751dd1ed108SMike Marciniszyn HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES,
752f48ad614SDennis Dalessandro dd->unit, pidx);
753f48ad614SDennis Dalessandro if (!ppd->hfi1_wq)
754f48ad614SDennis Dalessandro goto wq_error;
755f48ad614SDennis Dalessandro }
75671d47008SSebastian Sanchez if (!ppd->link_wq) {
75771d47008SSebastian Sanchez /*
75871d47008SSebastian Sanchez * Make the link workqueue single-threaded to enforce
75971d47008SSebastian Sanchez * serialization.
76071d47008SSebastian Sanchez */
76171d47008SSebastian Sanchez ppd->link_wq =
76271d47008SSebastian Sanchez alloc_workqueue(
76371d47008SSebastian Sanchez "hfi_link_%d_%d",
76471d47008SSebastian Sanchez WQ_SYSFS | WQ_MEM_RECLAIM | WQ_UNBOUND,
76571d47008SSebastian Sanchez 1, /* max_active */
76671d47008SSebastian Sanchez dd->unit, pidx);
76771d47008SSebastian Sanchez if (!ppd->link_wq)
76871d47008SSebastian Sanchez goto wq_error;
76971d47008SSebastian Sanchez }
770f48ad614SDennis Dalessandro }
771f48ad614SDennis Dalessandro return 0;
772f48ad614SDennis Dalessandro wq_error:
773f48ad614SDennis Dalessandro pr_err("alloc_workqueue failed for port %d\n", pidx + 1);
774f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
775f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
776f48ad614SDennis Dalessandro if (ppd->hfi1_wq) {
777f48ad614SDennis Dalessandro destroy_workqueue(ppd->hfi1_wq);
778f48ad614SDennis Dalessandro ppd->hfi1_wq = NULL;
779f48ad614SDennis Dalessandro }
78071d47008SSebastian Sanchez if (ppd->link_wq) {
78171d47008SSebastian Sanchez destroy_workqueue(ppd->link_wq);
78271d47008SSebastian Sanchez ppd->link_wq = NULL;
78371d47008SSebastian Sanchez }
784f48ad614SDennis Dalessandro }
785f48ad614SDennis Dalessandro return -ENOMEM;
786f48ad614SDennis Dalessandro }
787f48ad614SDennis Dalessandro
788f48ad614SDennis Dalessandro /**
78928b70cd9SKaike Wan * destroy_workqueues - destroy per port workqueues
79028b70cd9SKaike Wan * @dd: the hfi1_ib device
79128b70cd9SKaike Wan */
destroy_workqueues(struct hfi1_devdata * dd)79228b70cd9SKaike Wan static void destroy_workqueues(struct hfi1_devdata *dd)
79328b70cd9SKaike Wan {
79428b70cd9SKaike Wan int pidx;
79528b70cd9SKaike Wan struct hfi1_pportdata *ppd;
79628b70cd9SKaike Wan
79728b70cd9SKaike Wan for (pidx = 0; pidx < dd->num_pports; ++pidx) {
79828b70cd9SKaike Wan ppd = dd->pport + pidx;
79928b70cd9SKaike Wan
80028b70cd9SKaike Wan if (ppd->hfi1_wq) {
80128b70cd9SKaike Wan destroy_workqueue(ppd->hfi1_wq);
80228b70cd9SKaike Wan ppd->hfi1_wq = NULL;
80328b70cd9SKaike Wan }
8042315ec12SKaike Wan if (ppd->link_wq) {
8052315ec12SKaike Wan destroy_workqueue(ppd->link_wq);
8062315ec12SKaike Wan ppd->link_wq = NULL;
8072315ec12SKaike Wan }
80828b70cd9SKaike Wan }
80928b70cd9SKaike Wan }
81028b70cd9SKaike Wan
81128b70cd9SKaike Wan /**
812a2f7bbdcSMichael J. Ruhl * enable_general_intr() - Enable the IRQs that will be handled by the
813a2f7bbdcSMichael J. Ruhl * general interrupt handler.
814a2f7bbdcSMichael J. Ruhl * @dd: valid devdata
815a2f7bbdcSMichael J. Ruhl *
816a2f7bbdcSMichael J. Ruhl */
enable_general_intr(struct hfi1_devdata * dd)817a2f7bbdcSMichael J. Ruhl static void enable_general_intr(struct hfi1_devdata *dd)
818a2f7bbdcSMichael J. Ruhl {
819a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, CCE_ERR_INT, MISC_ERR_INT, true);
820a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, PIO_ERR_INT, TXE_ERR_INT, true);
821a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END, true);
822a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, PBC_INT, GPIO_ASSERT_INT, true);
823a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, TCRIT_INT, TCRIT_INT, true);
824a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, IS_DC_START, IS_DC_END, true);
825a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, IS_SENDCREDIT_START, IS_SENDCREDIT_END, true);
826a2f7bbdcSMichael J. Ruhl }
827a2f7bbdcSMichael J. Ruhl
828a2f7bbdcSMichael J. Ruhl /**
829f48ad614SDennis Dalessandro * hfi1_init - do the actual initialization sequence on the chip
830f48ad614SDennis Dalessandro * @dd: the hfi1_ib device
831f48ad614SDennis Dalessandro * @reinit: re-initializing, so don't allocate new memory
832f48ad614SDennis Dalessandro *
833f48ad614SDennis Dalessandro * Do the actual initialization sequence on the chip. This is done
834f48ad614SDennis Dalessandro * both from the init routine called from the PCI infrastructure, and
835f48ad614SDennis Dalessandro * when we reset the chip, or detect that it was reset internally,
836f48ad614SDennis Dalessandro * or it's administratively re-enabled.
837f48ad614SDennis Dalessandro *
838f48ad614SDennis Dalessandro * Memory allocation here and in called routines is only done in
839f48ad614SDennis Dalessandro * the first case (reinit == 0). We have to be careful, because even
840f48ad614SDennis Dalessandro * without memory allocation, we need to re-write all the chip registers
841f48ad614SDennis Dalessandro * TIDs, etc. after the reset or enable has completed.
842f48ad614SDennis Dalessandro */
hfi1_init(struct hfi1_devdata * dd,int reinit)843f48ad614SDennis Dalessandro int hfi1_init(struct hfi1_devdata *dd, int reinit)
844f48ad614SDennis Dalessandro {
845f48ad614SDennis Dalessandro int ret = 0, pidx, lastfail = 0;
846e6f7622dSMichael J. Ruhl unsigned long len;
847e6f7622dSMichael J. Ruhl u16 i;
848f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd;
849f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
850f48ad614SDennis Dalessandro
851f48ad614SDennis Dalessandro /* Set up send low level handlers */
852f48ad614SDennis Dalessandro dd->process_pio_send = hfi1_verbs_send_pio;
853f48ad614SDennis Dalessandro dd->process_dma_send = hfi1_verbs_send_dma;
854f48ad614SDennis Dalessandro dd->pio_inline_send = pio_copy;
85564551edeSVishwanathapura, Niranjana dd->process_vnic_dma_send = hfi1_vnic_send_dma;
856f48ad614SDennis Dalessandro
857f48ad614SDennis Dalessandro if (is_ax(dd)) {
858f48ad614SDennis Dalessandro atomic_set(&dd->drop_packet, DROP_PACKET_ON);
859cd47b594SMike Marciniszyn dd->do_drop = true;
860f48ad614SDennis Dalessandro } else {
861f48ad614SDennis Dalessandro atomic_set(&dd->drop_packet, DROP_PACKET_OFF);
862cd47b594SMike Marciniszyn dd->do_drop = false;
863f48ad614SDennis Dalessandro }
864f48ad614SDennis Dalessandro
865f48ad614SDennis Dalessandro /* make sure the link is not "up" */
866f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
867f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
868f48ad614SDennis Dalessandro ppd->linkup = 0;
869f48ad614SDennis Dalessandro }
870f48ad614SDennis Dalessandro
871f48ad614SDennis Dalessandro if (reinit)
872f48ad614SDennis Dalessandro ret = init_after_reset(dd);
873f48ad614SDennis Dalessandro else
874f48ad614SDennis Dalessandro ret = loadtime_init(dd);
875f48ad614SDennis Dalessandro if (ret)
876f48ad614SDennis Dalessandro goto done;
877f48ad614SDennis Dalessandro
878f48ad614SDennis Dalessandro /* dd->rcd can be NULL if early initialization failed */
8792280740fSVishwanathapura, Niranjana for (i = 0; dd->rcd && i < dd->first_dyn_alloc_ctxt; ++i) {
880f48ad614SDennis Dalessandro /*
881f48ad614SDennis Dalessandro * Set up the (kernel) rcvhdr queue and egr TIDs. If doing
882f48ad614SDennis Dalessandro * re-init, the simplest way to handle this is to free
883f48ad614SDennis Dalessandro * existing, and re-allocate.
884f48ad614SDennis Dalessandro * Need to re-create rest of ctxt 0 ctxtdata as well.
885f48ad614SDennis Dalessandro */
886d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
887f48ad614SDennis Dalessandro if (!rcd)
888f48ad614SDennis Dalessandro continue;
889f48ad614SDennis Dalessandro
890f48ad614SDennis Dalessandro lastfail = hfi1_create_rcvhdrq(dd, rcd);
891f48ad614SDennis Dalessandro if (!lastfail)
892f48ad614SDennis Dalessandro lastfail = hfi1_setup_eagerbufs(rcd);
893d22a207dSKaike Wan if (!lastfail)
894d22a207dSKaike Wan lastfail = hfi1_kern_exp_rcv_init(rcd, reinit);
895f48ad614SDennis Dalessandro if (lastfail) {
896f48ad614SDennis Dalessandro dd_dev_err(dd,
897f48ad614SDennis Dalessandro "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");
898f48ad614SDennis Dalessandro ret = lastfail;
899f48ad614SDennis Dalessandro }
900a2f7bbdcSMichael J. Ruhl /* enable IRQ */
901d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd);
902f48ad614SDennis Dalessandro }
903f48ad614SDennis Dalessandro
904f48ad614SDennis Dalessandro /* Allocate enough memory for user event notification. */
90506e81e3eSMike Marciniszyn len = PAGE_ALIGN(chip_rcv_contexts(dd) * HFI1_MAX_SHARED_CTXTS *
906f48ad614SDennis Dalessandro sizeof(*dd->events));
907f48ad614SDennis Dalessandro dd->events = vmalloc_user(len);
908f48ad614SDennis Dalessandro if (!dd->events)
909f48ad614SDennis Dalessandro dd_dev_err(dd, "Failed to allocate user events page\n");
910f48ad614SDennis Dalessandro /*
911f48ad614SDennis Dalessandro * Allocate a page for device and port status.
912f48ad614SDennis Dalessandro * Page will be shared amongst all user processes.
913f48ad614SDennis Dalessandro */
914f48ad614SDennis Dalessandro dd->status = vmalloc_user(PAGE_SIZE);
915f48ad614SDennis Dalessandro if (!dd->status)
916f48ad614SDennis Dalessandro dd_dev_err(dd, "Failed to allocate dev status page\n");
917f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
918f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
919f48ad614SDennis Dalessandro if (dd->status)
920f48ad614SDennis Dalessandro /* Currently, we only have one port */
921f48ad614SDennis Dalessandro ppd->statusp = &dd->status->port;
922f48ad614SDennis Dalessandro
923f48ad614SDennis Dalessandro set_mtu(ppd);
924f48ad614SDennis Dalessandro }
925f48ad614SDennis Dalessandro
926f48ad614SDennis Dalessandro /* enable chip even if we have an error, so we can debug cause */
927f48ad614SDennis Dalessandro enable_chip(dd);
928f48ad614SDennis Dalessandro
929f48ad614SDennis Dalessandro done:
930f48ad614SDennis Dalessandro /*
931f48ad614SDennis Dalessandro * Set status even if port serdes is not initialized
932f48ad614SDennis Dalessandro * so that diags will work.
933f48ad614SDennis Dalessandro */
934f48ad614SDennis Dalessandro if (dd->status)
935f48ad614SDennis Dalessandro dd->status->dev |= HFI1_STATUS_CHIP_PRESENT |
936f48ad614SDennis Dalessandro HFI1_STATUS_INITTED;
937f48ad614SDennis Dalessandro if (!ret) {
938f48ad614SDennis Dalessandro /* enable all interrupts from the chip */
939a2f7bbdcSMichael J. Ruhl enable_general_intr(dd);
940a2f7bbdcSMichael J. Ruhl init_qsfp_int(dd);
941f48ad614SDennis Dalessandro
942f48ad614SDennis Dalessandro /* chip is OK for user apps; mark it as initialized */
943f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
944f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
945f48ad614SDennis Dalessandro
946f48ad614SDennis Dalessandro /*
947f48ad614SDennis Dalessandro * start the serdes - must be after interrupts are
948f48ad614SDennis Dalessandro * enabled so we are notified when the link goes up
949f48ad614SDennis Dalessandro */
950f48ad614SDennis Dalessandro lastfail = bringup_serdes(ppd);
951f48ad614SDennis Dalessandro if (lastfail)
952f48ad614SDennis Dalessandro dd_dev_info(dd,
953f48ad614SDennis Dalessandro "Failed to bring up port %u\n",
954f48ad614SDennis Dalessandro ppd->port);
955f48ad614SDennis Dalessandro
956f48ad614SDennis Dalessandro /*
957f48ad614SDennis Dalessandro * Set status even if port serdes is not initialized
958f48ad614SDennis Dalessandro * so that diags will work.
959f48ad614SDennis Dalessandro */
960f48ad614SDennis Dalessandro if (ppd->statusp)
961f48ad614SDennis Dalessandro *ppd->statusp |= HFI1_STATUS_CHIP_PRESENT |
962f48ad614SDennis Dalessandro HFI1_STATUS_INITTED;
963f48ad614SDennis Dalessandro if (!ppd->link_speed_enabled)
964f48ad614SDennis Dalessandro continue;
965f48ad614SDennis Dalessandro }
966f48ad614SDennis Dalessandro }
967f48ad614SDennis Dalessandro
968f48ad614SDennis Dalessandro /* if ret is non-zero, we probably should do some cleanup here... */
969f48ad614SDennis Dalessandro return ret;
970f48ad614SDennis Dalessandro }
971f48ad614SDennis Dalessandro
hfi1_lookup(int unit)972f48ad614SDennis Dalessandro struct hfi1_devdata *hfi1_lookup(int unit)
973f48ad614SDennis Dalessandro {
97403b92789SMatthew Wilcox return xa_load(&hfi1_dev_table, unit);
975f48ad614SDennis Dalessandro }
976f48ad614SDennis Dalessandro
977f48ad614SDennis Dalessandro /*
978f48ad614SDennis Dalessandro * Stop the timers during unit shutdown, or after an error late
979f48ad614SDennis Dalessandro * in initialization.
980f48ad614SDennis Dalessandro */
stop_timers(struct hfi1_devdata * dd)981f48ad614SDennis Dalessandro static void stop_timers(struct hfi1_devdata *dd)
982f48ad614SDennis Dalessandro {
983f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
984f48ad614SDennis Dalessandro int pidx;
985f48ad614SDennis Dalessandro
986f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
987f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
9888064135eSKees Cook if (ppd->led_override_timer.function) {
989f48ad614SDennis Dalessandro del_timer_sync(&ppd->led_override_timer);
990f48ad614SDennis Dalessandro atomic_set(&ppd->led_override_timer_active, 0);
991f48ad614SDennis Dalessandro }
992f48ad614SDennis Dalessandro }
993f48ad614SDennis Dalessandro }
994f48ad614SDennis Dalessandro
995f48ad614SDennis Dalessandro /**
996f48ad614SDennis Dalessandro * shutdown_device - shut down a device
997f48ad614SDennis Dalessandro * @dd: the hfi1_ib device
998f48ad614SDennis Dalessandro *
999f48ad614SDennis Dalessandro * This is called to make the device quiet when we are about to
1000f48ad614SDennis Dalessandro * unload the driver, and also when the device is administratively
1001f48ad614SDennis Dalessandro * disabled. It does not free any data structures.
1002f48ad614SDennis Dalessandro * Everything it does has to be setup again by hfi1_init(dd, 1)
1003f48ad614SDennis Dalessandro */
shutdown_device(struct hfi1_devdata * dd)1004f48ad614SDennis Dalessandro static void shutdown_device(struct hfi1_devdata *dd)
1005f48ad614SDennis Dalessandro {
1006f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
1007d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd;
1008f48ad614SDennis Dalessandro unsigned pidx;
1009f48ad614SDennis Dalessandro int i;
1010f48ad614SDennis Dalessandro
10118d3e7113SAlex Estrin if (dd->flags & HFI1_SHUTDOWN)
10128d3e7113SAlex Estrin return;
10138d3e7113SAlex Estrin dd->flags |= HFI1_SHUTDOWN;
10148d3e7113SAlex Estrin
1015f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1016f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
1017f48ad614SDennis Dalessandro
1018f48ad614SDennis Dalessandro ppd->linkup = 0;
1019f48ad614SDennis Dalessandro if (ppd->statusp)
1020f48ad614SDennis Dalessandro *ppd->statusp &= ~(HFI1_STATUS_IB_CONF |
1021f48ad614SDennis Dalessandro HFI1_STATUS_IB_READY);
1022f48ad614SDennis Dalessandro }
1023f48ad614SDennis Dalessandro dd->flags &= ~HFI1_INITTED;
1024f48ad614SDennis Dalessandro
1025a2f7bbdcSMichael J. Ruhl /* mask and clean up interrupts */
1026a2f7bbdcSMichael J. Ruhl set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false);
10276eb4eb10SMichael J. Ruhl msix_clean_up_interrupts(dd);
1028f48ad614SDennis Dalessandro
1029f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1030f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
1031d295dbebSMichael J. Ruhl for (i = 0; i < dd->num_rcv_contexts; i++) {
1032d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i);
1033f48ad614SDennis Dalessandro hfi1_rcvctrl(dd, HFI1_RCVCTRL_TAILUPD_DIS |
1034f48ad614SDennis Dalessandro HFI1_RCVCTRL_CTXT_DIS |
1035f48ad614SDennis Dalessandro HFI1_RCVCTRL_INTRAVAIL_DIS |
1036f48ad614SDennis Dalessandro HFI1_RCVCTRL_PKEY_DIS |
1037d295dbebSMichael J. Ruhl HFI1_RCVCTRL_ONE_PKT_EGR_DIS, rcd);
1038d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd);
1039d295dbebSMichael J. Ruhl }
1040f48ad614SDennis Dalessandro /*
1041f48ad614SDennis Dalessandro * Gracefully stop all sends allowing any in progress to
1042f48ad614SDennis Dalessandro * trickle out first.
1043f48ad614SDennis Dalessandro */
1044f48ad614SDennis Dalessandro for (i = 0; i < dd->num_send_contexts; i++)
1045f48ad614SDennis Dalessandro sc_flush(dd->send_contexts[i].sc);
1046f48ad614SDennis Dalessandro }
1047f48ad614SDennis Dalessandro
1048f48ad614SDennis Dalessandro /*
1049f48ad614SDennis Dalessandro * Enough for anything that's going to trickle out to have actually
1050f48ad614SDennis Dalessandro * done so.
1051f48ad614SDennis Dalessandro */
1052f48ad614SDennis Dalessandro udelay(20);
1053f48ad614SDennis Dalessandro
1054f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1055f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
1056f48ad614SDennis Dalessandro
1057f48ad614SDennis Dalessandro /* disable all contexts */
1058f48ad614SDennis Dalessandro for (i = 0; i < dd->num_send_contexts; i++)
1059f48ad614SDennis Dalessandro sc_disable(dd->send_contexts[i].sc);
1060f48ad614SDennis Dalessandro /* disable the send device */
1061f48ad614SDennis Dalessandro pio_send_control(dd, PSC_GLOBAL_DISABLE);
1062f48ad614SDennis Dalessandro
1063f48ad614SDennis Dalessandro shutdown_led_override(ppd);
1064f48ad614SDennis Dalessandro
1065f48ad614SDennis Dalessandro /*
1066f48ad614SDennis Dalessandro * Clear SerdesEnable.
1067f48ad614SDennis Dalessandro * We can't count on interrupts since we are stopping.
1068f48ad614SDennis Dalessandro */
1069f48ad614SDennis Dalessandro hfi1_quiet_serdes(ppd);
107028b70cd9SKaike Wan if (ppd->hfi1_wq)
107128b70cd9SKaike Wan flush_workqueue(ppd->hfi1_wq);
10722315ec12SKaike Wan if (ppd->link_wq)
107328b70cd9SKaike Wan flush_workqueue(ppd->link_wq);
1074f48ad614SDennis Dalessandro }
1075f48ad614SDennis Dalessandro sdma_exit(dd);
1076f48ad614SDennis Dalessandro }
1077f48ad614SDennis Dalessandro
1078f48ad614SDennis Dalessandro /**
1079f48ad614SDennis Dalessandro * hfi1_free_ctxtdata - free a context's allocated data
1080f48ad614SDennis Dalessandro * @dd: the hfi1_ib device
1081f48ad614SDennis Dalessandro * @rcd: the ctxtdata structure
1082f48ad614SDennis Dalessandro *
1083f48ad614SDennis Dalessandro * free up any allocated data for a context
1084f48ad614SDennis Dalessandro * It should never change any chip state, or global driver state.
1085f48ad614SDennis Dalessandro */
hfi1_free_ctxtdata(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd)1086f48ad614SDennis Dalessandro void hfi1_free_ctxtdata(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1087f48ad614SDennis Dalessandro {
1088f683c80cSMichael J. Ruhl u32 e;
1089f48ad614SDennis Dalessandro
1090f48ad614SDennis Dalessandro if (!rcd)
1091f48ad614SDennis Dalessandro return;
1092f48ad614SDennis Dalessandro
1093f48ad614SDennis Dalessandro if (rcd->rcvhdrq) {
1094b2578431SMike Marciniszyn dma_free_coherent(&dd->pcidev->dev, rcvhdrq_size(rcd),
109560368186STymoteusz Kielan rcd->rcvhdrq, rcd->rcvhdrq_dma);
1096f48ad614SDennis Dalessandro rcd->rcvhdrq = NULL;
10972fb3b5aeSMike Marciniszyn if (hfi1_rcvhdrtail_kvaddr(rcd)) {
1098f48ad614SDennis Dalessandro dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,
10992fb3b5aeSMike Marciniszyn (void *)hfi1_rcvhdrtail_kvaddr(rcd),
110060368186STymoteusz Kielan rcd->rcvhdrqtailaddr_dma);
1101f48ad614SDennis Dalessandro rcd->rcvhdrtail_kvaddr = NULL;
1102f48ad614SDennis Dalessandro }
1103f48ad614SDennis Dalessandro }
1104f48ad614SDennis Dalessandro
1105f48ad614SDennis Dalessandro /* all the RcvArray entries should have been cleared by now */
1106f48ad614SDennis Dalessandro kfree(rcd->egrbufs.rcvtids);
1107f683c80cSMichael J. Ruhl rcd->egrbufs.rcvtids = NULL;
1108f48ad614SDennis Dalessandro
1109f48ad614SDennis Dalessandro for (e = 0; e < rcd->egrbufs.alloced; e++) {
11109292f8f9SMike Marciniszyn if (rcd->egrbufs.buffers[e].addr)
1111f48ad614SDennis Dalessandro dma_free_coherent(&dd->pcidev->dev,
1112f48ad614SDennis Dalessandro rcd->egrbufs.buffers[e].len,
1113f48ad614SDennis Dalessandro rcd->egrbufs.buffers[e].addr,
111460368186STymoteusz Kielan rcd->egrbufs.buffers[e].dma);
1115f48ad614SDennis Dalessandro }
1116f48ad614SDennis Dalessandro kfree(rcd->egrbufs.buffers);
1117f683c80cSMichael J. Ruhl rcd->egrbufs.alloced = 0;
1118f683c80cSMichael J. Ruhl rcd->egrbufs.buffers = NULL;
1119f48ad614SDennis Dalessandro
1120f48ad614SDennis Dalessandro sc_free(rcd->sc);
1121f683c80cSMichael J. Ruhl rcd->sc = NULL;
1122f683c80cSMichael J. Ruhl
1123f48ad614SDennis Dalessandro vfree(rcd->subctxt_uregbase);
1124f48ad614SDennis Dalessandro vfree(rcd->subctxt_rcvegrbuf);
1125f48ad614SDennis Dalessandro vfree(rcd->subctxt_rcvhdr_base);
1126f48ad614SDennis Dalessandro kfree(rcd->opstats);
1127f683c80cSMichael J. Ruhl
1128f683c80cSMichael J. Ruhl rcd->subctxt_uregbase = NULL;
1129f683c80cSMichael J. Ruhl rcd->subctxt_rcvegrbuf = NULL;
1130f683c80cSMichael J. Ruhl rcd->subctxt_rcvhdr_base = NULL;
1131f683c80cSMichael J. Ruhl rcd->opstats = NULL;
1132f48ad614SDennis Dalessandro }
1133f48ad614SDennis Dalessandro
1134f48ad614SDennis Dalessandro /*
1135f48ad614SDennis Dalessandro * Release our hold on the shared asic data. If we are the last one,
1136dba715f0SDean Luick * return the structure to be finalized outside the lock. Must be
113703b92789SMatthew Wilcox * holding hfi1_dev_table lock.
1138f48ad614SDennis Dalessandro */
release_asic_data(struct hfi1_devdata * dd)1139dba715f0SDean Luick static struct hfi1_asic_data *release_asic_data(struct hfi1_devdata *dd)
1140f48ad614SDennis Dalessandro {
1141dba715f0SDean Luick struct hfi1_asic_data *ad;
1142f48ad614SDennis Dalessandro int other;
1143f48ad614SDennis Dalessandro
1144f48ad614SDennis Dalessandro if (!dd->asic_data)
1145dba715f0SDean Luick return NULL;
1146f48ad614SDennis Dalessandro dd->asic_data->dds[dd->hfi1_id] = NULL;
1147f48ad614SDennis Dalessandro other = dd->hfi1_id ? 0 : 1;
1148dba715f0SDean Luick ad = dd->asic_data;
1149f48ad614SDennis Dalessandro dd->asic_data = NULL;
1150dba715f0SDean Luick /* return NULL if the other dd still has a link */
1151dba715f0SDean Luick return ad->dds[other] ? NULL : ad;
1152dba715f0SDean Luick }
1153dba715f0SDean Luick
finalize_asic_data(struct hfi1_devdata * dd,struct hfi1_asic_data * ad)1154dba715f0SDean Luick static void finalize_asic_data(struct hfi1_devdata *dd,
1155dba715f0SDean Luick struct hfi1_asic_data *ad)
1156dba715f0SDean Luick {
1157dba715f0SDean Luick clean_up_i2c(dd, ad);
1158dba715f0SDean Luick kfree(ad);
1159f48ad614SDennis Dalessandro }
1160f48ad614SDennis Dalessandro
1161e9777ad4SSebastian Sanchez /**
11625ab17a24SKaike Wan * hfi1_free_devdata - cleans up and frees per-unit data structure
1163e9777ad4SSebastian Sanchez * @dd: pointer to a valid devdata structure
1164e9777ad4SSebastian Sanchez *
11655ab17a24SKaike Wan * It cleans up and frees all data structures set up by
1166e9777ad4SSebastian Sanchez * by hfi1_alloc_devdata().
1167e9777ad4SSebastian Sanchez */
hfi1_free_devdata(struct hfi1_devdata * dd)11685ab17a24SKaike Wan void hfi1_free_devdata(struct hfi1_devdata *dd)
1169f48ad614SDennis Dalessandro {
1170dba715f0SDean Luick struct hfi1_asic_data *ad;
1171f48ad614SDennis Dalessandro unsigned long flags;
1172f48ad614SDennis Dalessandro
117303b92789SMatthew Wilcox xa_lock_irqsave(&hfi1_dev_table, flags);
117403b92789SMatthew Wilcox __xa_erase(&hfi1_dev_table, dd->unit);
1175dba715f0SDean Luick ad = release_asic_data(dd);
117603b92789SMatthew Wilcox xa_unlock_irqrestore(&hfi1_dev_table, flags);
1177e9777ad4SSebastian Sanchez
1178dba715f0SDean Luick finalize_asic_data(dd, ad);
1179f48ad614SDennis Dalessandro free_platform_config(dd);
1180f48ad614SDennis Dalessandro rcu_barrier(); /* wait for rcu callbacks to complete */
1181f48ad614SDennis Dalessandro free_percpu(dd->int_counter);
1182f48ad614SDennis Dalessandro free_percpu(dd->rcv_limit);
1183f48ad614SDennis Dalessandro free_percpu(dd->send_schedule);
11841b311f89SMike Marciniszyn free_percpu(dd->tx_opstats);
1185e9777ad4SSebastian Sanchez dd->int_counter = NULL;
1186e9777ad4SSebastian Sanchez dd->rcv_limit = NULL;
1187e9777ad4SSebastian Sanchez dd->send_schedule = NULL;
1188e9777ad4SSebastian Sanchez dd->tx_opstats = NULL;
11895d18ee67SSebastian Sanchez kfree(dd->comp_vect);
11905d18ee67SSebastian Sanchez dd->comp_vect = NULL;
119160a8b5a1SMike Marciniszyn if (dd->rcvhdrtail_dummy_kvaddr)
119260a8b5a1SMike Marciniszyn dma_free_coherent(&dd->pcidev->dev, sizeof(u64),
119360a8b5a1SMike Marciniszyn (void *)dd->rcvhdrtail_dummy_kvaddr,
119460a8b5a1SMike Marciniszyn dd->rcvhdrtail_dummy_dma);
119560a8b5a1SMike Marciniszyn dd->rcvhdrtail_dummy_kvaddr = NULL;
1196473291b3SAlex Estrin sdma_clean(dd, dd->num_sdma);
1197f48ad614SDennis Dalessandro rvt_dealloc_device(&dd->verbs_dev.rdi);
1198f48ad614SDennis Dalessandro }
1199f48ad614SDennis Dalessandro
120057f97e96SMichael J. Ruhl /**
120157f97e96SMichael J. Ruhl * hfi1_alloc_devdata - Allocate our primary per-unit data structure.
120257f97e96SMichael J. Ruhl * @pdev: Valid PCI device
120357f97e96SMichael J. Ruhl * @extra: How many bytes to alloc past the default
120457f97e96SMichael J. Ruhl *
120557f97e96SMichael J. Ruhl * Must be done via verbs allocator, because the verbs cleanup process
120657f97e96SMichael J. Ruhl * both does cleanup and free of the data structure.
1207f48ad614SDennis Dalessandro * "extra" is for chip-specific data.
1208f48ad614SDennis Dalessandro */
hfi1_alloc_devdata(struct pci_dev * pdev,size_t extra)120957f97e96SMichael J. Ruhl static struct hfi1_devdata *hfi1_alloc_devdata(struct pci_dev *pdev,
121057f97e96SMichael J. Ruhl size_t extra)
1211f48ad614SDennis Dalessandro {
1212f48ad614SDennis Dalessandro struct hfi1_devdata *dd;
1213f48ad614SDennis Dalessandro int ret, nports;
1214f48ad614SDennis Dalessandro
1215f48ad614SDennis Dalessandro /* extra is * number of ports */
1216f48ad614SDennis Dalessandro nports = extra / sizeof(struct hfi1_pportdata);
1217f48ad614SDennis Dalessandro
1218f48ad614SDennis Dalessandro dd = (struct hfi1_devdata *)rvt_alloc_device(sizeof(*dd) + extra,
1219f48ad614SDennis Dalessandro nports);
1220f48ad614SDennis Dalessandro if (!dd)
1221f48ad614SDennis Dalessandro return ERR_PTR(-ENOMEM);
1222f48ad614SDennis Dalessandro dd->num_pports = nports;
1223f48ad614SDennis Dalessandro dd->pport = (struct hfi1_pportdata *)(dd + 1);
122445d92457SSebastian Sanchez dd->pcidev = pdev;
122545d92457SSebastian Sanchez pci_set_drvdata(pdev, dd);
1226f48ad614SDennis Dalessandro
122703b92789SMatthew Wilcox ret = xa_alloc_irq(&hfi1_dev_table, &dd->unit, dd, xa_limit_32b,
122803b92789SMatthew Wilcox GFP_KERNEL);
1229f48ad614SDennis Dalessandro if (ret < 0) {
123057f97e96SMichael J. Ruhl dev_err(&pdev->dev,
1231f48ad614SDennis Dalessandro "Could not allocate unit ID: error %d\n", -ret);
1232f48ad614SDennis Dalessandro goto bail;
1233f48ad614SDennis Dalessandro }
12345084c8ffSMichael J. Ruhl rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s_%d", class_name(), dd->unit);
12355de61a47SMike Marciniszyn /*
12365de61a47SMike Marciniszyn * If the BIOS does not have the NUMA node information set, select
12375de61a47SMike Marciniszyn * NUMA 0 so we get consistent performance.
12385de61a47SMike Marciniszyn */
12395de61a47SMike Marciniszyn dd->node = pcibus_to_node(pdev->bus);
12405de61a47SMike Marciniszyn if (dd->node == NUMA_NO_NODE) {
12415de61a47SMike Marciniszyn dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
12425de61a47SMike Marciniszyn dd->node = 0;
12435de61a47SMike Marciniszyn }
12445084c8ffSMichael J. Ruhl
1245f48ad614SDennis Dalessandro /*
1246f48ad614SDennis Dalessandro * Initialize all locks for the device. This needs to be as early as
1247f48ad614SDennis Dalessandro * possible so locks are usable.
1248f48ad614SDennis Dalessandro */
1249f48ad614SDennis Dalessandro spin_lock_init(&dd->sc_lock);
1250f48ad614SDennis Dalessandro spin_lock_init(&dd->sendctrl_lock);
1251f48ad614SDennis Dalessandro spin_lock_init(&dd->rcvctrl_lock);
1252f48ad614SDennis Dalessandro spin_lock_init(&dd->uctxt_lock);
1253f48ad614SDennis Dalessandro spin_lock_init(&dd->hfi1_diag_trans_lock);
1254f48ad614SDennis Dalessandro spin_lock_init(&dd->sc_init_lock);
1255f48ad614SDennis Dalessandro spin_lock_init(&dd->dc8051_memlock);
1256f48ad614SDennis Dalessandro seqlock_init(&dd->sc2vl_lock);
1257f48ad614SDennis Dalessandro spin_lock_init(&dd->sde_map_lock);
1258f48ad614SDennis Dalessandro spin_lock_init(&dd->pio_map_lock);
125922546b74STadeusz Struk mutex_init(&dd->dc8051_lock);
1260f48ad614SDennis Dalessandro init_waitqueue_head(&dd->event_queue);
1261a2f7bbdcSMichael J. Ruhl spin_lock_init(&dd->irq_src_lock);
1262f48ad614SDennis Dalessandro
1263f48ad614SDennis Dalessandro dd->int_counter = alloc_percpu(u64);
1264f48ad614SDennis Dalessandro if (!dd->int_counter) {
1265f48ad614SDennis Dalessandro ret = -ENOMEM;
1266f48ad614SDennis Dalessandro goto bail;
1267f48ad614SDennis Dalessandro }
1268f48ad614SDennis Dalessandro
1269f48ad614SDennis Dalessandro dd->rcv_limit = alloc_percpu(u64);
1270f48ad614SDennis Dalessandro if (!dd->rcv_limit) {
1271f48ad614SDennis Dalessandro ret = -ENOMEM;
1272f48ad614SDennis Dalessandro goto bail;
1273f48ad614SDennis Dalessandro }
1274f48ad614SDennis Dalessandro
1275f48ad614SDennis Dalessandro dd->send_schedule = alloc_percpu(u64);
1276f48ad614SDennis Dalessandro if (!dd->send_schedule) {
1277f48ad614SDennis Dalessandro ret = -ENOMEM;
1278f48ad614SDennis Dalessandro goto bail;
1279f48ad614SDennis Dalessandro }
1280f48ad614SDennis Dalessandro
12811b311f89SMike Marciniszyn dd->tx_opstats = alloc_percpu(struct hfi1_opcode_stats_perctx);
12821b311f89SMike Marciniszyn if (!dd->tx_opstats) {
12831b311f89SMike Marciniszyn ret = -ENOMEM;
12841b311f89SMike Marciniszyn goto bail;
12851b311f89SMike Marciniszyn }
12861b311f89SMike Marciniszyn
12875d18ee67SSebastian Sanchez dd->comp_vect = kzalloc(sizeof(*dd->comp_vect), GFP_KERNEL);
12885d18ee67SSebastian Sanchez if (!dd->comp_vect) {
12895d18ee67SSebastian Sanchez ret = -ENOMEM;
12905d18ee67SSebastian Sanchez goto bail;
12915d18ee67SSebastian Sanchez }
12925d18ee67SSebastian Sanchez
129360a8b5a1SMike Marciniszyn /* allocate dummy tail memory for all receive contexts */
129460a8b5a1SMike Marciniszyn dd->rcvhdrtail_dummy_kvaddr =
129560a8b5a1SMike Marciniszyn dma_alloc_coherent(&dd->pcidev->dev, sizeof(u64),
129660a8b5a1SMike Marciniszyn &dd->rcvhdrtail_dummy_dma, GFP_KERNEL);
129760a8b5a1SMike Marciniszyn if (!dd->rcvhdrtail_dummy_kvaddr) {
129860a8b5a1SMike Marciniszyn ret = -ENOMEM;
129960a8b5a1SMike Marciniszyn goto bail;
130060a8b5a1SMike Marciniszyn }
130160a8b5a1SMike Marciniszyn
130219d8b90aSGrzegorz Andrejczuk atomic_set(&dd->ipoib_rsm_usr_num, 0);
1303f48ad614SDennis Dalessandro return dd;
1304f48ad614SDennis Dalessandro
1305f48ad614SDennis Dalessandro bail:
13065ab17a24SKaike Wan hfi1_free_devdata(dd);
1307f48ad614SDennis Dalessandro return ERR_PTR(ret);
1308f48ad614SDennis Dalessandro }
1309f48ad614SDennis Dalessandro
1310f48ad614SDennis Dalessandro /*
1311f48ad614SDennis Dalessandro * Called from freeze mode handlers, and from PCI error
1312f48ad614SDennis Dalessandro * reporting code. Should be paranoid about state of
1313f48ad614SDennis Dalessandro * system and data structures.
1314f48ad614SDennis Dalessandro */
hfi1_disable_after_error(struct hfi1_devdata * dd)1315f48ad614SDennis Dalessandro void hfi1_disable_after_error(struct hfi1_devdata *dd)
1316f48ad614SDennis Dalessandro {
1317f48ad614SDennis Dalessandro if (dd->flags & HFI1_INITTED) {
1318f48ad614SDennis Dalessandro u32 pidx;
1319f48ad614SDennis Dalessandro
1320f48ad614SDennis Dalessandro dd->flags &= ~HFI1_INITTED;
1321f48ad614SDennis Dalessandro if (dd->pport)
1322f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1323f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
1324f48ad614SDennis Dalessandro
1325f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
1326f48ad614SDennis Dalessandro if (dd->flags & HFI1_PRESENT)
1327f48ad614SDennis Dalessandro set_link_state(ppd, HLS_DN_DISABLE);
1328f48ad614SDennis Dalessandro
1329f48ad614SDennis Dalessandro if (ppd->statusp)
1330f48ad614SDennis Dalessandro *ppd->statusp &= ~HFI1_STATUS_IB_READY;
1331f48ad614SDennis Dalessandro }
1332f48ad614SDennis Dalessandro }
1333f48ad614SDennis Dalessandro
1334f48ad614SDennis Dalessandro /*
1335f48ad614SDennis Dalessandro * Mark as having had an error for driver, and also
1336f48ad614SDennis Dalessandro * for /sys and status word mapped to user programs.
1337f48ad614SDennis Dalessandro * This marks unit as not usable, until reset.
1338f48ad614SDennis Dalessandro */
1339f48ad614SDennis Dalessandro if (dd->status)
1340f48ad614SDennis Dalessandro dd->status->dev |= HFI1_STATUS_HWERROR;
1341f48ad614SDennis Dalessandro }
1342f48ad614SDennis Dalessandro
1343f48ad614SDennis Dalessandro static void remove_one(struct pci_dev *);
1344f48ad614SDennis Dalessandro static int init_one(struct pci_dev *, const struct pci_device_id *);
13458d3e7113SAlex Estrin static void shutdown_one(struct pci_dev *);
1346f48ad614SDennis Dalessandro
1347ddf65f28SScott Breyer #define DRIVER_LOAD_MSG "Cornelis " DRIVER_NAME " loaded: "
1348f48ad614SDennis Dalessandro #define PFX DRIVER_NAME ": "
1349f48ad614SDennis Dalessandro
1350d6373019SSebastian Sanchez const struct pci_device_id hfi1_pci_tbl[] = {
1351f48ad614SDennis Dalessandro { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL0) },
1352f48ad614SDennis Dalessandro { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL1) },
1353f48ad614SDennis Dalessandro { 0, }
1354f48ad614SDennis Dalessandro };
1355f48ad614SDennis Dalessandro
1356f48ad614SDennis Dalessandro MODULE_DEVICE_TABLE(pci, hfi1_pci_tbl);
1357f48ad614SDennis Dalessandro
1358f48ad614SDennis Dalessandro static struct pci_driver hfi1_pci_driver = {
1359f48ad614SDennis Dalessandro .name = DRIVER_NAME,
1360f48ad614SDennis Dalessandro .probe = init_one,
1361f48ad614SDennis Dalessandro .remove = remove_one,
13628d3e7113SAlex Estrin .shutdown = shutdown_one,
1363f48ad614SDennis Dalessandro .id_table = hfi1_pci_tbl,
1364f48ad614SDennis Dalessandro .err_handler = &hfi1_pci_err_handler,
1365f48ad614SDennis Dalessandro };
1366f48ad614SDennis Dalessandro
compute_krcvqs(void)1367f48ad614SDennis Dalessandro static void __init compute_krcvqs(void)
1368f48ad614SDennis Dalessandro {
1369f48ad614SDennis Dalessandro int i;
1370f48ad614SDennis Dalessandro
1371f48ad614SDennis Dalessandro for (i = 0; i < krcvqsset; i++)
1372f48ad614SDennis Dalessandro n_krcvqs += krcvqs[i];
1373f48ad614SDennis Dalessandro }
1374f48ad614SDennis Dalessandro
1375f48ad614SDennis Dalessandro /*
1376f48ad614SDennis Dalessandro * Do all the generic driver unit- and chip-independent memory
1377f48ad614SDennis Dalessandro * allocation and initialization.
1378f48ad614SDennis Dalessandro */
hfi1_mod_init(void)1379f48ad614SDennis Dalessandro static int __init hfi1_mod_init(void)
1380f48ad614SDennis Dalessandro {
1381f48ad614SDennis Dalessandro int ret;
1382f48ad614SDennis Dalessandro
1383f48ad614SDennis Dalessandro ret = dev_init();
1384f48ad614SDennis Dalessandro if (ret)
1385f48ad614SDennis Dalessandro goto bail;
1386f48ad614SDennis Dalessandro
1387d6373019SSebastian Sanchez ret = node_affinity_init();
1388d6373019SSebastian Sanchez if (ret)
1389d6373019SSebastian Sanchez goto bail;
13904197344bSDennis Dalessandro
1391f48ad614SDennis Dalessandro /* validate max MTU before any devices start */
1392f48ad614SDennis Dalessandro if (!valid_opa_max_mtu(hfi1_max_mtu)) {
1393f48ad614SDennis Dalessandro pr_err("Invalid max_mtu 0x%x, using 0x%x instead\n",
1394f48ad614SDennis Dalessandro hfi1_max_mtu, HFI1_DEFAULT_MAX_MTU);
1395f48ad614SDennis Dalessandro hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
1396f48ad614SDennis Dalessandro }
1397f48ad614SDennis Dalessandro /* valid CUs run from 1-128 in powers of 2 */
1398f48ad614SDennis Dalessandro if (hfi1_cu > 128 || !is_power_of_2(hfi1_cu))
1399f48ad614SDennis Dalessandro hfi1_cu = 1;
1400f48ad614SDennis Dalessandro /* valid credit return threshold is 0-100, variable is unsigned */
1401f48ad614SDennis Dalessandro if (user_credit_return_threshold > 100)
1402f48ad614SDennis Dalessandro user_credit_return_threshold = 100;
1403f48ad614SDennis Dalessandro
1404f48ad614SDennis Dalessandro compute_krcvqs();
1405f48ad614SDennis Dalessandro /*
1406f48ad614SDennis Dalessandro * sanitize receive interrupt count, time must wait until after
1407f48ad614SDennis Dalessandro * the hardware type is known
1408f48ad614SDennis Dalessandro */
1409f48ad614SDennis Dalessandro if (rcv_intr_count > RCV_HDR_HEAD_COUNTER_MASK)
1410f48ad614SDennis Dalessandro rcv_intr_count = RCV_HDR_HEAD_COUNTER_MASK;
1411f48ad614SDennis Dalessandro /* reject invalid combinations */
1412f48ad614SDennis Dalessandro if (rcv_intr_count == 0 && rcv_intr_timeout == 0) {
1413f48ad614SDennis Dalessandro pr_err("Invalid mode: both receive interrupt count and available timeout are zero - setting interrupt count to 1\n");
1414f48ad614SDennis Dalessandro rcv_intr_count = 1;
1415f48ad614SDennis Dalessandro }
1416f48ad614SDennis Dalessandro if (rcv_intr_count > 1 && rcv_intr_timeout == 0) {
1417f48ad614SDennis Dalessandro /*
1418f48ad614SDennis Dalessandro * Avoid indefinite packet delivery by requiring a timeout
1419f48ad614SDennis Dalessandro * if count is > 1.
1420f48ad614SDennis Dalessandro */
1421f48ad614SDennis Dalessandro pr_err("Invalid mode: receive interrupt count greater than 1 and available timeout is zero - setting available timeout to 1\n");
1422f48ad614SDennis Dalessandro rcv_intr_timeout = 1;
1423f48ad614SDennis Dalessandro }
1424f48ad614SDennis Dalessandro if (rcv_intr_dynamic && !(rcv_intr_count > 1 && rcv_intr_timeout > 0)) {
1425f48ad614SDennis Dalessandro /*
1426f48ad614SDennis Dalessandro * The dynamic algorithm expects a non-zero timeout
1427f48ad614SDennis Dalessandro * and a count > 1.
1428f48ad614SDennis Dalessandro */
1429f48ad614SDennis Dalessandro pr_err("Invalid mode: dynamic receive interrupt mitigation with invalid count and timeout - turning dynamic off\n");
1430f48ad614SDennis Dalessandro rcv_intr_dynamic = 0;
1431f48ad614SDennis Dalessandro }
1432f48ad614SDennis Dalessandro
1433f48ad614SDennis Dalessandro /* sanitize link CRC options */
1434f48ad614SDennis Dalessandro link_crc_mask &= SUPPORTED_CRCS;
1435f48ad614SDennis Dalessandro
143648a615dcSKaike Wan ret = opfn_init();
143748a615dcSKaike Wan if (ret < 0) {
143848a615dcSKaike Wan pr_err("Failed to allocate opfn_wq");
143948a615dcSKaike Wan goto bail_dev;
144048a615dcSKaike Wan }
144148a615dcSKaike Wan
1442f48ad614SDennis Dalessandro /*
1443f48ad614SDennis Dalessandro * These must be called before the driver is registered with
1444f48ad614SDennis Dalessandro * the PCI subsystem.
1445f48ad614SDennis Dalessandro */
1446f48ad614SDennis Dalessandro hfi1_dbg_init();
1447f48ad614SDennis Dalessandro ret = pci_register_driver(&hfi1_pci_driver);
1448f48ad614SDennis Dalessandro if (ret < 0) {
1449f48ad614SDennis Dalessandro pr_err("Unable to register driver: error %d\n", -ret);
1450f48ad614SDennis Dalessandro goto bail_dev;
1451f48ad614SDennis Dalessandro }
1452f48ad614SDennis Dalessandro goto bail; /* all OK */
1453f48ad614SDennis Dalessandro
1454f48ad614SDennis Dalessandro bail_dev:
1455f48ad614SDennis Dalessandro hfi1_dbg_exit();
1456f48ad614SDennis Dalessandro dev_cleanup();
1457f48ad614SDennis Dalessandro bail:
1458f48ad614SDennis Dalessandro return ret;
1459f48ad614SDennis Dalessandro }
1460f48ad614SDennis Dalessandro
1461f48ad614SDennis Dalessandro module_init(hfi1_mod_init);
1462f48ad614SDennis Dalessandro
1463f48ad614SDennis Dalessandro /*
1464f48ad614SDennis Dalessandro * Do the non-unit driver cleanup, memory free, etc. at unload.
1465f48ad614SDennis Dalessandro */
hfi1_mod_cleanup(void)1466f48ad614SDennis Dalessandro static void __exit hfi1_mod_cleanup(void)
1467f48ad614SDennis Dalessandro {
1468f48ad614SDennis Dalessandro pci_unregister_driver(&hfi1_pci_driver);
146948a615dcSKaike Wan opfn_exit();
14705d18ee67SSebastian Sanchez node_affinity_destroy_all();
1471f48ad614SDennis Dalessandro hfi1_dbg_exit();
1472f48ad614SDennis Dalessandro
147303b92789SMatthew Wilcox WARN_ON(!xa_empty(&hfi1_dev_table));
1474f48ad614SDennis Dalessandro dispose_firmware(); /* asymmetric with obtain_firmware() */
1475f48ad614SDennis Dalessandro dev_cleanup();
1476f48ad614SDennis Dalessandro }
1477f48ad614SDennis Dalessandro
1478f48ad614SDennis Dalessandro module_exit(hfi1_mod_cleanup);
1479f48ad614SDennis Dalessandro
1480f48ad614SDennis Dalessandro /* this can only be called after a successful initialization */
cleanup_device_data(struct hfi1_devdata * dd)1481f48ad614SDennis Dalessandro static void cleanup_device_data(struct hfi1_devdata *dd)
1482f48ad614SDennis Dalessandro {
1483f48ad614SDennis Dalessandro int ctxt;
1484f48ad614SDennis Dalessandro int pidx;
1485f48ad614SDennis Dalessandro
1486f48ad614SDennis Dalessandro /* users can't do anything more with chip */
1487f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1488f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd = &dd->pport[pidx];
1489f48ad614SDennis Dalessandro struct cc_state *cc_state;
1490f48ad614SDennis Dalessandro int i;
1491f48ad614SDennis Dalessandro
1492f48ad614SDennis Dalessandro if (ppd->statusp)
1493f48ad614SDennis Dalessandro *ppd->statusp &= ~HFI1_STATUS_CHIP_PRESENT;
1494f48ad614SDennis Dalessandro
1495f48ad614SDennis Dalessandro for (i = 0; i < OPA_MAX_SLS; i++)
1496f48ad614SDennis Dalessandro hrtimer_cancel(&ppd->cca_timer[i].hrtimer);
1497f48ad614SDennis Dalessandro
1498f48ad614SDennis Dalessandro spin_lock(&ppd->cc_state_lock);
14998adf71faSJianxin Xiong cc_state = get_cc_state_protected(ppd);
1500f48ad614SDennis Dalessandro RCU_INIT_POINTER(ppd->cc_state, NULL);
1501f48ad614SDennis Dalessandro spin_unlock(&ppd->cc_state_lock);
1502f48ad614SDennis Dalessandro
1503f48ad614SDennis Dalessandro if (cc_state)
1504476d95bdSWei Yongjun kfree_rcu(cc_state, rcu);
1505f48ad614SDennis Dalessandro }
1506f48ad614SDennis Dalessandro
1507f48ad614SDennis Dalessandro free_credit_return(dd);
1508f48ad614SDennis Dalessandro
1509d295dbebSMichael J. Ruhl /*
1510d295dbebSMichael J. Ruhl * Free any resources still in use (usually just kernel contexts)
1511d295dbebSMichael J. Ruhl * at unload; we do for ctxtcnt, because that's what we allocate.
1512d295dbebSMichael J. Ruhl */
1513d295dbebSMichael J. Ruhl for (ctxt = 0; dd->rcd && ctxt < dd->num_rcv_contexts; ctxt++) {
1514d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd = dd->rcd[ctxt];
1515f48ad614SDennis Dalessandro
1516f48ad614SDennis Dalessandro if (rcd) {
1517838b6fd2SKaike Wan hfi1_free_ctxt_rcv_groups(rcd);
1518d295dbebSMichael J. Ruhl hfi1_free_ctxt(rcd);
1519f48ad614SDennis Dalessandro }
1520f48ad614SDennis Dalessandro }
1521d295dbebSMichael J. Ruhl
1522d295dbebSMichael J. Ruhl kfree(dd->rcd);
1523d295dbebSMichael J. Ruhl dd->rcd = NULL;
1524d295dbebSMichael J. Ruhl
1525f48ad614SDennis Dalessandro free_pio_map(dd);
1526f48ad614SDennis Dalessandro /* must follow rcv context free - need to remove rcv's hooks */
1527f48ad614SDennis Dalessandro for (ctxt = 0; ctxt < dd->num_send_contexts; ctxt++)
1528f48ad614SDennis Dalessandro sc_free(dd->send_contexts[ctxt].sc);
1529f48ad614SDennis Dalessandro dd->num_send_contexts = 0;
1530f48ad614SDennis Dalessandro kfree(dd->send_contexts);
1531f48ad614SDennis Dalessandro dd->send_contexts = NULL;
1532f48ad614SDennis Dalessandro kfree(dd->hw_to_sw);
1533f48ad614SDennis Dalessandro dd->hw_to_sw = NULL;
1534f48ad614SDennis Dalessandro kfree(dd->boardname);
1535f48ad614SDennis Dalessandro vfree(dd->events);
1536f48ad614SDennis Dalessandro vfree(dd->status);
1537f48ad614SDennis Dalessandro }
1538f48ad614SDennis Dalessandro
1539f48ad614SDennis Dalessandro /*
1540f48ad614SDennis Dalessandro * Clean up on unit shutdown, or error during unit load after
1541f48ad614SDennis Dalessandro * successful initialization.
1542f48ad614SDennis Dalessandro */
postinit_cleanup(struct hfi1_devdata * dd)1543f48ad614SDennis Dalessandro static void postinit_cleanup(struct hfi1_devdata *dd)
1544f48ad614SDennis Dalessandro {
1545f48ad614SDennis Dalessandro hfi1_start_cleanup(dd);
15465d18ee67SSebastian Sanchez hfi1_comp_vectors_clean_up(dd);
15475d18ee67SSebastian Sanchez hfi1_dev_affinity_clean_up(dd);
1548f48ad614SDennis Dalessandro
1549f48ad614SDennis Dalessandro hfi1_pcie_ddcleanup(dd);
1550f48ad614SDennis Dalessandro hfi1_pcie_cleanup(dd->pcidev);
1551f48ad614SDennis Dalessandro
1552f48ad614SDennis Dalessandro cleanup_device_data(dd);
1553f48ad614SDennis Dalessandro
1554f48ad614SDennis Dalessandro hfi1_free_devdata(dd);
1555f48ad614SDennis Dalessandro }
1556f48ad614SDennis Dalessandro
init_one(struct pci_dev * pdev,const struct pci_device_id * ent)1557f48ad614SDennis Dalessandro static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
1558f48ad614SDennis Dalessandro {
1559f48ad614SDennis Dalessandro int ret = 0, j, pidx, initfail;
156083fb4af6SKrzysztof Blaszkowski struct hfi1_devdata *dd;
1561f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd;
1562f48ad614SDennis Dalessandro
1563f48ad614SDennis Dalessandro /* First, lock the non-writable module parameters */
1564f48ad614SDennis Dalessandro HFI1_CAP_LOCK();
1565f48ad614SDennis Dalessandro
15665d6f08afSTadeusz Struk /* Validate dev ids */
15675d6f08afSTadeusz Struk if (!(ent->device == PCI_DEVICE_ID_INTEL0 ||
15685d6f08afSTadeusz Struk ent->device == PCI_DEVICE_ID_INTEL1)) {
156957f97e96SMichael J. Ruhl dev_err(&pdev->dev, "Failing on unknown Intel deviceid 0x%x\n",
15705d6f08afSTadeusz Struk ent->device);
15715d6f08afSTadeusz Struk ret = -ENODEV;
15725d6f08afSTadeusz Struk goto bail;
15735d6f08afSTadeusz Struk }
15745d6f08afSTadeusz Struk
157557f97e96SMichael J. Ruhl /* Allocate the dd so we can get to work */
157657f97e96SMichael J. Ruhl dd = hfi1_alloc_devdata(pdev, NUM_IB_PORTS *
157757f97e96SMichael J. Ruhl sizeof(struct hfi1_pportdata));
157857f97e96SMichael J. Ruhl if (IS_ERR(dd)) {
157957f97e96SMichael J. Ruhl ret = PTR_ERR(dd);
158057f97e96SMichael J. Ruhl goto bail;
158157f97e96SMichael J. Ruhl }
158257f97e96SMichael J. Ruhl
1583f48ad614SDennis Dalessandro /* Validate some global module parameters */
1584de730f71SMike Marciniszyn ret = hfi1_validate_rcvhdrcnt(dd, rcvhdrcnt);
158511501ab9SKrzysztof Blaszkowski if (ret)
1586f48ad614SDennis Dalessandro goto bail;
158711501ab9SKrzysztof Blaszkowski
1588f48ad614SDennis Dalessandro /* use the encoding function as a sanitization check */
1589f48ad614SDennis Dalessandro if (!encode_rcv_header_entry_size(hfi1_hdrq_entsize)) {
159057f97e96SMichael J. Ruhl dd_dev_err(dd, "Invalid HdrQ Entry size %u\n",
1591f48ad614SDennis Dalessandro hfi1_hdrq_entsize);
1592f48ad614SDennis Dalessandro ret = -EINVAL;
1593f48ad614SDennis Dalessandro goto bail;
1594f48ad614SDennis Dalessandro }
1595f48ad614SDennis Dalessandro
1596f48ad614SDennis Dalessandro /* The receive eager buffer size must be set before the receive
1597f48ad614SDennis Dalessandro * contexts are created.
1598f48ad614SDennis Dalessandro *
1599f48ad614SDennis Dalessandro * Set the eager buffer size. Validate that it falls in a range
1600f48ad614SDennis Dalessandro * allowed by the hardware - all powers of 2 between the min and
1601f48ad614SDennis Dalessandro * max. The maximum valid MTU is within the eager buffer range
1602f48ad614SDennis Dalessandro * so we do not need to cap the max_mtu by an eager buffer size
1603f48ad614SDennis Dalessandro * setting.
1604f48ad614SDennis Dalessandro */
1605f48ad614SDennis Dalessandro if (eager_buffer_size) {
1606f48ad614SDennis Dalessandro if (!is_power_of_2(eager_buffer_size))
1607f48ad614SDennis Dalessandro eager_buffer_size =
1608f48ad614SDennis Dalessandro roundup_pow_of_two(eager_buffer_size);
1609f48ad614SDennis Dalessandro eager_buffer_size =
1610f48ad614SDennis Dalessandro clamp_val(eager_buffer_size,
1611f48ad614SDennis Dalessandro MIN_EAGER_BUFFER * 8,
1612f48ad614SDennis Dalessandro MAX_EAGER_BUFFER_TOTAL);
161357f97e96SMichael J. Ruhl dd_dev_info(dd, "Eager buffer size %u\n",
1614f48ad614SDennis Dalessandro eager_buffer_size);
1615f48ad614SDennis Dalessandro } else {
161657f97e96SMichael J. Ruhl dd_dev_err(dd, "Invalid Eager buffer size of 0\n");
1617f48ad614SDennis Dalessandro ret = -EINVAL;
1618f48ad614SDennis Dalessandro goto bail;
1619f48ad614SDennis Dalessandro }
1620f48ad614SDennis Dalessandro
1621f48ad614SDennis Dalessandro /* restrict value of hfi1_rcvarr_split */
1622f48ad614SDennis Dalessandro hfi1_rcvarr_split = clamp_val(hfi1_rcvarr_split, 0, 100);
1623f48ad614SDennis Dalessandro
162457f97e96SMichael J. Ruhl ret = hfi1_pcie_init(dd);
1625f48ad614SDennis Dalessandro if (ret)
1626f48ad614SDennis Dalessandro goto bail;
1627f48ad614SDennis Dalessandro
162883fb4af6SKrzysztof Blaszkowski /*
162983fb4af6SKrzysztof Blaszkowski * Do device-specific initialization, function table setup, dd
163083fb4af6SKrzysztof Blaszkowski * allocation, etc.
163183fb4af6SKrzysztof Blaszkowski */
163257f97e96SMichael J. Ruhl ret = hfi1_init_dd(dd);
163357f97e96SMichael J. Ruhl if (ret)
1634f48ad614SDennis Dalessandro goto clean_bail; /* error already printed */
1635f48ad614SDennis Dalessandro
1636f48ad614SDennis Dalessandro ret = create_workqueues(dd);
1637f48ad614SDennis Dalessandro if (ret)
1638f48ad614SDennis Dalessandro goto clean_bail;
1639f48ad614SDennis Dalessandro
1640f48ad614SDennis Dalessandro /* do the generic initialization */
1641f48ad614SDennis Dalessandro initfail = hfi1_init(dd, 0);
1642f48ad614SDennis Dalessandro
1643f48ad614SDennis Dalessandro ret = hfi1_register_ib_device(dd);
1644f48ad614SDennis Dalessandro
1645f48ad614SDennis Dalessandro /*
1646f48ad614SDennis Dalessandro * Now ready for use. this should be cleared whenever we
1647f48ad614SDennis Dalessandro * detect a reset, or initiate one. If earlier failure,
1648f48ad614SDennis Dalessandro * we still create devices, so diags, etc. can be used
1649f48ad614SDennis Dalessandro * to determine cause of problem.
1650f48ad614SDennis Dalessandro */
1651f48ad614SDennis Dalessandro if (!initfail && !ret) {
1652f48ad614SDennis Dalessandro dd->flags |= HFI1_INITTED;
1653f48ad614SDennis Dalessandro /* create debufs files after init and ib register */
1654f48ad614SDennis Dalessandro hfi1_dbg_ibdev_init(&dd->verbs_dev);
1655f48ad614SDennis Dalessandro }
1656f48ad614SDennis Dalessandro
1657f48ad614SDennis Dalessandro j = hfi1_device_create(dd);
1658f48ad614SDennis Dalessandro if (j)
1659f48ad614SDennis Dalessandro dd_dev_err(dd, "Failed to create /dev devices: %d\n", -j);
1660f48ad614SDennis Dalessandro
1661f48ad614SDennis Dalessandro if (initfail || ret) {
16626eb4eb10SMichael J. Ruhl msix_clean_up_interrupts(dd);
1663f48ad614SDennis Dalessandro stop_timers(dd);
1664f48ad614SDennis Dalessandro flush_workqueue(ib_wq);
1665f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1666f48ad614SDennis Dalessandro hfi1_quiet_serdes(dd->pport + pidx);
1667f48ad614SDennis Dalessandro ppd = dd->pport + pidx;
1668f48ad614SDennis Dalessandro if (ppd->hfi1_wq) {
1669f48ad614SDennis Dalessandro destroy_workqueue(ppd->hfi1_wq);
1670f48ad614SDennis Dalessandro ppd->hfi1_wq = NULL;
1671f48ad614SDennis Dalessandro }
167271d47008SSebastian Sanchez if (ppd->link_wq) {
167371d47008SSebastian Sanchez destroy_workqueue(ppd->link_wq);
167471d47008SSebastian Sanchez ppd->link_wq = NULL;
167571d47008SSebastian Sanchez }
1676f48ad614SDennis Dalessandro }
1677f48ad614SDennis Dalessandro if (!j)
1678f48ad614SDennis Dalessandro hfi1_device_remove(dd);
1679f48ad614SDennis Dalessandro if (!ret)
1680f48ad614SDennis Dalessandro hfi1_unregister_ib_device(dd);
1681f48ad614SDennis Dalessandro postinit_cleanup(dd);
1682f48ad614SDennis Dalessandro if (initfail)
1683f48ad614SDennis Dalessandro ret = initfail;
1684f48ad614SDennis Dalessandro goto bail; /* everything already cleaned */
1685f48ad614SDennis Dalessandro }
1686f48ad614SDennis Dalessandro
1687f48ad614SDennis Dalessandro sdma_start(dd);
1688f48ad614SDennis Dalessandro
1689f48ad614SDennis Dalessandro return 0;
1690f48ad614SDennis Dalessandro
1691f48ad614SDennis Dalessandro clean_bail:
1692f48ad614SDennis Dalessandro hfi1_pcie_cleanup(pdev);
1693f48ad614SDennis Dalessandro bail:
1694f48ad614SDennis Dalessandro return ret;
1695f48ad614SDennis Dalessandro }
1696f48ad614SDennis Dalessandro
wait_for_clients(struct hfi1_devdata * dd)1697acd7c8feSTadeusz Struk static void wait_for_clients(struct hfi1_devdata *dd)
1698acd7c8feSTadeusz Struk {
1699acd7c8feSTadeusz Struk /*
1700acd7c8feSTadeusz Struk * Remove the device init value and complete the device if there is
1701acd7c8feSTadeusz Struk * no clients or wait for active clients to finish.
1702acd7c8feSTadeusz Struk */
1703a0293eb2SXiyu Yang if (refcount_dec_and_test(&dd->user_refcount))
1704acd7c8feSTadeusz Struk complete(&dd->user_comp);
1705acd7c8feSTadeusz Struk
1706acd7c8feSTadeusz Struk wait_for_completion(&dd->user_comp);
1707acd7c8feSTadeusz Struk }
1708acd7c8feSTadeusz Struk
remove_one(struct pci_dev * pdev)1709f48ad614SDennis Dalessandro static void remove_one(struct pci_dev *pdev)
1710f48ad614SDennis Dalessandro {
1711f48ad614SDennis Dalessandro struct hfi1_devdata *dd = pci_get_drvdata(pdev);
1712f48ad614SDennis Dalessandro
1713f48ad614SDennis Dalessandro /* close debugfs files before ib unregister */
1714f48ad614SDennis Dalessandro hfi1_dbg_ibdev_exit(&dd->verbs_dev);
1715acd7c8feSTadeusz Struk
1716acd7c8feSTadeusz Struk /* remove the /dev hfi1 interface */
1717acd7c8feSTadeusz Struk hfi1_device_remove(dd);
1718acd7c8feSTadeusz Struk
1719acd7c8feSTadeusz Struk /* wait for existing user space clients to finish */
1720acd7c8feSTadeusz Struk wait_for_clients(dd);
1721acd7c8feSTadeusz Struk
1722f48ad614SDennis Dalessandro /* unregister from IB core */
1723f48ad614SDennis Dalessandro hfi1_unregister_ib_device(dd);
1724f48ad614SDennis Dalessandro
17254730f4a6SGrzegorz Andrejczuk /* free netdev data */
1726780278c2SMike Marciniszyn hfi1_free_rx(dd);
1727d4829ea6SVishwanathapura, Niranjana
1728f48ad614SDennis Dalessandro /*
1729f48ad614SDennis Dalessandro * Disable the IB link, disable interrupts on the device,
1730f48ad614SDennis Dalessandro * clear dma engines, etc.
1731f48ad614SDennis Dalessandro */
1732f48ad614SDennis Dalessandro shutdown_device(dd);
173328b70cd9SKaike Wan destroy_workqueues(dd);
1734f48ad614SDennis Dalessandro
1735f48ad614SDennis Dalessandro stop_timers(dd);
1736f48ad614SDennis Dalessandro
1737f48ad614SDennis Dalessandro /* wait until all of our (qsfp) queue_work() calls complete */
1738f48ad614SDennis Dalessandro flush_workqueue(ib_wq);
1739f48ad614SDennis Dalessandro
1740f48ad614SDennis Dalessandro postinit_cleanup(dd);
1741f48ad614SDennis Dalessandro }
1742f48ad614SDennis Dalessandro
shutdown_one(struct pci_dev * pdev)17438d3e7113SAlex Estrin static void shutdown_one(struct pci_dev *pdev)
17448d3e7113SAlex Estrin {
17458d3e7113SAlex Estrin struct hfi1_devdata *dd = pci_get_drvdata(pdev);
17468d3e7113SAlex Estrin
17478d3e7113SAlex Estrin shutdown_device(dd);
17488d3e7113SAlex Estrin }
17498d3e7113SAlex Estrin
1750f48ad614SDennis Dalessandro /**
1751f48ad614SDennis Dalessandro * hfi1_create_rcvhdrq - create a receive header queue
1752f48ad614SDennis Dalessandro * @dd: the hfi1_ib device
1753f48ad614SDennis Dalessandro * @rcd: the context data
1754f48ad614SDennis Dalessandro *
1755f48ad614SDennis Dalessandro * This must be contiguous memory (from an i/o perspective), and must be
1756f48ad614SDennis Dalessandro * DMA'able (which means for some systems, it will go through an IOMMU,
1757f48ad614SDennis Dalessandro * or be forced into a low address range).
1758f48ad614SDennis Dalessandro */
hfi1_create_rcvhdrq(struct hfi1_devdata * dd,struct hfi1_ctxtdata * rcd)1759f48ad614SDennis Dalessandro int hfi1_create_rcvhdrq(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd)
1760f48ad614SDennis Dalessandro {
1761f48ad614SDennis Dalessandro unsigned amt;
1762f48ad614SDennis Dalessandro
1763f48ad614SDennis Dalessandro if (!rcd->rcvhdrq) {
1764b2578431SMike Marciniszyn amt = rcvhdrq_size(rcd);
1765f48ad614SDennis Dalessandro
1766750afb08SLuis Chamberlain rcd->rcvhdrq = dma_alloc_coherent(&dd->pcidev->dev, amt,
1767750afb08SLuis Chamberlain &rcd->rcvhdrq_dma,
176882c310c3SChristoph Hellwig GFP_KERNEL);
1769f48ad614SDennis Dalessandro
1770f48ad614SDennis Dalessandro if (!rcd->rcvhdrq) {
1771f48ad614SDennis Dalessandro dd_dev_err(dd,
1772f48ad614SDennis Dalessandro "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",
1773f48ad614SDennis Dalessandro amt, rcd->ctxt);
1774f48ad614SDennis Dalessandro goto bail;
1775f48ad614SDennis Dalessandro }
1776f48ad614SDennis Dalessandro
17771bc0299dSMike Marciniszyn if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
17781bc0299dSMike Marciniszyn HFI1_CAP_UGET_MASK(rcd->flags, DMA_RTAIL)) {
1779750afb08SLuis Chamberlain rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(&dd->pcidev->dev,
1780750afb08SLuis Chamberlain PAGE_SIZE,
1781750afb08SLuis Chamberlain &rcd->rcvhdrqtailaddr_dma,
178282c310c3SChristoph Hellwig GFP_KERNEL);
1783f48ad614SDennis Dalessandro if (!rcd->rcvhdrtail_kvaddr)
1784f48ad614SDennis Dalessandro goto bail_free;
1785f48ad614SDennis Dalessandro }
1786f48ad614SDennis Dalessandro }
1787f48ad614SDennis Dalessandro
1788de730f71SMike Marciniszyn set_hdrq_regs(rcd->dd, rcd->ctxt, rcd->rcvhdrqentsize,
1789de730f71SMike Marciniszyn rcd->rcvhdrq_cnt);
1790f48ad614SDennis Dalessandro
1791f48ad614SDennis Dalessandro return 0;
1792f48ad614SDennis Dalessandro
1793f48ad614SDennis Dalessandro bail_free:
1794f48ad614SDennis Dalessandro dd_dev_err(dd,
1795f48ad614SDennis Dalessandro "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",
1796f48ad614SDennis Dalessandro rcd->ctxt);
1797f48ad614SDennis Dalessandro dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,
179860368186STymoteusz Kielan rcd->rcvhdrq_dma);
1799f48ad614SDennis Dalessandro rcd->rcvhdrq = NULL;
1800f48ad614SDennis Dalessandro bail:
1801f48ad614SDennis Dalessandro return -ENOMEM;
1802f48ad614SDennis Dalessandro }
1803f48ad614SDennis Dalessandro
1804f48ad614SDennis Dalessandro /**
1805ae360f41SLeon Romanovsky * hfi1_setup_eagerbufs - llocate eager buffers, both kernel and user
1806ae360f41SLeon Romanovsky * contexts.
1807f48ad614SDennis Dalessandro * @rcd: the context we are setting up.
1808f48ad614SDennis Dalessandro *
1809f48ad614SDennis Dalessandro * Allocate the eager TID buffers and program them into hip.
1810f48ad614SDennis Dalessandro * They are no longer completely contiguous, we do multiple allocation
1811f48ad614SDennis Dalessandro * calls. Otherwise we get the OOM code involved, by asking for too
1812f48ad614SDennis Dalessandro * much per call, with disastrous results on some kernels.
1813f48ad614SDennis Dalessandro */
hfi1_setup_eagerbufs(struct hfi1_ctxtdata * rcd)1814f48ad614SDennis Dalessandro int hfi1_setup_eagerbufs(struct hfi1_ctxtdata *rcd)
1815f48ad614SDennis Dalessandro {
1816f48ad614SDennis Dalessandro struct hfi1_devdata *dd = rcd->dd;
1817071e4fecSMike Marciniszyn u32 max_entries, egrtop, alloced_bytes = 0;
1818071e4fecSMike Marciniszyn u16 order, idx = 0;
1819f48ad614SDennis Dalessandro int ret = 0;
1820f48ad614SDennis Dalessandro u16 round_mtu = roundup_pow_of_two(hfi1_max_mtu);
1821f48ad614SDennis Dalessandro
1822f48ad614SDennis Dalessandro /*
1823f48ad614SDennis Dalessandro * The minimum size of the eager buffers is a groups of MTU-sized
1824f48ad614SDennis Dalessandro * buffers.
1825f48ad614SDennis Dalessandro * The global eager_buffer_size parameter is checked against the
1826f48ad614SDennis Dalessandro * theoretical lower limit of the value. Here, we check against the
1827f48ad614SDennis Dalessandro * MTU.
1828f48ad614SDennis Dalessandro */
1829f48ad614SDennis Dalessandro if (rcd->egrbufs.size < (round_mtu * dd->rcv_entries.group_size))
1830f48ad614SDennis Dalessandro rcd->egrbufs.size = round_mtu * dd->rcv_entries.group_size;
1831f48ad614SDennis Dalessandro /*
1832f48ad614SDennis Dalessandro * If using one-pkt-per-egr-buffer, lower the eager buffer
1833f48ad614SDennis Dalessandro * size to the max MTU (page-aligned).
1834f48ad614SDennis Dalessandro */
1835f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR))
1836f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size = round_mtu;
1837f48ad614SDennis Dalessandro
1838f48ad614SDennis Dalessandro /*
1839f48ad614SDennis Dalessandro * Eager buffers sizes of 1MB or less require smaller TID sizes
1840f48ad614SDennis Dalessandro * to satisfy the "multiple of 8 RcvArray entries" requirement.
1841f48ad614SDennis Dalessandro */
1842f48ad614SDennis Dalessandro if (rcd->egrbufs.size <= (1 << 20))
1843f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size = max((unsigned long)round_mtu,
1844f48ad614SDennis Dalessandro rounddown_pow_of_two(rcd->egrbufs.size / 8));
1845f48ad614SDennis Dalessandro
1846f48ad614SDennis Dalessandro while (alloced_bytes < rcd->egrbufs.size &&
1847f48ad614SDennis Dalessandro rcd->egrbufs.alloced < rcd->egrbufs.count) {
1848f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].addr =
1849750afb08SLuis Chamberlain dma_alloc_coherent(&dd->pcidev->dev,
1850f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size,
185160368186STymoteusz Kielan &rcd->egrbufs.buffers[idx].dma,
185282c310c3SChristoph Hellwig GFP_KERNEL);
1853f48ad614SDennis Dalessandro if (rcd->egrbufs.buffers[idx].addr) {
1854f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].len =
1855f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size;
1856f48ad614SDennis Dalessandro rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].addr =
1857f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].addr;
185860368186STymoteusz Kielan rcd->egrbufs.rcvtids[rcd->egrbufs.alloced].dma =
185960368186STymoteusz Kielan rcd->egrbufs.buffers[idx].dma;
1860f48ad614SDennis Dalessandro rcd->egrbufs.alloced++;
1861f48ad614SDennis Dalessandro alloced_bytes += rcd->egrbufs.rcvtid_size;
1862f48ad614SDennis Dalessandro idx++;
1863f48ad614SDennis Dalessandro } else {
1864f48ad614SDennis Dalessandro u32 new_size, i, j;
1865f48ad614SDennis Dalessandro u64 offset = 0;
1866f48ad614SDennis Dalessandro
1867f48ad614SDennis Dalessandro /*
1868f48ad614SDennis Dalessandro * Fail the eager buffer allocation if:
1869f48ad614SDennis Dalessandro * - we are already using the lowest acceptable size
1870f48ad614SDennis Dalessandro * - we are using one-pkt-per-egr-buffer (this implies
1871f48ad614SDennis Dalessandro * that we are accepting only one size)
1872f48ad614SDennis Dalessandro */
1873f48ad614SDennis Dalessandro if (rcd->egrbufs.rcvtid_size == round_mtu ||
1874f48ad614SDennis Dalessandro !HFI1_CAP_KGET_MASK(rcd->flags, MULTI_PKT_EGR)) {
1875f48ad614SDennis Dalessandro dd_dev_err(dd, "ctxt%u: Failed to allocate eager buffers\n",
1876f48ad614SDennis Dalessandro rcd->ctxt);
187794679061SMichael J. Ruhl ret = -ENOMEM;
1878f48ad614SDennis Dalessandro goto bail_rcvegrbuf_phys;
1879f48ad614SDennis Dalessandro }
1880f48ad614SDennis Dalessandro
1881f48ad614SDennis Dalessandro new_size = rcd->egrbufs.rcvtid_size / 2;
1882f48ad614SDennis Dalessandro
1883f48ad614SDennis Dalessandro /*
1884f48ad614SDennis Dalessandro * If the first attempt to allocate memory failed, don't
1885f48ad614SDennis Dalessandro * fail everything but continue with the next lower
1886f48ad614SDennis Dalessandro * size.
1887f48ad614SDennis Dalessandro */
1888f48ad614SDennis Dalessandro if (idx == 0) {
1889f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size = new_size;
1890f48ad614SDennis Dalessandro continue;
1891f48ad614SDennis Dalessandro }
1892f48ad614SDennis Dalessandro
1893f48ad614SDennis Dalessandro /*
1894f48ad614SDennis Dalessandro * Re-partition already allocated buffers to a smaller
1895f48ad614SDennis Dalessandro * size.
1896f48ad614SDennis Dalessandro */
1897f48ad614SDennis Dalessandro rcd->egrbufs.alloced = 0;
1898f48ad614SDennis Dalessandro for (i = 0, j = 0, offset = 0; j < idx; i++) {
1899f48ad614SDennis Dalessandro if (i >= rcd->egrbufs.count)
1900f48ad614SDennis Dalessandro break;
190160368186STymoteusz Kielan rcd->egrbufs.rcvtids[i].dma =
190260368186STymoteusz Kielan rcd->egrbufs.buffers[j].dma + offset;
1903f48ad614SDennis Dalessandro rcd->egrbufs.rcvtids[i].addr =
1904f48ad614SDennis Dalessandro rcd->egrbufs.buffers[j].addr + offset;
1905f48ad614SDennis Dalessandro rcd->egrbufs.alloced++;
190660368186STymoteusz Kielan if ((rcd->egrbufs.buffers[j].dma + offset +
1907f48ad614SDennis Dalessandro new_size) ==
190860368186STymoteusz Kielan (rcd->egrbufs.buffers[j].dma +
1909f48ad614SDennis Dalessandro rcd->egrbufs.buffers[j].len)) {
1910f48ad614SDennis Dalessandro j++;
1911f48ad614SDennis Dalessandro offset = 0;
1912f48ad614SDennis Dalessandro } else {
1913f48ad614SDennis Dalessandro offset += new_size;
1914f48ad614SDennis Dalessandro }
1915f48ad614SDennis Dalessandro }
1916f48ad614SDennis Dalessandro rcd->egrbufs.rcvtid_size = new_size;
1917f48ad614SDennis Dalessandro }
1918f48ad614SDennis Dalessandro }
1919f48ad614SDennis Dalessandro rcd->egrbufs.numbufs = idx;
1920f48ad614SDennis Dalessandro rcd->egrbufs.size = alloced_bytes;
1921f48ad614SDennis Dalessandro
1922f48ad614SDennis Dalessandro hfi1_cdbg(PROC,
1923*d2590edcSDean Luick "ctxt%u: Alloced %u rcv tid entries @ %uKB, total %uKB",
192423002d5bSGrzegorz Heldt rcd->ctxt, rcd->egrbufs.alloced,
192523002d5bSGrzegorz Heldt rcd->egrbufs.rcvtid_size / 1024, rcd->egrbufs.size / 1024);
1926f48ad614SDennis Dalessandro
1927f48ad614SDennis Dalessandro /*
1928f48ad614SDennis Dalessandro * Set the contexts rcv array head update threshold to the closest
1929f48ad614SDennis Dalessandro * power of 2 (so we can use a mask instead of modulo) below half
1930f48ad614SDennis Dalessandro * the allocated entries.
1931f48ad614SDennis Dalessandro */
1932f48ad614SDennis Dalessandro rcd->egrbufs.threshold =
1933f48ad614SDennis Dalessandro rounddown_pow_of_two(rcd->egrbufs.alloced / 2);
1934f48ad614SDennis Dalessandro /*
1935f48ad614SDennis Dalessandro * Compute the expected RcvArray entry base. This is done after
1936f48ad614SDennis Dalessandro * allocating the eager buffers in order to maximize the
1937f48ad614SDennis Dalessandro * expected RcvArray entries for the context.
1938f48ad614SDennis Dalessandro */
1939f48ad614SDennis Dalessandro max_entries = rcd->rcv_array_groups * dd->rcv_entries.group_size;
1940f48ad614SDennis Dalessandro egrtop = roundup(rcd->egrbufs.alloced, dd->rcv_entries.group_size);
1941f48ad614SDennis Dalessandro rcd->expected_count = max_entries - egrtop;
1942f48ad614SDennis Dalessandro if (rcd->expected_count > MAX_TID_PAIR_ENTRIES * 2)
1943f48ad614SDennis Dalessandro rcd->expected_count = MAX_TID_PAIR_ENTRIES * 2;
1944f48ad614SDennis Dalessandro
1945f48ad614SDennis Dalessandro rcd->expected_base = rcd->eager_base + egrtop;
1946*d2590edcSDean Luick hfi1_cdbg(PROC, "ctxt%u: eager:%u, exp:%u, egrbase:%u, expbase:%u",
1947f48ad614SDennis Dalessandro rcd->ctxt, rcd->egrbufs.alloced, rcd->expected_count,
1948f48ad614SDennis Dalessandro rcd->eager_base, rcd->expected_base);
1949f48ad614SDennis Dalessandro
1950f48ad614SDennis Dalessandro if (!hfi1_rcvbuf_validate(rcd->egrbufs.rcvtid_size, PT_EAGER, &order)) {
1951f48ad614SDennis Dalessandro hfi1_cdbg(PROC,
1952*d2590edcSDean Luick "ctxt%u: current Eager buffer size is invalid %u",
1953f48ad614SDennis Dalessandro rcd->ctxt, rcd->egrbufs.rcvtid_size);
1954f48ad614SDennis Dalessandro ret = -EINVAL;
195562239fc6SMichael J. Ruhl goto bail_rcvegrbuf_phys;
1956f48ad614SDennis Dalessandro }
1957f48ad614SDennis Dalessandro
1958f48ad614SDennis Dalessandro for (idx = 0; idx < rcd->egrbufs.alloced; idx++) {
1959f48ad614SDennis Dalessandro hfi1_put_tid(dd, rcd->eager_base + idx, PT_EAGER,
196060368186STymoteusz Kielan rcd->egrbufs.rcvtids[idx].dma, order);
1961f48ad614SDennis Dalessandro cond_resched();
1962f48ad614SDennis Dalessandro }
196362239fc6SMichael J. Ruhl
196462239fc6SMichael J. Ruhl return 0;
1965f48ad614SDennis Dalessandro
1966f48ad614SDennis Dalessandro bail_rcvegrbuf_phys:
1967f48ad614SDennis Dalessandro for (idx = 0; idx < rcd->egrbufs.alloced &&
1968f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].addr;
1969f48ad614SDennis Dalessandro idx++) {
1970f48ad614SDennis Dalessandro dma_free_coherent(&dd->pcidev->dev,
1971f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].len,
1972f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].addr,
197360368186STymoteusz Kielan rcd->egrbufs.buffers[idx].dma);
1974f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].addr = NULL;
197560368186STymoteusz Kielan rcd->egrbufs.buffers[idx].dma = 0;
1976f48ad614SDennis Dalessandro rcd->egrbufs.buffers[idx].len = 0;
1977f48ad614SDennis Dalessandro }
197862239fc6SMichael J. Ruhl
1979f48ad614SDennis Dalessandro return ret;
1980f48ad614SDennis Dalessandro }
1981