1f48ad614SDennis Dalessandro /*
2f48ad614SDennis Dalessandro  * Copyright(c) 2015, 2016 Intel Corporation.
3f48ad614SDennis Dalessandro  *
4f48ad614SDennis Dalessandro  * This file is provided under a dual BSD/GPLv2 license.  When using or
5f48ad614SDennis Dalessandro  * redistributing this file, you may do so under either license.
6f48ad614SDennis Dalessandro  *
7f48ad614SDennis Dalessandro  * GPL LICENSE SUMMARY
8f48ad614SDennis Dalessandro  *
9f48ad614SDennis Dalessandro  * This program is free software; you can redistribute it and/or modify
10f48ad614SDennis Dalessandro  * it under the terms of version 2 of the GNU General Public License as
11f48ad614SDennis Dalessandro  * published by the Free Software Foundation.
12f48ad614SDennis Dalessandro  *
13f48ad614SDennis Dalessandro  * This program is distributed in the hope that it will be useful, but
14f48ad614SDennis Dalessandro  * WITHOUT ANY WARRANTY; without even the implied warranty of
15f48ad614SDennis Dalessandro  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16f48ad614SDennis Dalessandro  * General Public License for more details.
17f48ad614SDennis Dalessandro  *
18f48ad614SDennis Dalessandro  * BSD LICENSE
19f48ad614SDennis Dalessandro  *
20f48ad614SDennis Dalessandro  * Redistribution and use in source and binary forms, with or without
21f48ad614SDennis Dalessandro  * modification, are permitted provided that the following conditions
22f48ad614SDennis Dalessandro  * are met:
23f48ad614SDennis Dalessandro  *
24f48ad614SDennis Dalessandro  *  - Redistributions of source code must retain the above copyright
25f48ad614SDennis Dalessandro  *    notice, this list of conditions and the following disclaimer.
26f48ad614SDennis Dalessandro  *  - Redistributions in binary form must reproduce the above copyright
27f48ad614SDennis Dalessandro  *    notice, this list of conditions and the following disclaimer in
28f48ad614SDennis Dalessandro  *    the documentation and/or other materials provided with the
29f48ad614SDennis Dalessandro  *    distribution.
30f48ad614SDennis Dalessandro  *  - Neither the name of Intel Corporation nor the names of its
31f48ad614SDennis Dalessandro  *    contributors may be used to endorse or promote products derived
32f48ad614SDennis Dalessandro  *    from this software without specific prior written permission.
33f48ad614SDennis Dalessandro  *
34f48ad614SDennis Dalessandro  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35f48ad614SDennis Dalessandro  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36f48ad614SDennis Dalessandro  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37f48ad614SDennis Dalessandro  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38f48ad614SDennis Dalessandro  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39f48ad614SDennis Dalessandro  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40f48ad614SDennis Dalessandro  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41f48ad614SDennis Dalessandro  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42f48ad614SDennis Dalessandro  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43f48ad614SDennis Dalessandro  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44f48ad614SDennis Dalessandro  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45f48ad614SDennis Dalessandro  *
46f48ad614SDennis Dalessandro  */
47f48ad614SDennis Dalessandro 
48f48ad614SDennis Dalessandro #include <linux/spinlock.h>
49f48ad614SDennis Dalessandro #include <linux/pci.h>
50f48ad614SDennis Dalessandro #include <linux/io.h>
51f48ad614SDennis Dalessandro #include <linux/delay.h>
52f48ad614SDennis Dalessandro #include <linux/netdevice.h>
53f48ad614SDennis Dalessandro #include <linux/vmalloc.h>
54f48ad614SDennis Dalessandro #include <linux/module.h>
55f48ad614SDennis Dalessandro #include <linux/prefetch.h>
56f48ad614SDennis Dalessandro #include <rdma/ib_verbs.h>
57f48ad614SDennis Dalessandro 
58f48ad614SDennis Dalessandro #include "hfi.h"
59f48ad614SDennis Dalessandro #include "trace.h"
60f48ad614SDennis Dalessandro #include "qp.h"
61f48ad614SDennis Dalessandro #include "sdma.h"
620181ce31SDon Hiatt #include "debugfs.h"
63f48ad614SDennis Dalessandro 
64f48ad614SDennis Dalessandro #undef pr_fmt
65f48ad614SDennis Dalessandro #define pr_fmt(fmt) DRIVER_NAME ": " fmt
66f48ad614SDennis Dalessandro 
67f48ad614SDennis Dalessandro /*
68f48ad614SDennis Dalessandro  * The size has to be longer than this string, so we can append
69f48ad614SDennis Dalessandro  * board/chip information to it in the initialization code.
70f48ad614SDennis Dalessandro  */
71f48ad614SDennis Dalessandro const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
72f48ad614SDennis Dalessandro 
73f48ad614SDennis Dalessandro DEFINE_SPINLOCK(hfi1_devs_lock);
74f48ad614SDennis Dalessandro LIST_HEAD(hfi1_dev_list);
75f48ad614SDennis Dalessandro DEFINE_MUTEX(hfi1_mutex);	/* general driver use */
76f48ad614SDennis Dalessandro 
77f48ad614SDennis Dalessandro unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
78f48ad614SDennis Dalessandro module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
79f48ad614SDennis Dalessandro MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
80f48ad614SDennis Dalessandro 		 HFI1_DEFAULT_MAX_MTU));
81f48ad614SDennis Dalessandro 
82f48ad614SDennis Dalessandro unsigned int hfi1_cu = 1;
83f48ad614SDennis Dalessandro module_param_named(cu, hfi1_cu, uint, S_IRUGO);
84f48ad614SDennis Dalessandro MODULE_PARM_DESC(cu, "Credit return units");
85f48ad614SDennis Dalessandro 
86f48ad614SDennis Dalessandro unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
87f48ad614SDennis Dalessandro static int hfi1_caps_set(const char *, const struct kernel_param *);
88f48ad614SDennis Dalessandro static int hfi1_caps_get(char *, const struct kernel_param *);
89f48ad614SDennis Dalessandro static const struct kernel_param_ops cap_ops = {
90f48ad614SDennis Dalessandro 	.set = hfi1_caps_set,
91f48ad614SDennis Dalessandro 	.get = hfi1_caps_get
92f48ad614SDennis Dalessandro };
93f48ad614SDennis Dalessandro module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
94f48ad614SDennis Dalessandro MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
95f48ad614SDennis Dalessandro 
96f48ad614SDennis Dalessandro MODULE_LICENSE("Dual BSD/GPL");
97f48ad614SDennis Dalessandro MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
98f48ad614SDennis Dalessandro MODULE_VERSION(HFI1_DRIVER_VERSION);
99f48ad614SDennis Dalessandro 
100f48ad614SDennis Dalessandro /*
101f48ad614SDennis Dalessandro  * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
102f48ad614SDennis Dalessandro  */
103f48ad614SDennis Dalessandro #define MAX_PKT_RECV 64
104a82a7fcdSMike Marciniszyn /*
105a82a7fcdSMike Marciniszyn  * MAX_PKT_THREAD_RCV is the max # of packets processed before
106a82a7fcdSMike Marciniszyn  * the qp_wait_list queue is flushed.
107a82a7fcdSMike Marciniszyn  */
108a82a7fcdSMike Marciniszyn #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4)
109f48ad614SDennis Dalessandro #define EGR_HEAD_UPDATE_THRESHOLD 16
110f48ad614SDennis Dalessandro 
111f48ad614SDennis Dalessandro struct hfi1_ib_stats hfi1_stats;
112f48ad614SDennis Dalessandro 
113f48ad614SDennis Dalessandro static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
114f48ad614SDennis Dalessandro {
115f48ad614SDennis Dalessandro 	int ret = 0;
116f48ad614SDennis Dalessandro 	unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
117f48ad614SDennis Dalessandro 		cap_mask = *cap_mask_ptr, value, diff,
118f48ad614SDennis Dalessandro 		write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
119f48ad614SDennis Dalessandro 			      HFI1_CAP_WRITABLE_MASK);
120f48ad614SDennis Dalessandro 
121f48ad614SDennis Dalessandro 	ret = kstrtoul(val, 0, &value);
122f48ad614SDennis Dalessandro 	if (ret) {
123f48ad614SDennis Dalessandro 		pr_warn("Invalid module parameter value for 'cap_mask'\n");
124f48ad614SDennis Dalessandro 		goto done;
125f48ad614SDennis Dalessandro 	}
126f48ad614SDennis Dalessandro 	/* Get the changed bits (except the locked bit) */
127f48ad614SDennis Dalessandro 	diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);
128f48ad614SDennis Dalessandro 
129f48ad614SDennis Dalessandro 	/* Remove any bits that are not allowed to change after driver load */
130f48ad614SDennis Dalessandro 	if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
131f48ad614SDennis Dalessandro 		pr_warn("Ignoring non-writable capability bits %#lx\n",
132f48ad614SDennis Dalessandro 			diff & ~write_mask);
133f48ad614SDennis Dalessandro 		diff &= write_mask;
134f48ad614SDennis Dalessandro 	}
135f48ad614SDennis Dalessandro 
136f48ad614SDennis Dalessandro 	/* Mask off any reserved bits */
137f48ad614SDennis Dalessandro 	diff &= ~HFI1_CAP_RESERVED_MASK;
138f48ad614SDennis Dalessandro 	/* Clear any previously set and changing bits */
139f48ad614SDennis Dalessandro 	cap_mask &= ~diff;
140f48ad614SDennis Dalessandro 	/* Update the bits with the new capability */
141f48ad614SDennis Dalessandro 	cap_mask |= (value & diff);
142f48ad614SDennis Dalessandro 	/* Check for any kernel/user restrictions */
143f48ad614SDennis Dalessandro 	diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
144f48ad614SDennis Dalessandro 		((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
145f48ad614SDennis Dalessandro 	cap_mask &= ~diff;
146f48ad614SDennis Dalessandro 	/* Set the bitmask to the final set */
147f48ad614SDennis Dalessandro 	*cap_mask_ptr = cap_mask;
148f48ad614SDennis Dalessandro done:
149f48ad614SDennis Dalessandro 	return ret;
150f48ad614SDennis Dalessandro }
151f48ad614SDennis Dalessandro 
152f48ad614SDennis Dalessandro static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
153f48ad614SDennis Dalessandro {
154f48ad614SDennis Dalessandro 	unsigned long cap_mask = *(unsigned long *)kp->arg;
155f48ad614SDennis Dalessandro 
156f48ad614SDennis Dalessandro 	cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
157f48ad614SDennis Dalessandro 	cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
158f48ad614SDennis Dalessandro 
159f48ad614SDennis Dalessandro 	return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
160f48ad614SDennis Dalessandro }
161f48ad614SDennis Dalessandro 
162f48ad614SDennis Dalessandro const char *get_unit_name(int unit)
163f48ad614SDennis Dalessandro {
164f48ad614SDennis Dalessandro 	static char iname[16];
165f48ad614SDennis Dalessandro 
166f48ad614SDennis Dalessandro 	snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit);
167f48ad614SDennis Dalessandro 	return iname;
168f48ad614SDennis Dalessandro }
169f48ad614SDennis Dalessandro 
170f48ad614SDennis Dalessandro const char *get_card_name(struct rvt_dev_info *rdi)
171f48ad614SDennis Dalessandro {
172f48ad614SDennis Dalessandro 	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
173f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = container_of(ibdev,
174f48ad614SDennis Dalessandro 					       struct hfi1_devdata, verbs_dev);
175f48ad614SDennis Dalessandro 	return get_unit_name(dd->unit);
176f48ad614SDennis Dalessandro }
177f48ad614SDennis Dalessandro 
178f48ad614SDennis Dalessandro struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
179f48ad614SDennis Dalessandro {
180f48ad614SDennis Dalessandro 	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
181f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = container_of(ibdev,
182f48ad614SDennis Dalessandro 					       struct hfi1_devdata, verbs_dev);
183f48ad614SDennis Dalessandro 	return dd->pcidev;
184f48ad614SDennis Dalessandro }
185f48ad614SDennis Dalessandro 
186f48ad614SDennis Dalessandro /*
187f48ad614SDennis Dalessandro  * Return count of units with at least one port ACTIVE.
188f48ad614SDennis Dalessandro  */
189f48ad614SDennis Dalessandro int hfi1_count_active_units(void)
190f48ad614SDennis Dalessandro {
191f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd;
192f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd;
193f48ad614SDennis Dalessandro 	unsigned long flags;
194f48ad614SDennis Dalessandro 	int pidx, nunits_active = 0;
195f48ad614SDennis Dalessandro 
196f48ad614SDennis Dalessandro 	spin_lock_irqsave(&hfi1_devs_lock, flags);
197f48ad614SDennis Dalessandro 	list_for_each_entry(dd, &hfi1_dev_list, list) {
198f48ad614SDennis Dalessandro 		if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase)
199f48ad614SDennis Dalessandro 			continue;
200f48ad614SDennis Dalessandro 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
201f48ad614SDennis Dalessandro 			ppd = dd->pport + pidx;
202f48ad614SDennis Dalessandro 			if (ppd->lid && ppd->linkup) {
203f48ad614SDennis Dalessandro 				nunits_active++;
204f48ad614SDennis Dalessandro 				break;
205f48ad614SDennis Dalessandro 			}
206f48ad614SDennis Dalessandro 		}
207f48ad614SDennis Dalessandro 	}
208f48ad614SDennis Dalessandro 	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
209f48ad614SDennis Dalessandro 	return nunits_active;
210f48ad614SDennis Dalessandro }
211f48ad614SDennis Dalessandro 
212f48ad614SDennis Dalessandro /*
213f48ad614SDennis Dalessandro  * Return count of all units, optionally return in arguments
214f48ad614SDennis Dalessandro  * the number of usable (present) units, and the number of
215f48ad614SDennis Dalessandro  * ports that are up.
216f48ad614SDennis Dalessandro  */
217f48ad614SDennis Dalessandro int hfi1_count_units(int *npresentp, int *nupp)
218f48ad614SDennis Dalessandro {
219f48ad614SDennis Dalessandro 	int nunits = 0, npresent = 0, nup = 0;
220f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd;
221f48ad614SDennis Dalessandro 	unsigned long flags;
222f48ad614SDennis Dalessandro 	int pidx;
223f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd;
224f48ad614SDennis Dalessandro 
225f48ad614SDennis Dalessandro 	spin_lock_irqsave(&hfi1_devs_lock, flags);
226f48ad614SDennis Dalessandro 
227f48ad614SDennis Dalessandro 	list_for_each_entry(dd, &hfi1_dev_list, list) {
228f48ad614SDennis Dalessandro 		nunits++;
229f48ad614SDennis Dalessandro 		if ((dd->flags & HFI1_PRESENT) && dd->kregbase)
230f48ad614SDennis Dalessandro 			npresent++;
231f48ad614SDennis Dalessandro 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
232f48ad614SDennis Dalessandro 			ppd = dd->pport + pidx;
233f48ad614SDennis Dalessandro 			if (ppd->lid && ppd->linkup)
234f48ad614SDennis Dalessandro 				nup++;
235f48ad614SDennis Dalessandro 		}
236f48ad614SDennis Dalessandro 	}
237f48ad614SDennis Dalessandro 
238f48ad614SDennis Dalessandro 	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
239f48ad614SDennis Dalessandro 
240f48ad614SDennis Dalessandro 	if (npresentp)
241f48ad614SDennis Dalessandro 		*npresentp = npresent;
242f48ad614SDennis Dalessandro 	if (nupp)
243f48ad614SDennis Dalessandro 		*nupp = nup;
244f48ad614SDennis Dalessandro 
245f48ad614SDennis Dalessandro 	return nunits;
246f48ad614SDennis Dalessandro }
247f48ad614SDennis Dalessandro 
248f48ad614SDennis Dalessandro /*
249f48ad614SDennis Dalessandro  * Get address of eager buffer from it's index (allocated in chunks, not
250f48ad614SDennis Dalessandro  * contiguous).
251f48ad614SDennis Dalessandro  */
252f48ad614SDennis Dalessandro static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
253f48ad614SDennis Dalessandro 			       u8 *update)
254f48ad614SDennis Dalessandro {
255f48ad614SDennis Dalessandro 	u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);
256f48ad614SDennis Dalessandro 
257f48ad614SDennis Dalessandro 	*update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
258f48ad614SDennis Dalessandro 	return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
259f48ad614SDennis Dalessandro 			(offset * RCV_BUF_BLOCK_SIZE));
260f48ad614SDennis Dalessandro }
261f48ad614SDennis Dalessandro 
262f48ad614SDennis Dalessandro /*
263f48ad614SDennis Dalessandro  * Validate and encode the a given RcvArray Buffer size.
264f48ad614SDennis Dalessandro  * The function will check whether the given size falls within
265f48ad614SDennis Dalessandro  * allowed size ranges for the respective type and, optionally,
266f48ad614SDennis Dalessandro  * return the proper encoding.
267f48ad614SDennis Dalessandro  */
268a82a7fcdSMike Marciniszyn int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
269f48ad614SDennis Dalessandro {
270f48ad614SDennis Dalessandro 	if (unlikely(!PAGE_ALIGNED(size)))
271f48ad614SDennis Dalessandro 		return 0;
272f48ad614SDennis Dalessandro 	if (unlikely(size < MIN_EAGER_BUFFER))
273f48ad614SDennis Dalessandro 		return 0;
274f48ad614SDennis Dalessandro 	if (size >
275f48ad614SDennis Dalessandro 	    (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
276f48ad614SDennis Dalessandro 		return 0;
277f48ad614SDennis Dalessandro 	if (encoded)
278f48ad614SDennis Dalessandro 		*encoded = ilog2(size / PAGE_SIZE) + 1;
279f48ad614SDennis Dalessandro 	return 1;
280f48ad614SDennis Dalessandro }
281f48ad614SDennis Dalessandro 
282f48ad614SDennis Dalessandro static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
283f48ad614SDennis Dalessandro 		       struct hfi1_packet *packet)
284f48ad614SDennis Dalessandro {
285261a4351SMike Marciniszyn 	struct ib_header *rhdr = packet->hdr;
286f48ad614SDennis Dalessandro 	u32 rte = rhf_rcv_type_err(packet->rhf);
287f48ad614SDennis Dalessandro 	int lnh = be16_to_cpu(rhdr->lrh[0]) & 3;
288f3e862cbSSebastian Sanchez 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
289f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
290f48ad614SDennis Dalessandro 	struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
291f48ad614SDennis Dalessandro 
292f48ad614SDennis Dalessandro 	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
293f48ad614SDennis Dalessandro 		return;
294f48ad614SDennis Dalessandro 
295f48ad614SDennis Dalessandro 	if (packet->rhf & RHF_TID_ERR) {
296f48ad614SDennis Dalessandro 		/* For TIDERR and RC QPs preemptively schedule a NAK */
297261a4351SMike Marciniszyn 		struct ib_other_headers *ohdr = NULL;
298f48ad614SDennis Dalessandro 		u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
299261a4351SMike Marciniszyn 		u16 lid  = be16_to_cpu(rhdr->lrh[1]);
300f48ad614SDennis Dalessandro 		u32 qp_num;
301f48ad614SDennis Dalessandro 		u32 rcv_flags = 0;
302f48ad614SDennis Dalessandro 
303f48ad614SDennis Dalessandro 		/* Sanity check packet */
304f48ad614SDennis Dalessandro 		if (tlen < 24)
305f48ad614SDennis Dalessandro 			goto drop;
306f48ad614SDennis Dalessandro 
307f48ad614SDennis Dalessandro 		/* Check for GRH */
308f48ad614SDennis Dalessandro 		if (lnh == HFI1_LRH_BTH) {
309261a4351SMike Marciniszyn 			ohdr = &rhdr->u.oth;
310f48ad614SDennis Dalessandro 		} else if (lnh == HFI1_LRH_GRH) {
311f48ad614SDennis Dalessandro 			u32 vtf;
312f48ad614SDennis Dalessandro 
313261a4351SMike Marciniszyn 			ohdr = &rhdr->u.l.oth;
314261a4351SMike Marciniszyn 			if (rhdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
315f48ad614SDennis Dalessandro 				goto drop;
316261a4351SMike Marciniszyn 			vtf = be32_to_cpu(rhdr->u.l.grh.version_tclass_flow);
317f48ad614SDennis Dalessandro 			if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
318f48ad614SDennis Dalessandro 				goto drop;
319f48ad614SDennis Dalessandro 			rcv_flags |= HFI1_HAS_GRH;
320f48ad614SDennis Dalessandro 		} else {
321f48ad614SDennis Dalessandro 			goto drop;
322f48ad614SDennis Dalessandro 		}
323f48ad614SDennis Dalessandro 		/* Get the destination QP number. */
324f48ad614SDennis Dalessandro 		qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
325f48ad614SDennis Dalessandro 		if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) {
326f48ad614SDennis Dalessandro 			struct rvt_qp *qp;
327f48ad614SDennis Dalessandro 			unsigned long flags;
328f48ad614SDennis Dalessandro 
329f48ad614SDennis Dalessandro 			rcu_read_lock();
330f48ad614SDennis Dalessandro 			qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
331f48ad614SDennis Dalessandro 			if (!qp) {
332f48ad614SDennis Dalessandro 				rcu_read_unlock();
333f48ad614SDennis Dalessandro 				goto drop;
334f48ad614SDennis Dalessandro 			}
335f48ad614SDennis Dalessandro 
336f48ad614SDennis Dalessandro 			/*
337f48ad614SDennis Dalessandro 			 * Handle only RC QPs - for other QP types drop error
338f48ad614SDennis Dalessandro 			 * packet.
339f48ad614SDennis Dalessandro 			 */
340f48ad614SDennis Dalessandro 			spin_lock_irqsave(&qp->r_lock, flags);
341f48ad614SDennis Dalessandro 
342f48ad614SDennis Dalessandro 			/* Check for valid receive state. */
343f48ad614SDennis Dalessandro 			if (!(ib_rvt_state_ops[qp->state] &
344f48ad614SDennis Dalessandro 			      RVT_PROCESS_RECV_OK)) {
345f48ad614SDennis Dalessandro 				ibp->rvp.n_pkt_drops++;
346f48ad614SDennis Dalessandro 			}
347f48ad614SDennis Dalessandro 
348f48ad614SDennis Dalessandro 			switch (qp->ibqp.qp_type) {
349f48ad614SDennis Dalessandro 			case IB_QPT_RC:
350f48ad614SDennis Dalessandro 				hfi1_rc_hdrerr(
351f48ad614SDennis Dalessandro 					rcd,
352261a4351SMike Marciniszyn 					rhdr,
353f48ad614SDennis Dalessandro 					rcv_flags,
354f48ad614SDennis Dalessandro 					qp);
355f48ad614SDennis Dalessandro 				break;
356f48ad614SDennis Dalessandro 			default:
357f48ad614SDennis Dalessandro 				/* For now don't handle any other QP types */
358f48ad614SDennis Dalessandro 				break;
359f48ad614SDennis Dalessandro 			}
360f48ad614SDennis Dalessandro 
361f48ad614SDennis Dalessandro 			spin_unlock_irqrestore(&qp->r_lock, flags);
362f48ad614SDennis Dalessandro 			rcu_read_unlock();
363f48ad614SDennis Dalessandro 		} /* Unicast QP */
364f48ad614SDennis Dalessandro 	} /* Valid packet with TIDErr */
365f48ad614SDennis Dalessandro 
366f48ad614SDennis Dalessandro 	/* handle "RcvTypeErr" flags */
367f48ad614SDennis Dalessandro 	switch (rte) {
368f48ad614SDennis Dalessandro 	case RHF_RTE_ERROR_OP_CODE_ERR:
369f48ad614SDennis Dalessandro 	{
370f48ad614SDennis Dalessandro 		u32 opcode;
371f48ad614SDennis Dalessandro 		void *ebuf = NULL;
372f48ad614SDennis Dalessandro 		__be32 *bth = NULL;
373f48ad614SDennis Dalessandro 
374f48ad614SDennis Dalessandro 		if (rhf_use_egr_bfr(packet->rhf))
375f48ad614SDennis Dalessandro 			ebuf = packet->ebuf;
376f48ad614SDennis Dalessandro 
377f48ad614SDennis Dalessandro 		if (!ebuf)
378f48ad614SDennis Dalessandro 			goto drop; /* this should never happen */
379f48ad614SDennis Dalessandro 
380f48ad614SDennis Dalessandro 		if (lnh == HFI1_LRH_BTH)
381f48ad614SDennis Dalessandro 			bth = (__be32 *)ebuf;
382f48ad614SDennis Dalessandro 		else if (lnh == HFI1_LRH_GRH)
383f48ad614SDennis Dalessandro 			bth = (__be32 *)((char *)ebuf + sizeof(struct ib_grh));
384f48ad614SDennis Dalessandro 		else
385f48ad614SDennis Dalessandro 			goto drop;
386f48ad614SDennis Dalessandro 
387f48ad614SDennis Dalessandro 		opcode = be32_to_cpu(bth[0]) >> 24;
388f48ad614SDennis Dalessandro 		opcode &= 0xff;
389f48ad614SDennis Dalessandro 
390f48ad614SDennis Dalessandro 		if (opcode == IB_OPCODE_CNP) {
391f48ad614SDennis Dalessandro 			/*
392f48ad614SDennis Dalessandro 			 * Only in pre-B0 h/w is the CNP_OPCODE handled
393f48ad614SDennis Dalessandro 			 * via this code path.
394f48ad614SDennis Dalessandro 			 */
395f48ad614SDennis Dalessandro 			struct rvt_qp *qp = NULL;
396f48ad614SDennis Dalessandro 			u32 lqpn, rqpn;
397f48ad614SDennis Dalessandro 			u16 rlid;
398f48ad614SDennis Dalessandro 			u8 svc_type, sl, sc5;
399f48ad614SDennis Dalessandro 
400b736a469SDasaratharaman Chandramouli 			sc5 = hdr2sc(rhdr, packet->rhf);
401f48ad614SDennis Dalessandro 			sl = ibp->sc_to_sl[sc5];
402f48ad614SDennis Dalessandro 
403f48ad614SDennis Dalessandro 			lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK;
404f48ad614SDennis Dalessandro 			rcu_read_lock();
405f48ad614SDennis Dalessandro 			qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
406f48ad614SDennis Dalessandro 			if (!qp) {
407f48ad614SDennis Dalessandro 				rcu_read_unlock();
408f48ad614SDennis Dalessandro 				goto drop;
409f48ad614SDennis Dalessandro 			}
410f48ad614SDennis Dalessandro 
411f48ad614SDennis Dalessandro 			switch (qp->ibqp.qp_type) {
412f48ad614SDennis Dalessandro 			case IB_QPT_UD:
413f48ad614SDennis Dalessandro 				rlid = 0;
414f48ad614SDennis Dalessandro 				rqpn = 0;
415f48ad614SDennis Dalessandro 				svc_type = IB_CC_SVCTYPE_UD;
416f48ad614SDennis Dalessandro 				break;
417f48ad614SDennis Dalessandro 			case IB_QPT_UC:
418f48ad614SDennis Dalessandro 				rlid = be16_to_cpu(rhdr->lrh[3]);
419f48ad614SDennis Dalessandro 				rqpn = qp->remote_qpn;
420f48ad614SDennis Dalessandro 				svc_type = IB_CC_SVCTYPE_UC;
421f48ad614SDennis Dalessandro 				break;
422f48ad614SDennis Dalessandro 			default:
423f48ad614SDennis Dalessandro 				goto drop;
424f48ad614SDennis Dalessandro 			}
425f48ad614SDennis Dalessandro 
426f48ad614SDennis Dalessandro 			process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
427f48ad614SDennis Dalessandro 			rcu_read_unlock();
428f48ad614SDennis Dalessandro 		}
429f48ad614SDennis Dalessandro 
430f48ad614SDennis Dalessandro 		packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
431f48ad614SDennis Dalessandro 		break;
432f48ad614SDennis Dalessandro 	}
433f48ad614SDennis Dalessandro 	default:
434f48ad614SDennis Dalessandro 		break;
435f48ad614SDennis Dalessandro 	}
436f48ad614SDennis Dalessandro 
437f48ad614SDennis Dalessandro drop:
438f48ad614SDennis Dalessandro 	return;
439f48ad614SDennis Dalessandro }
440f48ad614SDennis Dalessandro 
441f48ad614SDennis Dalessandro static inline void init_packet(struct hfi1_ctxtdata *rcd,
442f48ad614SDennis Dalessandro 			       struct hfi1_packet *packet)
443f48ad614SDennis Dalessandro {
444f48ad614SDennis Dalessandro 	packet->rsize = rcd->rcvhdrqentsize; /* words */
445f48ad614SDennis Dalessandro 	packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */
446f48ad614SDennis Dalessandro 	packet->rcd = rcd;
447f48ad614SDennis Dalessandro 	packet->updegr = 0;
448f48ad614SDennis Dalessandro 	packet->etail = -1;
449f48ad614SDennis Dalessandro 	packet->rhf_addr = get_rhf_addr(rcd);
450f48ad614SDennis Dalessandro 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
451f48ad614SDennis Dalessandro 	packet->rhqoff = rcd->head;
452f48ad614SDennis Dalessandro 	packet->numpkt = 0;
453f48ad614SDennis Dalessandro 	packet->rcv_flags = 0;
454f48ad614SDennis Dalessandro }
455f48ad614SDennis Dalessandro 
4565fd2b562SMitko Haralanov void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
4575fd2b562SMitko Haralanov 			       bool do_cnp)
458f48ad614SDennis Dalessandro {
459f48ad614SDennis Dalessandro 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
460261a4351SMike Marciniszyn 	struct ib_header *hdr = pkt->hdr;
461261a4351SMike Marciniszyn 	struct ib_other_headers *ohdr = pkt->ohdr;
4625fd2b562SMitko Haralanov 	struct ib_grh *grh = NULL;
4635fd2b562SMitko Haralanov 	u32 rqpn = 0, bth1;
4645fd2b562SMitko Haralanov 	u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]);
4655fd2b562SMitko Haralanov 	u8 sc, svc_type;
4665fd2b562SMitko Haralanov 	bool is_mcast = false;
4675fd2b562SMitko Haralanov 
4685fd2b562SMitko Haralanov 	if (pkt->rcv_flags & HFI1_HAS_GRH)
4695fd2b562SMitko Haralanov 		grh = &hdr->u.l.grh;
470f48ad614SDennis Dalessandro 
471f48ad614SDennis Dalessandro 	switch (qp->ibqp.qp_type) {
472f48ad614SDennis Dalessandro 	case IB_QPT_SMI:
473f48ad614SDennis Dalessandro 	case IB_QPT_GSI:
474f48ad614SDennis Dalessandro 	case IB_QPT_UD:
475f48ad614SDennis Dalessandro 		rlid = be16_to_cpu(hdr->lrh[3]);
476f48ad614SDennis Dalessandro 		rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK;
477f48ad614SDennis Dalessandro 		svc_type = IB_CC_SVCTYPE_UD;
4785fd2b562SMitko Haralanov 		is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
4795fd2b562SMitko Haralanov 			(dlid != be16_to_cpu(IB_LID_PERMISSIVE));
480f48ad614SDennis Dalessandro 		break;
481f48ad614SDennis Dalessandro 	case IB_QPT_UC:
482f48ad614SDennis Dalessandro 		rlid = qp->remote_ah_attr.dlid;
483f48ad614SDennis Dalessandro 		rqpn = qp->remote_qpn;
484f48ad614SDennis Dalessandro 		svc_type = IB_CC_SVCTYPE_UC;
485f48ad614SDennis Dalessandro 		break;
486f48ad614SDennis Dalessandro 	case IB_QPT_RC:
487f48ad614SDennis Dalessandro 		rlid = qp->remote_ah_attr.dlid;
488f48ad614SDennis Dalessandro 		rqpn = qp->remote_qpn;
489f48ad614SDennis Dalessandro 		svc_type = IB_CC_SVCTYPE_RC;
490f48ad614SDennis Dalessandro 		break;
491f48ad614SDennis Dalessandro 	default:
492f48ad614SDennis Dalessandro 		return;
493f48ad614SDennis Dalessandro 	}
494f48ad614SDennis Dalessandro 
495261a4351SMike Marciniszyn 	sc = hdr2sc(hdr, pkt->rhf);
496f48ad614SDennis Dalessandro 
4975fd2b562SMitko Haralanov 	bth1 = be32_to_cpu(ohdr->bth[1]);
4985fd2b562SMitko Haralanov 	if (do_cnp && (bth1 & HFI1_FECN_SMASK)) {
499f48ad614SDennis Dalessandro 		u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]);
500f48ad614SDennis Dalessandro 
5015fd2b562SMitko Haralanov 		return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh);
502f48ad614SDennis Dalessandro 	}
503f48ad614SDennis Dalessandro 
5045fd2b562SMitko Haralanov 	if (!is_mcast && (bth1 & HFI1_BECN_SMASK)) {
505f48ad614SDennis Dalessandro 		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
506f48ad614SDennis Dalessandro 		u32 lqpn = bth1 & RVT_QPN_MASK;
5075fd2b562SMitko Haralanov 		u8 sl = ibp->sc_to_sl[sc];
508f48ad614SDennis Dalessandro 
509f48ad614SDennis Dalessandro 		process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
510f48ad614SDennis Dalessandro 	}
5115fd2b562SMitko Haralanov 
512f48ad614SDennis Dalessandro }
513f48ad614SDennis Dalessandro 
514f48ad614SDennis Dalessandro struct ps_mdata {
515f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd;
516f48ad614SDennis Dalessandro 	u32 rsize;
517f48ad614SDennis Dalessandro 	u32 maxcnt;
518f48ad614SDennis Dalessandro 	u32 ps_head;
519f48ad614SDennis Dalessandro 	u32 ps_tail;
520f48ad614SDennis Dalessandro 	u32 ps_seq;
521f48ad614SDennis Dalessandro };
522f48ad614SDennis Dalessandro 
523f48ad614SDennis Dalessandro static inline void init_ps_mdata(struct ps_mdata *mdata,
524f48ad614SDennis Dalessandro 				 struct hfi1_packet *packet)
525f48ad614SDennis Dalessandro {
526f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd = packet->rcd;
527f48ad614SDennis Dalessandro 
528f48ad614SDennis Dalessandro 	mdata->rcd = rcd;
529f48ad614SDennis Dalessandro 	mdata->rsize = packet->rsize;
530f48ad614SDennis Dalessandro 	mdata->maxcnt = packet->maxcnt;
531f48ad614SDennis Dalessandro 	mdata->ps_head = packet->rhqoff;
532f48ad614SDennis Dalessandro 
533f48ad614SDennis Dalessandro 	if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
534f48ad614SDennis Dalessandro 		mdata->ps_tail = get_rcvhdrtail(rcd);
535f48ad614SDennis Dalessandro 		if (rcd->ctxt == HFI1_CTRL_CTXT)
536f48ad614SDennis Dalessandro 			mdata->ps_seq = rcd->seq_cnt;
537f48ad614SDennis Dalessandro 		else
538f48ad614SDennis Dalessandro 			mdata->ps_seq = 0; /* not used with DMA_RTAIL */
539f48ad614SDennis Dalessandro 	} else {
540f48ad614SDennis Dalessandro 		mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
541f48ad614SDennis Dalessandro 		mdata->ps_seq = rcd->seq_cnt;
542f48ad614SDennis Dalessandro 	}
543f48ad614SDennis Dalessandro }
544f48ad614SDennis Dalessandro 
545f48ad614SDennis Dalessandro static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
546f48ad614SDennis Dalessandro 			  struct hfi1_ctxtdata *rcd)
547f48ad614SDennis Dalessandro {
548f48ad614SDennis Dalessandro 	if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
549f48ad614SDennis Dalessandro 		return mdata->ps_head == mdata->ps_tail;
550f48ad614SDennis Dalessandro 	return mdata->ps_seq != rhf_rcv_seq(rhf);
551f48ad614SDennis Dalessandro }
552f48ad614SDennis Dalessandro 
553f48ad614SDennis Dalessandro static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
554f48ad614SDennis Dalessandro 			  struct hfi1_ctxtdata *rcd)
555f48ad614SDennis Dalessandro {
556f48ad614SDennis Dalessandro 	/*
557f48ad614SDennis Dalessandro 	 * Control context can potentially receive an invalid rhf.
558f48ad614SDennis Dalessandro 	 * Drop such packets.
559f48ad614SDennis Dalessandro 	 */
560f48ad614SDennis Dalessandro 	if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
561f48ad614SDennis Dalessandro 		return mdata->ps_seq != rhf_rcv_seq(rhf);
562f48ad614SDennis Dalessandro 
563f48ad614SDennis Dalessandro 	return 0;
564f48ad614SDennis Dalessandro }
565f48ad614SDennis Dalessandro 
566f48ad614SDennis Dalessandro static inline void update_ps_mdata(struct ps_mdata *mdata,
567f48ad614SDennis Dalessandro 				   struct hfi1_ctxtdata *rcd)
568f48ad614SDennis Dalessandro {
569f48ad614SDennis Dalessandro 	mdata->ps_head += mdata->rsize;
570f48ad614SDennis Dalessandro 	if (mdata->ps_head >= mdata->maxcnt)
571f48ad614SDennis Dalessandro 		mdata->ps_head = 0;
572f48ad614SDennis Dalessandro 
573f48ad614SDennis Dalessandro 	/* Control context must do seq counting */
574f48ad614SDennis Dalessandro 	if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
575f48ad614SDennis Dalessandro 	    (rcd->ctxt == HFI1_CTRL_CTXT)) {
576f48ad614SDennis Dalessandro 		if (++mdata->ps_seq > 13)
577f48ad614SDennis Dalessandro 			mdata->ps_seq = 1;
578f48ad614SDennis Dalessandro 	}
579f48ad614SDennis Dalessandro }
580f48ad614SDennis Dalessandro 
581f48ad614SDennis Dalessandro /*
582f48ad614SDennis Dalessandro  * prescan_rxq - search through the receive queue looking for packets
583f48ad614SDennis Dalessandro  * containing Excplicit Congestion Notifications (FECNs, or BECNs).
584f48ad614SDennis Dalessandro  * When an ECN is found, process the Congestion Notification, and toggle
585f48ad614SDennis Dalessandro  * it off.
586f48ad614SDennis Dalessandro  * This is declared as a macro to allow quick checking of the port to avoid
587f48ad614SDennis Dalessandro  * the overhead of a function call if not enabled.
588f48ad614SDennis Dalessandro  */
589f48ad614SDennis Dalessandro #define prescan_rxq(rcd, packet) \
590f48ad614SDennis Dalessandro 	do { \
591f48ad614SDennis Dalessandro 		if (rcd->ppd->cc_prescan) \
592f48ad614SDennis Dalessandro 			__prescan_rxq(packet); \
593f48ad614SDennis Dalessandro 	} while (0)
594f48ad614SDennis Dalessandro static void __prescan_rxq(struct hfi1_packet *packet)
595f48ad614SDennis Dalessandro {
596f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd = packet->rcd;
597f48ad614SDennis Dalessandro 	struct ps_mdata mdata;
598f48ad614SDennis Dalessandro 
599f48ad614SDennis Dalessandro 	init_ps_mdata(&mdata, packet);
600f48ad614SDennis Dalessandro 
601f48ad614SDennis Dalessandro 	while (1) {
602f48ad614SDennis Dalessandro 		struct hfi1_devdata *dd = rcd->dd;
603f3e862cbSSebastian Sanchez 		struct hfi1_ibport *ibp = rcd_to_iport(rcd);
604f48ad614SDennis Dalessandro 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
605f48ad614SDennis Dalessandro 					 dd->rhf_offset;
606f48ad614SDennis Dalessandro 		struct rvt_qp *qp;
607261a4351SMike Marciniszyn 		struct ib_header *hdr;
608f48ad614SDennis Dalessandro 		struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
609f48ad614SDennis Dalessandro 		u64 rhf = rhf_to_cpu(rhf_addr);
610f48ad614SDennis Dalessandro 		u32 etype = rhf_rcv_type(rhf), qpn, bth1;
611f48ad614SDennis Dalessandro 		int is_ecn = 0;
612f48ad614SDennis Dalessandro 		u8 lnh;
613f48ad614SDennis Dalessandro 
614f48ad614SDennis Dalessandro 		if (ps_done(&mdata, rhf, rcd))
615f48ad614SDennis Dalessandro 			break;
616f48ad614SDennis Dalessandro 
617f48ad614SDennis Dalessandro 		if (ps_skip(&mdata, rhf, rcd))
618f48ad614SDennis Dalessandro 			goto next;
619f48ad614SDennis Dalessandro 
620f48ad614SDennis Dalessandro 		if (etype != RHF_RCV_TYPE_IB)
621f48ad614SDennis Dalessandro 			goto next;
622f48ad614SDennis Dalessandro 
623f2d8a0b3SDasaratharaman Chandramouli 		packet->hdr = hfi1_get_msgheader(dd, rhf_addr);
624f2d8a0b3SDasaratharaman Chandramouli 		hdr = packet->hdr;
625261a4351SMike Marciniszyn 
626f48ad614SDennis Dalessandro 		lnh = be16_to_cpu(hdr->lrh[0]) & 3;
627f48ad614SDennis Dalessandro 
628f2d8a0b3SDasaratharaman Chandramouli 		if (lnh == HFI1_LRH_BTH) {
629f2d8a0b3SDasaratharaman Chandramouli 			packet->ohdr = &hdr->u.oth;
630f2d8a0b3SDasaratharaman Chandramouli 		} else if (lnh == HFI1_LRH_GRH) {
631f2d8a0b3SDasaratharaman Chandramouli 			packet->ohdr = &hdr->u.l.oth;
632f2d8a0b3SDasaratharaman Chandramouli 			packet->rcv_flags |= HFI1_HAS_GRH;
633f2d8a0b3SDasaratharaman Chandramouli 		} else {
634f48ad614SDennis Dalessandro 			goto next; /* just in case */
635f2d8a0b3SDasaratharaman Chandramouli 		}
6365fd2b562SMitko Haralanov 
637f2d8a0b3SDasaratharaman Chandramouli 		bth1 = be32_to_cpu(packet->ohdr->bth[1]);
638f48ad614SDennis Dalessandro 		is_ecn = !!(bth1 & (HFI1_FECN_SMASK | HFI1_BECN_SMASK));
639f48ad614SDennis Dalessandro 
640f48ad614SDennis Dalessandro 		if (!is_ecn)
641f48ad614SDennis Dalessandro 			goto next;
642f48ad614SDennis Dalessandro 
643f48ad614SDennis Dalessandro 		qpn = bth1 & RVT_QPN_MASK;
644f48ad614SDennis Dalessandro 		rcu_read_lock();
645f48ad614SDennis Dalessandro 		qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
646f48ad614SDennis Dalessandro 
647f48ad614SDennis Dalessandro 		if (!qp) {
648f48ad614SDennis Dalessandro 			rcu_read_unlock();
649f48ad614SDennis Dalessandro 			goto next;
650f48ad614SDennis Dalessandro 		}
651f48ad614SDennis Dalessandro 
6525fd2b562SMitko Haralanov 		process_ecn(qp, packet, true);
653f48ad614SDennis Dalessandro 		rcu_read_unlock();
654f48ad614SDennis Dalessandro 
655f48ad614SDennis Dalessandro 		/* turn off BECN, FECN */
656f48ad614SDennis Dalessandro 		bth1 &= ~(HFI1_FECN_SMASK | HFI1_BECN_SMASK);
657f2d8a0b3SDasaratharaman Chandramouli 		packet->ohdr->bth[1] = cpu_to_be32(bth1);
658f48ad614SDennis Dalessandro next:
659f48ad614SDennis Dalessandro 		update_ps_mdata(&mdata, rcd);
660f48ad614SDennis Dalessandro 	}
661f48ad614SDennis Dalessandro }
662f48ad614SDennis Dalessandro 
663a82a7fcdSMike Marciniszyn static void process_rcv_qp_work(struct hfi1_ctxtdata *rcd)
664a82a7fcdSMike Marciniszyn {
665a82a7fcdSMike Marciniszyn 	struct rvt_qp *qp, *nqp;
666a82a7fcdSMike Marciniszyn 
667a82a7fcdSMike Marciniszyn 	/*
668a82a7fcdSMike Marciniszyn 	 * Iterate over all QPs waiting to respond.
669a82a7fcdSMike Marciniszyn 	 * The list won't change since the IRQ is only run on one CPU.
670a82a7fcdSMike Marciniszyn 	 */
671a82a7fcdSMike Marciniszyn 	list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
672a82a7fcdSMike Marciniszyn 		list_del_init(&qp->rspwait);
673a82a7fcdSMike Marciniszyn 		if (qp->r_flags & RVT_R_RSP_NAK) {
674a82a7fcdSMike Marciniszyn 			qp->r_flags &= ~RVT_R_RSP_NAK;
675a82a7fcdSMike Marciniszyn 			hfi1_send_rc_ack(rcd, qp, 0);
676a82a7fcdSMike Marciniszyn 		}
677a82a7fcdSMike Marciniszyn 		if (qp->r_flags & RVT_R_RSP_SEND) {
678a82a7fcdSMike Marciniszyn 			unsigned long flags;
679a82a7fcdSMike Marciniszyn 
680a82a7fcdSMike Marciniszyn 			qp->r_flags &= ~RVT_R_RSP_SEND;
681a82a7fcdSMike Marciniszyn 			spin_lock_irqsave(&qp->s_lock, flags);
682a82a7fcdSMike Marciniszyn 			if (ib_rvt_state_ops[qp->state] &
683a82a7fcdSMike Marciniszyn 					RVT_PROCESS_OR_FLUSH_SEND)
684a82a7fcdSMike Marciniszyn 				hfi1_schedule_send(qp);
685a82a7fcdSMike Marciniszyn 			spin_unlock_irqrestore(&qp->s_lock, flags);
686a82a7fcdSMike Marciniszyn 		}
687a82a7fcdSMike Marciniszyn 		rvt_put_qp(qp);
688a82a7fcdSMike Marciniszyn 	}
689a82a7fcdSMike Marciniszyn }
690a82a7fcdSMike Marciniszyn 
691a82a7fcdSMike Marciniszyn static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread)
692a82a7fcdSMike Marciniszyn {
693a82a7fcdSMike Marciniszyn 	if (thread) {
694a82a7fcdSMike Marciniszyn 		if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
695a82a7fcdSMike Marciniszyn 			/* allow defered processing */
696a82a7fcdSMike Marciniszyn 			process_rcv_qp_work(packet->rcd);
697a82a7fcdSMike Marciniszyn 		cond_resched();
698a82a7fcdSMike Marciniszyn 		return RCV_PKT_OK;
699a82a7fcdSMike Marciniszyn 	} else {
700a82a7fcdSMike Marciniszyn 		this_cpu_inc(*packet->rcd->dd->rcv_limit);
701a82a7fcdSMike Marciniszyn 		return RCV_PKT_LIMIT;
702a82a7fcdSMike Marciniszyn 	}
703a82a7fcdSMike Marciniszyn }
704a82a7fcdSMike Marciniszyn 
705a82a7fcdSMike Marciniszyn static inline int check_max_packet(struct hfi1_packet *packet, int thread)
706f48ad614SDennis Dalessandro {
707f48ad614SDennis Dalessandro 	int ret = RCV_PKT_OK;
708f48ad614SDennis Dalessandro 
709a82a7fcdSMike Marciniszyn 	if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0))
710a82a7fcdSMike Marciniszyn 		ret = max_packet_exceeded(packet, thread);
711a82a7fcdSMike Marciniszyn 	return ret;
712a82a7fcdSMike Marciniszyn }
713a82a7fcdSMike Marciniszyn 
714a82a7fcdSMike Marciniszyn static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
715a82a7fcdSMike Marciniszyn {
716a82a7fcdSMike Marciniszyn 	int ret;
717a82a7fcdSMike Marciniszyn 
718f48ad614SDennis Dalessandro 	/* Set up for the next packet */
719f48ad614SDennis Dalessandro 	packet->rhqoff += packet->rsize;
720f48ad614SDennis Dalessandro 	if (packet->rhqoff >= packet->maxcnt)
721f48ad614SDennis Dalessandro 		packet->rhqoff = 0;
722f48ad614SDennis Dalessandro 
723f48ad614SDennis Dalessandro 	packet->numpkt++;
724a82a7fcdSMike Marciniszyn 	ret = check_max_packet(packet, thread);
725f48ad614SDennis Dalessandro 
726f48ad614SDennis Dalessandro 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
727f48ad614SDennis Dalessandro 				     packet->rcd->dd->rhf_offset;
728f48ad614SDennis Dalessandro 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
729f48ad614SDennis Dalessandro 
730f48ad614SDennis Dalessandro 	return ret;
731f48ad614SDennis Dalessandro }
732f48ad614SDennis Dalessandro 
733f48ad614SDennis Dalessandro static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
734f48ad614SDennis Dalessandro {
735a82a7fcdSMike Marciniszyn 	int ret;
736f48ad614SDennis Dalessandro 
737f48ad614SDennis Dalessandro 	packet->hdr = hfi1_get_msgheader(packet->rcd->dd,
738f48ad614SDennis Dalessandro 					 packet->rhf_addr);
739f48ad614SDennis Dalessandro 	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
740f48ad614SDennis Dalessandro 	packet->etype = rhf_rcv_type(packet->rhf);
741f48ad614SDennis Dalessandro 	/* total length */
742f48ad614SDennis Dalessandro 	packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
743f48ad614SDennis Dalessandro 	/* retrieve eager buffer details */
744f48ad614SDennis Dalessandro 	packet->ebuf = NULL;
745f48ad614SDennis Dalessandro 	if (rhf_use_egr_bfr(packet->rhf)) {
746f48ad614SDennis Dalessandro 		packet->etail = rhf_egr_index(packet->rhf);
747f48ad614SDennis Dalessandro 		packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
748f48ad614SDennis Dalessandro 				 &packet->updegr);
749f48ad614SDennis Dalessandro 		/*
750f48ad614SDennis Dalessandro 		 * Prefetch the contents of the eager buffer.  It is
751f48ad614SDennis Dalessandro 		 * OK to send a negative length to prefetch_range().
752f48ad614SDennis Dalessandro 		 * The +2 is the size of the RHF.
753f48ad614SDennis Dalessandro 		 */
754f48ad614SDennis Dalessandro 		prefetch_range(packet->ebuf,
755f48ad614SDennis Dalessandro 			       packet->tlen - ((packet->rcd->rcvhdrqentsize -
756f48ad614SDennis Dalessandro 					       (rhf_hdrq_offset(packet->rhf)
757f48ad614SDennis Dalessandro 						+ 2)) * 4));
758f48ad614SDennis Dalessandro 	}
759f48ad614SDennis Dalessandro 
760f48ad614SDennis Dalessandro 	/*
761f48ad614SDennis Dalessandro 	 * Call a type specific handler for the packet. We
762f48ad614SDennis Dalessandro 	 * should be able to trust that etype won't be beyond
763f48ad614SDennis Dalessandro 	 * the range of valid indexes. If so something is really
764f48ad614SDennis Dalessandro 	 * wrong and we can probably just let things come
765f48ad614SDennis Dalessandro 	 * crashing down. There is no need to eat another
766f48ad614SDennis Dalessandro 	 * comparison in this performance critical code.
767f48ad614SDennis Dalessandro 	 */
768f48ad614SDennis Dalessandro 	packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet);
769f48ad614SDennis Dalessandro 	packet->numpkt++;
770f48ad614SDennis Dalessandro 
771f48ad614SDennis Dalessandro 	/* Set up for the next packet */
772f48ad614SDennis Dalessandro 	packet->rhqoff += packet->rsize;
773f48ad614SDennis Dalessandro 	if (packet->rhqoff >= packet->maxcnt)
774f48ad614SDennis Dalessandro 		packet->rhqoff = 0;
775f48ad614SDennis Dalessandro 
776a82a7fcdSMike Marciniszyn 	ret = check_max_packet(packet, thread);
777f48ad614SDennis Dalessandro 
778f48ad614SDennis Dalessandro 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
779f48ad614SDennis Dalessandro 				      packet->rcd->dd->rhf_offset;
780f48ad614SDennis Dalessandro 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
781f48ad614SDennis Dalessandro 
782f48ad614SDennis Dalessandro 	return ret;
783f48ad614SDennis Dalessandro }
784f48ad614SDennis Dalessandro 
785f48ad614SDennis Dalessandro static inline void process_rcv_update(int last, struct hfi1_packet *packet)
786f48ad614SDennis Dalessandro {
787f48ad614SDennis Dalessandro 	/*
788f48ad614SDennis Dalessandro 	 * Update head regs etc., every 16 packets, if not last pkt,
789f48ad614SDennis Dalessandro 	 * to help prevent rcvhdrq overflows, when many packets
790f48ad614SDennis Dalessandro 	 * are processed and queue is nearly full.
791f48ad614SDennis Dalessandro 	 * Don't request an interrupt for intermediate updates.
792f48ad614SDennis Dalessandro 	 */
793f48ad614SDennis Dalessandro 	if (!last && !(packet->numpkt & 0xf)) {
794f48ad614SDennis Dalessandro 		update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
795f48ad614SDennis Dalessandro 			       packet->etail, 0, 0);
796f48ad614SDennis Dalessandro 		packet->updegr = 0;
797f48ad614SDennis Dalessandro 	}
798f48ad614SDennis Dalessandro 	packet->rcv_flags = 0;
799f48ad614SDennis Dalessandro }
800f48ad614SDennis Dalessandro 
801f48ad614SDennis Dalessandro static inline void finish_packet(struct hfi1_packet *packet)
802f48ad614SDennis Dalessandro {
803f48ad614SDennis Dalessandro 	/*
804f48ad614SDennis Dalessandro 	 * Nothing we need to free for the packet.
805f48ad614SDennis Dalessandro 	 *
806f48ad614SDennis Dalessandro 	 * The only thing we need to do is a final update and call for an
807f48ad614SDennis Dalessandro 	 * interrupt
808f48ad614SDennis Dalessandro 	 */
809f48ad614SDennis Dalessandro 	update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
810f48ad614SDennis Dalessandro 		       packet->etail, rcv_intr_dynamic, packet->numpkt);
811f48ad614SDennis Dalessandro }
812f48ad614SDennis Dalessandro 
813f48ad614SDennis Dalessandro /*
814f48ad614SDennis Dalessandro  * Handle receive interrupts when using the no dma rtail option.
815f48ad614SDennis Dalessandro  */
816f48ad614SDennis Dalessandro int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
817f48ad614SDennis Dalessandro {
818f48ad614SDennis Dalessandro 	u32 seq;
819f48ad614SDennis Dalessandro 	int last = RCV_PKT_OK;
820f48ad614SDennis Dalessandro 	struct hfi1_packet packet;
821f48ad614SDennis Dalessandro 
822f48ad614SDennis Dalessandro 	init_packet(rcd, &packet);
823f48ad614SDennis Dalessandro 	seq = rhf_rcv_seq(packet.rhf);
824f48ad614SDennis Dalessandro 	if (seq != rcd->seq_cnt) {
825f48ad614SDennis Dalessandro 		last = RCV_PKT_DONE;
826f48ad614SDennis Dalessandro 		goto bail;
827f48ad614SDennis Dalessandro 	}
828f48ad614SDennis Dalessandro 
829f48ad614SDennis Dalessandro 	prescan_rxq(rcd, &packet);
830f48ad614SDennis Dalessandro 
831f48ad614SDennis Dalessandro 	while (last == RCV_PKT_OK) {
832f48ad614SDennis Dalessandro 		last = process_rcv_packet(&packet, thread);
833f48ad614SDennis Dalessandro 		seq = rhf_rcv_seq(packet.rhf);
834f48ad614SDennis Dalessandro 		if (++rcd->seq_cnt > 13)
835f48ad614SDennis Dalessandro 			rcd->seq_cnt = 1;
836f48ad614SDennis Dalessandro 		if (seq != rcd->seq_cnt)
837f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
838f48ad614SDennis Dalessandro 		process_rcv_update(last, &packet);
839f48ad614SDennis Dalessandro 	}
840a82a7fcdSMike Marciniszyn 	process_rcv_qp_work(rcd);
841a82a7fcdSMike Marciniszyn 	rcd->head = packet.rhqoff;
842f48ad614SDennis Dalessandro bail:
843f48ad614SDennis Dalessandro 	finish_packet(&packet);
844f48ad614SDennis Dalessandro 	return last;
845f48ad614SDennis Dalessandro }
846f48ad614SDennis Dalessandro 
847f48ad614SDennis Dalessandro int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
848f48ad614SDennis Dalessandro {
849f48ad614SDennis Dalessandro 	u32 hdrqtail;
850f48ad614SDennis Dalessandro 	int last = RCV_PKT_OK;
851f48ad614SDennis Dalessandro 	struct hfi1_packet packet;
852f48ad614SDennis Dalessandro 
853f48ad614SDennis Dalessandro 	init_packet(rcd, &packet);
854f48ad614SDennis Dalessandro 	hdrqtail = get_rcvhdrtail(rcd);
855f48ad614SDennis Dalessandro 	if (packet.rhqoff == hdrqtail) {
856f48ad614SDennis Dalessandro 		last = RCV_PKT_DONE;
857f48ad614SDennis Dalessandro 		goto bail;
858f48ad614SDennis Dalessandro 	}
859f48ad614SDennis Dalessandro 	smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
860f48ad614SDennis Dalessandro 
861f48ad614SDennis Dalessandro 	prescan_rxq(rcd, &packet);
862f48ad614SDennis Dalessandro 
863f48ad614SDennis Dalessandro 	while (last == RCV_PKT_OK) {
864f48ad614SDennis Dalessandro 		last = process_rcv_packet(&packet, thread);
865f48ad614SDennis Dalessandro 		if (packet.rhqoff == hdrqtail)
866f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
867f48ad614SDennis Dalessandro 		process_rcv_update(last, &packet);
868f48ad614SDennis Dalessandro 	}
869a82a7fcdSMike Marciniszyn 	process_rcv_qp_work(rcd);
870a82a7fcdSMike Marciniszyn 	rcd->head = packet.rhqoff;
871f48ad614SDennis Dalessandro bail:
872f48ad614SDennis Dalessandro 	finish_packet(&packet);
873f48ad614SDennis Dalessandro 	return last;
874f48ad614SDennis Dalessandro }
875f48ad614SDennis Dalessandro 
876f48ad614SDennis Dalessandro static inline void set_all_nodma_rtail(struct hfi1_devdata *dd)
877f48ad614SDennis Dalessandro {
878f48ad614SDennis Dalessandro 	int i;
879f48ad614SDennis Dalessandro 
880f48ad614SDennis Dalessandro 	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
881f48ad614SDennis Dalessandro 		dd->rcd[i]->do_interrupt =
882f48ad614SDennis Dalessandro 			&handle_receive_interrupt_nodma_rtail;
883f48ad614SDennis Dalessandro }
884f48ad614SDennis Dalessandro 
885f48ad614SDennis Dalessandro static inline void set_all_dma_rtail(struct hfi1_devdata *dd)
886f48ad614SDennis Dalessandro {
887f48ad614SDennis Dalessandro 	int i;
888f48ad614SDennis Dalessandro 
889f48ad614SDennis Dalessandro 	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
890f48ad614SDennis Dalessandro 		dd->rcd[i]->do_interrupt =
891f48ad614SDennis Dalessandro 			&handle_receive_interrupt_dma_rtail;
892f48ad614SDennis Dalessandro }
893f48ad614SDennis Dalessandro 
894f48ad614SDennis Dalessandro void set_all_slowpath(struct hfi1_devdata *dd)
895f48ad614SDennis Dalessandro {
896f48ad614SDennis Dalessandro 	int i;
897f48ad614SDennis Dalessandro 
898f48ad614SDennis Dalessandro 	/* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
899f48ad614SDennis Dalessandro 	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++)
900f48ad614SDennis Dalessandro 		dd->rcd[i]->do_interrupt = &handle_receive_interrupt;
901f48ad614SDennis Dalessandro }
902f48ad614SDennis Dalessandro 
903f48ad614SDennis Dalessandro static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
904c867caafSMike Marciniszyn 				      struct hfi1_packet *packet,
905f48ad614SDennis Dalessandro 				      struct hfi1_devdata *dd)
906f48ad614SDennis Dalessandro {
907f48ad614SDennis Dalessandro 	struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
908261a4351SMike Marciniszyn 	struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
909c867caafSMike Marciniszyn 						   packet->rhf_addr);
91069b9f4a4SMike Marciniszyn 	u8 etype = rhf_rcv_type(packet->rhf);
911f48ad614SDennis Dalessandro 
91269b9f4a4SMike Marciniszyn 	if (etype == RHF_RCV_TYPE_IB && hdr2sc(hdr, packet->rhf) != 0xf) {
913f48ad614SDennis Dalessandro 		int hwstate = read_logical_state(dd);
914f48ad614SDennis Dalessandro 
915f48ad614SDennis Dalessandro 		if (hwstate != LSTATE_ACTIVE) {
916f48ad614SDennis Dalessandro 			dd_dev_info(dd, "Unexpected link state %d\n", hwstate);
917f48ad614SDennis Dalessandro 			return 0;
918f48ad614SDennis Dalessandro 		}
919f48ad614SDennis Dalessandro 
920f48ad614SDennis Dalessandro 		queue_work(rcd->ppd->hfi1_wq, lsaw);
921f48ad614SDennis Dalessandro 		return 1;
922f48ad614SDennis Dalessandro 	}
923f48ad614SDennis Dalessandro 	return 0;
924f48ad614SDennis Dalessandro }
925f48ad614SDennis Dalessandro 
926f48ad614SDennis Dalessandro /*
927f48ad614SDennis Dalessandro  * handle_receive_interrupt - receive a packet
928f48ad614SDennis Dalessandro  * @rcd: the context
929f48ad614SDennis Dalessandro  *
930f48ad614SDennis Dalessandro  * Called from interrupt handler for errors or receive interrupt.
931f48ad614SDennis Dalessandro  * This is the slow path interrupt handler.
932f48ad614SDennis Dalessandro  */
933f48ad614SDennis Dalessandro int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
934f48ad614SDennis Dalessandro {
935f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = rcd->dd;
936f48ad614SDennis Dalessandro 	u32 hdrqtail;
937f48ad614SDennis Dalessandro 	int needset, last = RCV_PKT_OK;
938f48ad614SDennis Dalessandro 	struct hfi1_packet packet;
939f48ad614SDennis Dalessandro 	int skip_pkt = 0;
940f48ad614SDennis Dalessandro 
941f48ad614SDennis Dalessandro 	/* Control context will always use the slow path interrupt handler */
942f48ad614SDennis Dalessandro 	needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
943f48ad614SDennis Dalessandro 
944f48ad614SDennis Dalessandro 	init_packet(rcd, &packet);
945f48ad614SDennis Dalessandro 
946f48ad614SDennis Dalessandro 	if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
947f48ad614SDennis Dalessandro 		u32 seq = rhf_rcv_seq(packet.rhf);
948f48ad614SDennis Dalessandro 
949f48ad614SDennis Dalessandro 		if (seq != rcd->seq_cnt) {
950f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
951f48ad614SDennis Dalessandro 			goto bail;
952f48ad614SDennis Dalessandro 		}
953f48ad614SDennis Dalessandro 		hdrqtail = 0;
954f48ad614SDennis Dalessandro 	} else {
955f48ad614SDennis Dalessandro 		hdrqtail = get_rcvhdrtail(rcd);
956f48ad614SDennis Dalessandro 		if (packet.rhqoff == hdrqtail) {
957f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
958f48ad614SDennis Dalessandro 			goto bail;
959f48ad614SDennis Dalessandro 		}
960f48ad614SDennis Dalessandro 		smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
961f48ad614SDennis Dalessandro 
962f48ad614SDennis Dalessandro 		/*
963f48ad614SDennis Dalessandro 		 * Control context can potentially receive an invalid
964f48ad614SDennis Dalessandro 		 * rhf. Drop such packets.
965f48ad614SDennis Dalessandro 		 */
966f48ad614SDennis Dalessandro 		if (rcd->ctxt == HFI1_CTRL_CTXT) {
967f48ad614SDennis Dalessandro 			u32 seq = rhf_rcv_seq(packet.rhf);
968f48ad614SDennis Dalessandro 
969f48ad614SDennis Dalessandro 			if (seq != rcd->seq_cnt)
970f48ad614SDennis Dalessandro 				skip_pkt = 1;
971f48ad614SDennis Dalessandro 		}
972f48ad614SDennis Dalessandro 	}
973f48ad614SDennis Dalessandro 
974f48ad614SDennis Dalessandro 	prescan_rxq(rcd, &packet);
975f48ad614SDennis Dalessandro 
976f48ad614SDennis Dalessandro 	while (last == RCV_PKT_OK) {
977f48ad614SDennis Dalessandro 		if (unlikely(dd->do_drop &&
978f48ad614SDennis Dalessandro 			     atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
979f48ad614SDennis Dalessandro 			     DROP_PACKET_ON)) {
980f48ad614SDennis Dalessandro 			dd->do_drop = 0;
981f48ad614SDennis Dalessandro 
982f48ad614SDennis Dalessandro 			/* On to the next packet */
983f48ad614SDennis Dalessandro 			packet.rhqoff += packet.rsize;
984f48ad614SDennis Dalessandro 			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
985f48ad614SDennis Dalessandro 					  packet.rhqoff +
986f48ad614SDennis Dalessandro 					  dd->rhf_offset;
987f48ad614SDennis Dalessandro 			packet.rhf = rhf_to_cpu(packet.rhf_addr);
988f48ad614SDennis Dalessandro 
989f48ad614SDennis Dalessandro 		} else if (skip_pkt) {
990f48ad614SDennis Dalessandro 			last = skip_rcv_packet(&packet, thread);
991f48ad614SDennis Dalessandro 			skip_pkt = 0;
992f48ad614SDennis Dalessandro 		} else {
993f48ad614SDennis Dalessandro 			/* Auto activate link on non-SC15 packet receive */
994f48ad614SDennis Dalessandro 			if (unlikely(rcd->ppd->host_link_state ==
995f48ad614SDennis Dalessandro 				     HLS_UP_ARMED) &&
996c867caafSMike Marciniszyn 			    set_armed_to_active(rcd, &packet, dd))
997f48ad614SDennis Dalessandro 				goto bail;
998f48ad614SDennis Dalessandro 			last = process_rcv_packet(&packet, thread);
999f48ad614SDennis Dalessandro 		}
1000f48ad614SDennis Dalessandro 
1001f48ad614SDennis Dalessandro 		if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1002f48ad614SDennis Dalessandro 			u32 seq = rhf_rcv_seq(packet.rhf);
1003f48ad614SDennis Dalessandro 
1004f48ad614SDennis Dalessandro 			if (++rcd->seq_cnt > 13)
1005f48ad614SDennis Dalessandro 				rcd->seq_cnt = 1;
1006f48ad614SDennis Dalessandro 			if (seq != rcd->seq_cnt)
1007f48ad614SDennis Dalessandro 				last = RCV_PKT_DONE;
1008f48ad614SDennis Dalessandro 			if (needset) {
1009f48ad614SDennis Dalessandro 				dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
1010f48ad614SDennis Dalessandro 				set_all_nodma_rtail(dd);
1011f48ad614SDennis Dalessandro 				needset = 0;
1012f48ad614SDennis Dalessandro 			}
1013f48ad614SDennis Dalessandro 		} else {
1014f48ad614SDennis Dalessandro 			if (packet.rhqoff == hdrqtail)
1015f48ad614SDennis Dalessandro 				last = RCV_PKT_DONE;
1016f48ad614SDennis Dalessandro 			/*
1017f48ad614SDennis Dalessandro 			 * Control context can potentially receive an invalid
1018f48ad614SDennis Dalessandro 			 * rhf. Drop such packets.
1019f48ad614SDennis Dalessandro 			 */
1020f48ad614SDennis Dalessandro 			if (rcd->ctxt == HFI1_CTRL_CTXT) {
1021f48ad614SDennis Dalessandro 				u32 seq = rhf_rcv_seq(packet.rhf);
1022f48ad614SDennis Dalessandro 
1023f48ad614SDennis Dalessandro 				if (++rcd->seq_cnt > 13)
1024f48ad614SDennis Dalessandro 					rcd->seq_cnt = 1;
1025f48ad614SDennis Dalessandro 				if (!last && (seq != rcd->seq_cnt))
1026f48ad614SDennis Dalessandro 					skip_pkt = 1;
1027f48ad614SDennis Dalessandro 			}
1028f48ad614SDennis Dalessandro 
1029f48ad614SDennis Dalessandro 			if (needset) {
1030f48ad614SDennis Dalessandro 				dd_dev_info(dd,
1031f48ad614SDennis Dalessandro 					    "Switching to DMA_RTAIL\n");
1032f48ad614SDennis Dalessandro 				set_all_dma_rtail(dd);
1033f48ad614SDennis Dalessandro 				needset = 0;
1034f48ad614SDennis Dalessandro 			}
1035f48ad614SDennis Dalessandro 		}
1036f48ad614SDennis Dalessandro 
1037f48ad614SDennis Dalessandro 		process_rcv_update(last, &packet);
1038f48ad614SDennis Dalessandro 	}
1039f48ad614SDennis Dalessandro 
1040a82a7fcdSMike Marciniszyn 	process_rcv_qp_work(rcd);
1041a82a7fcdSMike Marciniszyn 	rcd->head = packet.rhqoff;
1042f48ad614SDennis Dalessandro 
1043f48ad614SDennis Dalessandro bail:
1044f48ad614SDennis Dalessandro 	/*
1045f48ad614SDennis Dalessandro 	 * Always write head at end, and setup rcv interrupt, even
1046f48ad614SDennis Dalessandro 	 * if no packets were processed.
1047f48ad614SDennis Dalessandro 	 */
1048f48ad614SDennis Dalessandro 	finish_packet(&packet);
1049f48ad614SDennis Dalessandro 	return last;
1050f48ad614SDennis Dalessandro }
1051f48ad614SDennis Dalessandro 
1052f48ad614SDennis Dalessandro /*
1053f48ad614SDennis Dalessandro  * We may discover in the interrupt that the hardware link state has
1054f48ad614SDennis Dalessandro  * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
1055f48ad614SDennis Dalessandro  * and we need to update the driver's notion of the link state.  We cannot
1056f48ad614SDennis Dalessandro  * run set_link_state from interrupt context, so we queue this function on
1057f48ad614SDennis Dalessandro  * a workqueue.
1058f48ad614SDennis Dalessandro  *
1059f48ad614SDennis Dalessandro  * We delay the regular interrupt processing until after the state changes
1060f48ad614SDennis Dalessandro  * so that the link will be in the correct state by the time any application
1061f48ad614SDennis Dalessandro  * we wake up attempts to send a reply to any message it received.
1062f48ad614SDennis Dalessandro  * (Subsequent receive interrupts may possibly force the wakeup before we
1063f48ad614SDennis Dalessandro  * update the link state.)
1064f48ad614SDennis Dalessandro  *
1065f48ad614SDennis Dalessandro  * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes
1066f48ad614SDennis Dalessandro  * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues,
1067f48ad614SDennis Dalessandro  * so we're safe from use-after-free of the rcd.
1068f48ad614SDennis Dalessandro  */
1069f48ad614SDennis Dalessandro void receive_interrupt_work(struct work_struct *work)
1070f48ad614SDennis Dalessandro {
1071f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
1072f48ad614SDennis Dalessandro 						  linkstate_active_work);
1073f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1074f48ad614SDennis Dalessandro 	int i;
1075f48ad614SDennis Dalessandro 
1076f48ad614SDennis Dalessandro 	/* Received non-SC15 packet implies neighbor_normal */
1077f48ad614SDennis Dalessandro 	ppd->neighbor_normal = 1;
1078f48ad614SDennis Dalessandro 	set_link_state(ppd, HLS_UP_ACTIVE);
1079f48ad614SDennis Dalessandro 
1080f48ad614SDennis Dalessandro 	/*
1081f48ad614SDennis Dalessandro 	 * Interrupt all kernel contexts that could have had an
1082f48ad614SDennis Dalessandro 	 * interrupt during auto activation.
1083f48ad614SDennis Dalessandro 	 */
1084f48ad614SDennis Dalessandro 	for (i = HFI1_CTRL_CTXT; i < dd->first_user_ctxt; i++)
1085f48ad614SDennis Dalessandro 		force_recv_intr(dd->rcd[i]);
1086f48ad614SDennis Dalessandro }
1087f48ad614SDennis Dalessandro 
1088f48ad614SDennis Dalessandro /*
1089f48ad614SDennis Dalessandro  * Convert a given MTU size to the on-wire MAD packet enumeration.
1090f48ad614SDennis Dalessandro  * Return -1 if the size is invalid.
1091f48ad614SDennis Dalessandro  */
1092f48ad614SDennis Dalessandro int mtu_to_enum(u32 mtu, int default_if_bad)
1093f48ad614SDennis Dalessandro {
1094f48ad614SDennis Dalessandro 	switch (mtu) {
1095f48ad614SDennis Dalessandro 	case     0: return OPA_MTU_0;
1096f48ad614SDennis Dalessandro 	case   256: return OPA_MTU_256;
1097f48ad614SDennis Dalessandro 	case   512: return OPA_MTU_512;
1098f48ad614SDennis Dalessandro 	case  1024: return OPA_MTU_1024;
1099f48ad614SDennis Dalessandro 	case  2048: return OPA_MTU_2048;
1100f48ad614SDennis Dalessandro 	case  4096: return OPA_MTU_4096;
1101f48ad614SDennis Dalessandro 	case  8192: return OPA_MTU_8192;
1102f48ad614SDennis Dalessandro 	case 10240: return OPA_MTU_10240;
1103f48ad614SDennis Dalessandro 	}
1104f48ad614SDennis Dalessandro 	return default_if_bad;
1105f48ad614SDennis Dalessandro }
1106f48ad614SDennis Dalessandro 
1107f48ad614SDennis Dalessandro u16 enum_to_mtu(int mtu)
1108f48ad614SDennis Dalessandro {
1109f48ad614SDennis Dalessandro 	switch (mtu) {
1110f48ad614SDennis Dalessandro 	case OPA_MTU_0:     return 0;
1111f48ad614SDennis Dalessandro 	case OPA_MTU_256:   return 256;
1112f48ad614SDennis Dalessandro 	case OPA_MTU_512:   return 512;
1113f48ad614SDennis Dalessandro 	case OPA_MTU_1024:  return 1024;
1114f48ad614SDennis Dalessandro 	case OPA_MTU_2048:  return 2048;
1115f48ad614SDennis Dalessandro 	case OPA_MTU_4096:  return 4096;
1116f48ad614SDennis Dalessandro 	case OPA_MTU_8192:  return 8192;
1117f48ad614SDennis Dalessandro 	case OPA_MTU_10240: return 10240;
1118f48ad614SDennis Dalessandro 	default: return 0xffff;
1119f48ad614SDennis Dalessandro 	}
1120f48ad614SDennis Dalessandro }
1121f48ad614SDennis Dalessandro 
1122f48ad614SDennis Dalessandro /*
1123f48ad614SDennis Dalessandro  * set_mtu - set the MTU
1124f48ad614SDennis Dalessandro  * @ppd: the per port data
1125f48ad614SDennis Dalessandro  *
1126f48ad614SDennis Dalessandro  * We can handle "any" incoming size, the issue here is whether we
1127f48ad614SDennis Dalessandro  * need to restrict our outgoing size.  We do not deal with what happens
1128f48ad614SDennis Dalessandro  * to programs that are already running when the size changes.
1129f48ad614SDennis Dalessandro  */
1130f48ad614SDennis Dalessandro int set_mtu(struct hfi1_pportdata *ppd)
1131f48ad614SDennis Dalessandro {
1132f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1133f48ad614SDennis Dalessandro 	int i, drain, ret = 0, is_up = 0;
1134f48ad614SDennis Dalessandro 
1135f48ad614SDennis Dalessandro 	ppd->ibmtu = 0;
1136f48ad614SDennis Dalessandro 	for (i = 0; i < ppd->vls_supported; i++)
1137f48ad614SDennis Dalessandro 		if (ppd->ibmtu < dd->vld[i].mtu)
1138f48ad614SDennis Dalessandro 			ppd->ibmtu = dd->vld[i].mtu;
1139f48ad614SDennis Dalessandro 	ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);
1140f48ad614SDennis Dalessandro 
1141f48ad614SDennis Dalessandro 	mutex_lock(&ppd->hls_lock);
1142f48ad614SDennis Dalessandro 	if (ppd->host_link_state == HLS_UP_INIT ||
1143f48ad614SDennis Dalessandro 	    ppd->host_link_state == HLS_UP_ARMED ||
1144f48ad614SDennis Dalessandro 	    ppd->host_link_state == HLS_UP_ACTIVE)
1145f48ad614SDennis Dalessandro 		is_up = 1;
1146f48ad614SDennis Dalessandro 
1147f48ad614SDennis Dalessandro 	drain = !is_ax(dd) && is_up;
1148f48ad614SDennis Dalessandro 
1149f48ad614SDennis Dalessandro 	if (drain)
1150f48ad614SDennis Dalessandro 		/*
1151f48ad614SDennis Dalessandro 		 * MTU is specified per-VL. To ensure that no packet gets
1152f48ad614SDennis Dalessandro 		 * stuck (due, e.g., to the MTU for the packet's VL being
1153f48ad614SDennis Dalessandro 		 * reduced), empty the per-VL FIFOs before adjusting MTU.
1154f48ad614SDennis Dalessandro 		 */
1155f48ad614SDennis Dalessandro 		ret = stop_drain_data_vls(dd);
1156f48ad614SDennis Dalessandro 
1157f48ad614SDennis Dalessandro 	if (ret) {
1158f48ad614SDennis Dalessandro 		dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
1159f48ad614SDennis Dalessandro 			   __func__);
1160f48ad614SDennis Dalessandro 		goto err;
1161f48ad614SDennis Dalessandro 	}
1162f48ad614SDennis Dalessandro 
1163f48ad614SDennis Dalessandro 	hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);
1164f48ad614SDennis Dalessandro 
1165f48ad614SDennis Dalessandro 	if (drain)
1166f48ad614SDennis Dalessandro 		open_fill_data_vls(dd); /* reopen all VLs */
1167f48ad614SDennis Dalessandro 
1168f48ad614SDennis Dalessandro err:
1169f48ad614SDennis Dalessandro 	mutex_unlock(&ppd->hls_lock);
1170f48ad614SDennis Dalessandro 
1171f48ad614SDennis Dalessandro 	return ret;
1172f48ad614SDennis Dalessandro }
1173f48ad614SDennis Dalessandro 
1174f48ad614SDennis Dalessandro int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
1175f48ad614SDennis Dalessandro {
1176f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1177f48ad614SDennis Dalessandro 
1178f48ad614SDennis Dalessandro 	ppd->lid = lid;
1179f48ad614SDennis Dalessandro 	ppd->lmc = lmc;
1180f48ad614SDennis Dalessandro 	hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);
1181f48ad614SDennis Dalessandro 
1182f48ad614SDennis Dalessandro 	dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
1183f48ad614SDennis Dalessandro 
1184f48ad614SDennis Dalessandro 	return 0;
1185f48ad614SDennis Dalessandro }
1186f48ad614SDennis Dalessandro 
1187f48ad614SDennis Dalessandro void shutdown_led_override(struct hfi1_pportdata *ppd)
1188f48ad614SDennis Dalessandro {
1189f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1190f48ad614SDennis Dalessandro 
1191f48ad614SDennis Dalessandro 	/*
1192f48ad614SDennis Dalessandro 	 * This pairs with the memory barrier in hfi1_start_led_override to
1193f48ad614SDennis Dalessandro 	 * ensure that we read the correct state of LED beaconing represented
1194f48ad614SDennis Dalessandro 	 * by led_override_timer_active
1195f48ad614SDennis Dalessandro 	 */
1196f48ad614SDennis Dalessandro 	smp_rmb();
1197f48ad614SDennis Dalessandro 	if (atomic_read(&ppd->led_override_timer_active)) {
1198f48ad614SDennis Dalessandro 		del_timer_sync(&ppd->led_override_timer);
1199f48ad614SDennis Dalessandro 		atomic_set(&ppd->led_override_timer_active, 0);
1200f48ad614SDennis Dalessandro 		/* Ensure the atomic_set is visible to all CPUs */
1201f48ad614SDennis Dalessandro 		smp_wmb();
1202f48ad614SDennis Dalessandro 	}
1203f48ad614SDennis Dalessandro 
1204f48ad614SDennis Dalessandro 	/* Hand control of the LED to the DC for normal operation */
1205f48ad614SDennis Dalessandro 	write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1206f48ad614SDennis Dalessandro }
1207f48ad614SDennis Dalessandro 
1208f48ad614SDennis Dalessandro static void run_led_override(unsigned long opaque)
1209f48ad614SDennis Dalessandro {
1210f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)opaque;
1211f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1212f48ad614SDennis Dalessandro 	unsigned long timeout;
1213f48ad614SDennis Dalessandro 	int phase_idx;
1214f48ad614SDennis Dalessandro 
1215f48ad614SDennis Dalessandro 	if (!(dd->flags & HFI1_INITTED))
1216f48ad614SDennis Dalessandro 		return;
1217f48ad614SDennis Dalessandro 
1218f48ad614SDennis Dalessandro 	phase_idx = ppd->led_override_phase & 1;
1219f48ad614SDennis Dalessandro 
1220f48ad614SDennis Dalessandro 	setextled(dd, phase_idx);
1221f48ad614SDennis Dalessandro 
1222f48ad614SDennis Dalessandro 	timeout = ppd->led_override_vals[phase_idx];
1223f48ad614SDennis Dalessandro 
1224f48ad614SDennis Dalessandro 	/* Set up for next phase */
1225f48ad614SDennis Dalessandro 	ppd->led_override_phase = !ppd->led_override_phase;
1226f48ad614SDennis Dalessandro 
1227f48ad614SDennis Dalessandro 	mod_timer(&ppd->led_override_timer, jiffies + timeout);
1228f48ad614SDennis Dalessandro }
1229f48ad614SDennis Dalessandro 
1230f48ad614SDennis Dalessandro /*
1231f48ad614SDennis Dalessandro  * To have the LED blink in a particular pattern, provide timeon and timeoff
1232f48ad614SDennis Dalessandro  * in milliseconds.
1233f48ad614SDennis Dalessandro  * To turn off custom blinking and return to normal operation, use
1234f48ad614SDennis Dalessandro  * shutdown_led_override()
1235f48ad614SDennis Dalessandro  */
1236f48ad614SDennis Dalessandro void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
1237f48ad614SDennis Dalessandro 			     unsigned int timeoff)
1238f48ad614SDennis Dalessandro {
1239f48ad614SDennis Dalessandro 	if (!(ppd->dd->flags & HFI1_INITTED))
1240f48ad614SDennis Dalessandro 		return;
1241f48ad614SDennis Dalessandro 
1242f48ad614SDennis Dalessandro 	/* Convert to jiffies for direct use in timer */
1243f48ad614SDennis Dalessandro 	ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
1244f48ad614SDennis Dalessandro 	ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
1245f48ad614SDennis Dalessandro 
1246f48ad614SDennis Dalessandro 	/* Arbitrarily start from LED on phase */
1247f48ad614SDennis Dalessandro 	ppd->led_override_phase = 1;
1248f48ad614SDennis Dalessandro 
1249f48ad614SDennis Dalessandro 	/*
1250f48ad614SDennis Dalessandro 	 * If the timer has not already been started, do so. Use a "quick"
1251f48ad614SDennis Dalessandro 	 * timeout so the handler will be called soon to look at our request.
1252f48ad614SDennis Dalessandro 	 */
1253f48ad614SDennis Dalessandro 	if (!timer_pending(&ppd->led_override_timer)) {
1254f48ad614SDennis Dalessandro 		setup_timer(&ppd->led_override_timer, run_led_override,
1255f48ad614SDennis Dalessandro 			    (unsigned long)ppd);
1256f48ad614SDennis Dalessandro 		ppd->led_override_timer.expires = jiffies + 1;
1257f48ad614SDennis Dalessandro 		add_timer(&ppd->led_override_timer);
1258f48ad614SDennis Dalessandro 		atomic_set(&ppd->led_override_timer_active, 1);
1259f48ad614SDennis Dalessandro 		/* Ensure the atomic_set is visible to all CPUs */
1260f48ad614SDennis Dalessandro 		smp_wmb();
1261f48ad614SDennis Dalessandro 	}
1262f48ad614SDennis Dalessandro }
1263f48ad614SDennis Dalessandro 
1264f48ad614SDennis Dalessandro /**
1265f48ad614SDennis Dalessandro  * hfi1_reset_device - reset the chip if possible
1266f48ad614SDennis Dalessandro  * @unit: the device to reset
1267f48ad614SDennis Dalessandro  *
1268f48ad614SDennis Dalessandro  * Whether or not reset is successful, we attempt to re-initialize the chip
1269f48ad614SDennis Dalessandro  * (that is, much like a driver unload/reload).  We clear the INITTED flag
1270f48ad614SDennis Dalessandro  * so that the various entry points will fail until we reinitialize.  For
1271f48ad614SDennis Dalessandro  * now, we only allow this if no user contexts are open that use chip resources
1272f48ad614SDennis Dalessandro  */
1273f48ad614SDennis Dalessandro int hfi1_reset_device(int unit)
1274f48ad614SDennis Dalessandro {
1275f48ad614SDennis Dalessandro 	int ret, i;
1276f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = hfi1_lookup(unit);
1277f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd;
1278f48ad614SDennis Dalessandro 	unsigned long flags;
1279f48ad614SDennis Dalessandro 	int pidx;
1280f48ad614SDennis Dalessandro 
1281f48ad614SDennis Dalessandro 	if (!dd) {
1282f48ad614SDennis Dalessandro 		ret = -ENODEV;
1283f48ad614SDennis Dalessandro 		goto bail;
1284f48ad614SDennis Dalessandro 	}
1285f48ad614SDennis Dalessandro 
1286f48ad614SDennis Dalessandro 	dd_dev_info(dd, "Reset on unit %u requested\n", unit);
1287f48ad614SDennis Dalessandro 
1288f48ad614SDennis Dalessandro 	if (!dd->kregbase || !(dd->flags & HFI1_PRESENT)) {
1289f48ad614SDennis Dalessandro 		dd_dev_info(dd,
1290f48ad614SDennis Dalessandro 			    "Invalid unit number %u or not initialized or not present\n",
1291f48ad614SDennis Dalessandro 			    unit);
1292f48ad614SDennis Dalessandro 		ret = -ENXIO;
1293f48ad614SDennis Dalessandro 		goto bail;
1294f48ad614SDennis Dalessandro 	}
1295f48ad614SDennis Dalessandro 
1296f48ad614SDennis Dalessandro 	spin_lock_irqsave(&dd->uctxt_lock, flags);
1297f48ad614SDennis Dalessandro 	if (dd->rcd)
1298f48ad614SDennis Dalessandro 		for (i = dd->first_user_ctxt; i < dd->num_rcv_contexts; i++) {
1299f48ad614SDennis Dalessandro 			if (!dd->rcd[i] || !dd->rcd[i]->cnt)
1300f48ad614SDennis Dalessandro 				continue;
1301f48ad614SDennis Dalessandro 			spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1302f48ad614SDennis Dalessandro 			ret = -EBUSY;
1303f48ad614SDennis Dalessandro 			goto bail;
1304f48ad614SDennis Dalessandro 		}
1305f48ad614SDennis Dalessandro 	spin_unlock_irqrestore(&dd->uctxt_lock, flags);
1306f48ad614SDennis Dalessandro 
1307f48ad614SDennis Dalessandro 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1308f48ad614SDennis Dalessandro 		ppd = dd->pport + pidx;
1309f48ad614SDennis Dalessandro 
1310f48ad614SDennis Dalessandro 		shutdown_led_override(ppd);
1311f48ad614SDennis Dalessandro 	}
1312f48ad614SDennis Dalessandro 	if (dd->flags & HFI1_HAS_SEND_DMA)
1313f48ad614SDennis Dalessandro 		sdma_exit(dd);
1314f48ad614SDennis Dalessandro 
1315f48ad614SDennis Dalessandro 	hfi1_reset_cpu_counters(dd);
1316f48ad614SDennis Dalessandro 
1317f48ad614SDennis Dalessandro 	ret = hfi1_init(dd, 1);
1318f48ad614SDennis Dalessandro 
1319f48ad614SDennis Dalessandro 	if (ret)
1320f48ad614SDennis Dalessandro 		dd_dev_err(dd,
1321f48ad614SDennis Dalessandro 			   "Reinitialize unit %u after reset failed with %d\n",
1322f48ad614SDennis Dalessandro 			   unit, ret);
1323f48ad614SDennis Dalessandro 	else
1324f48ad614SDennis Dalessandro 		dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1325f48ad614SDennis Dalessandro 			    unit);
1326f48ad614SDennis Dalessandro 
1327f48ad614SDennis Dalessandro bail:
1328f48ad614SDennis Dalessandro 	return ret;
1329f48ad614SDennis Dalessandro }
1330f48ad614SDennis Dalessandro 
1331f48ad614SDennis Dalessandro void handle_eflags(struct hfi1_packet *packet)
1332f48ad614SDennis Dalessandro {
1333f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd = packet->rcd;
1334f48ad614SDennis Dalessandro 	u32 rte = rhf_rcv_type_err(packet->rhf);
1335f48ad614SDennis Dalessandro 
1336f48ad614SDennis Dalessandro 	rcv_hdrerr(rcd, rcd->ppd, packet);
1337f48ad614SDennis Dalessandro 	if (rhf_err_flags(packet->rhf))
1338f48ad614SDennis Dalessandro 		dd_dev_err(rcd->dd,
1339f48ad614SDennis Dalessandro 			   "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
1340f48ad614SDennis Dalessandro 			   rcd->ctxt, packet->rhf,
1341f48ad614SDennis Dalessandro 			   packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1342f48ad614SDennis Dalessandro 			   packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1343f48ad614SDennis Dalessandro 			   packet->rhf & RHF_DC_ERR ? "dc " : "",
1344f48ad614SDennis Dalessandro 			   packet->rhf & RHF_TID_ERR ? "tid " : "",
1345f48ad614SDennis Dalessandro 			   packet->rhf & RHF_LEN_ERR ? "len " : "",
1346f48ad614SDennis Dalessandro 			   packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1347f48ad614SDennis Dalessandro 			   packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
1348f48ad614SDennis Dalessandro 			   packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1349f48ad614SDennis Dalessandro 			   rte);
1350f48ad614SDennis Dalessandro }
1351f48ad614SDennis Dalessandro 
1352f48ad614SDennis Dalessandro /*
1353f48ad614SDennis Dalessandro  * The following functions are called by the interrupt handler. They are type
1354f48ad614SDennis Dalessandro  * specific handlers for each packet type.
1355f48ad614SDennis Dalessandro  */
1356f48ad614SDennis Dalessandro int process_receive_ib(struct hfi1_packet *packet)
1357f48ad614SDennis Dalessandro {
13580181ce31SDon Hiatt 	if (unlikely(hfi1_dbg_fault_packet(packet)))
13590181ce31SDon Hiatt 		return RHF_RCV_CONTINUE;
13600181ce31SDon Hiatt 
1361f48ad614SDennis Dalessandro 	trace_hfi1_rcvhdr(packet->rcd->ppd->dd,
1362f48ad614SDennis Dalessandro 			  packet->rcd->ctxt,
1363f48ad614SDennis Dalessandro 			  rhf_err_flags(packet->rhf),
1364f48ad614SDennis Dalessandro 			  RHF_RCV_TYPE_IB,
1365f48ad614SDennis Dalessandro 			  packet->hlen,
1366f48ad614SDennis Dalessandro 			  packet->tlen,
1367f48ad614SDennis Dalessandro 			  packet->updegr,
1368f48ad614SDennis Dalessandro 			  rhf_egr_index(packet->rhf));
1369f48ad614SDennis Dalessandro 
1370243d9f43SDon Hiatt 	if (unlikely(
1371243d9f43SDon Hiatt 		 (hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1372243d9f43SDon Hiatt 		 (packet->rhf & RHF_DC_ERR))))
1373243d9f43SDon Hiatt 		return RHF_RCV_CONTINUE;
1374243d9f43SDon Hiatt 
1375f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf))) {
1376f48ad614SDennis Dalessandro 		handle_eflags(packet);
1377f48ad614SDennis Dalessandro 		return RHF_RCV_CONTINUE;
1378f48ad614SDennis Dalessandro 	}
1379f48ad614SDennis Dalessandro 
1380f48ad614SDennis Dalessandro 	hfi1_ib_rcv(packet);
1381f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1382f48ad614SDennis Dalessandro }
1383f48ad614SDennis Dalessandro 
1384f48ad614SDennis Dalessandro int process_receive_bypass(struct hfi1_packet *packet)
1385f48ad614SDennis Dalessandro {
1386505efe3eSJakub Pawlak 	struct hfi1_devdata *dd = packet->rcd->dd;
1387505efe3eSJakub Pawlak 
1388f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1389f48ad614SDennis Dalessandro 		handle_eflags(packet);
1390f48ad614SDennis Dalessandro 
1391505efe3eSJakub Pawlak 	dd_dev_err(dd,
1392f48ad614SDennis Dalessandro 		   "Bypass packets are not supported in normal operation. Dropping\n");
1393505efe3eSJakub Pawlak 	incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
1394505efe3eSJakub Pawlak 	if (!(dd->err_info_rcvport.status_and_code & OPA_EI_STATUS_SMASK)) {
1395505efe3eSJakub Pawlak 		u64 *flits = packet->ebuf;
1396505efe3eSJakub Pawlak 
1397505efe3eSJakub Pawlak 		if (flits && !(packet->rhf & RHF_LEN_ERR)) {
1398505efe3eSJakub Pawlak 			dd->err_info_rcvport.packet_flit1 = flits[0];
1399505efe3eSJakub Pawlak 			dd->err_info_rcvport.packet_flit2 =
1400505efe3eSJakub Pawlak 				packet->tlen > sizeof(flits[0]) ? flits[1] : 0;
1401505efe3eSJakub Pawlak 		}
1402505efe3eSJakub Pawlak 		dd->err_info_rcvport.status_and_code |=
1403505efe3eSJakub Pawlak 			(OPA_EI_STATUS_SMASK | BAD_L2_ERR);
1404505efe3eSJakub Pawlak 	}
1405f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1406f48ad614SDennis Dalessandro }
1407f48ad614SDennis Dalessandro 
1408f48ad614SDennis Dalessandro int process_receive_error(struct hfi1_packet *packet)
1409f48ad614SDennis Dalessandro {
1410243d9f43SDon Hiatt 	/* KHdrHCRCErr -- KDETH packet with a bad HCRC */
1411243d9f43SDon Hiatt 	if (unlikely(
1412243d9f43SDon Hiatt 		 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1413243d9f43SDon Hiatt 		 rhf_rcv_type_err(packet->rhf) == 3))
1414243d9f43SDon Hiatt 		return RHF_RCV_CONTINUE;
1415243d9f43SDon Hiatt 
1416f48ad614SDennis Dalessandro 	handle_eflags(packet);
1417f48ad614SDennis Dalessandro 
1418f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1419f48ad614SDennis Dalessandro 		dd_dev_err(packet->rcd->dd,
1420f48ad614SDennis Dalessandro 			   "Unhandled error packet received. Dropping.\n");
1421f48ad614SDennis Dalessandro 
1422f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1423f48ad614SDennis Dalessandro }
1424f48ad614SDennis Dalessandro 
1425f48ad614SDennis Dalessandro int kdeth_process_expected(struct hfi1_packet *packet)
1426f48ad614SDennis Dalessandro {
14270181ce31SDon Hiatt 	if (unlikely(hfi1_dbg_fault_packet(packet)))
14280181ce31SDon Hiatt 		return RHF_RCV_CONTINUE;
1429f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1430f48ad614SDennis Dalessandro 		handle_eflags(packet);
1431f48ad614SDennis Dalessandro 
1432f48ad614SDennis Dalessandro 	dd_dev_err(packet->rcd->dd,
1433f48ad614SDennis Dalessandro 		   "Unhandled expected packet received. Dropping.\n");
1434f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1435f48ad614SDennis Dalessandro }
1436f48ad614SDennis Dalessandro 
1437f48ad614SDennis Dalessandro int kdeth_process_eager(struct hfi1_packet *packet)
1438f48ad614SDennis Dalessandro {
1439f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1440f48ad614SDennis Dalessandro 		handle_eflags(packet);
14410181ce31SDon Hiatt 	if (unlikely(hfi1_dbg_fault_packet(packet)))
14420181ce31SDon Hiatt 		return RHF_RCV_CONTINUE;
1443f48ad614SDennis Dalessandro 
1444f48ad614SDennis Dalessandro 	dd_dev_err(packet->rcd->dd,
1445f48ad614SDennis Dalessandro 		   "Unhandled eager packet received. Dropping.\n");
1446f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1447f48ad614SDennis Dalessandro }
1448f48ad614SDennis Dalessandro 
1449f48ad614SDennis Dalessandro int process_receive_invalid(struct hfi1_packet *packet)
1450f48ad614SDennis Dalessandro {
1451f48ad614SDennis Dalessandro 	dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1452f48ad614SDennis Dalessandro 		   rhf_rcv_type(packet->rhf));
1453f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1454f48ad614SDennis Dalessandro }
1455