1f48ad614SDennis Dalessandro /*
2d4829ea6SVishwanathapura, Niranjana  * Copyright(c) 2015-2017 Intel Corporation.
3f48ad614SDennis Dalessandro  *
4f48ad614SDennis Dalessandro  * This file is provided under a dual BSD/GPLv2 license.  When using or
5f48ad614SDennis Dalessandro  * redistributing this file, you may do so under either license.
6f48ad614SDennis Dalessandro  *
7f48ad614SDennis Dalessandro  * GPL LICENSE SUMMARY
8f48ad614SDennis Dalessandro  *
9f48ad614SDennis Dalessandro  * This program is free software; you can redistribute it and/or modify
10f48ad614SDennis Dalessandro  * it under the terms of version 2 of the GNU General Public License as
11f48ad614SDennis Dalessandro  * published by the Free Software Foundation.
12f48ad614SDennis Dalessandro  *
13f48ad614SDennis Dalessandro  * This program is distributed in the hope that it will be useful, but
14f48ad614SDennis Dalessandro  * WITHOUT ANY WARRANTY; without even the implied warranty of
15f48ad614SDennis Dalessandro  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16f48ad614SDennis Dalessandro  * General Public License for more details.
17f48ad614SDennis Dalessandro  *
18f48ad614SDennis Dalessandro  * BSD LICENSE
19f48ad614SDennis Dalessandro  *
20f48ad614SDennis Dalessandro  * Redistribution and use in source and binary forms, with or without
21f48ad614SDennis Dalessandro  * modification, are permitted provided that the following conditions
22f48ad614SDennis Dalessandro  * are met:
23f48ad614SDennis Dalessandro  *
24f48ad614SDennis Dalessandro  *  - Redistributions of source code must retain the above copyright
25f48ad614SDennis Dalessandro  *    notice, this list of conditions and the following disclaimer.
26f48ad614SDennis Dalessandro  *  - Redistributions in binary form must reproduce the above copyright
27f48ad614SDennis Dalessandro  *    notice, this list of conditions and the following disclaimer in
28f48ad614SDennis Dalessandro  *    the documentation and/or other materials provided with the
29f48ad614SDennis Dalessandro  *    distribution.
30f48ad614SDennis Dalessandro  *  - Neither the name of Intel Corporation nor the names of its
31f48ad614SDennis Dalessandro  *    contributors may be used to endorse or promote products derived
32f48ad614SDennis Dalessandro  *    from this software without specific prior written permission.
33f48ad614SDennis Dalessandro  *
34f48ad614SDennis Dalessandro  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35f48ad614SDennis Dalessandro  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36f48ad614SDennis Dalessandro  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37f48ad614SDennis Dalessandro  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38f48ad614SDennis Dalessandro  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39f48ad614SDennis Dalessandro  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40f48ad614SDennis Dalessandro  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41f48ad614SDennis Dalessandro  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42f48ad614SDennis Dalessandro  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43f48ad614SDennis Dalessandro  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44f48ad614SDennis Dalessandro  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45f48ad614SDennis Dalessandro  *
46f48ad614SDennis Dalessandro  */
47f48ad614SDennis Dalessandro 
48f48ad614SDennis Dalessandro #include <linux/spinlock.h>
49f48ad614SDennis Dalessandro #include <linux/pci.h>
50f48ad614SDennis Dalessandro #include <linux/io.h>
51f48ad614SDennis Dalessandro #include <linux/delay.h>
52f48ad614SDennis Dalessandro #include <linux/netdevice.h>
53f48ad614SDennis Dalessandro #include <linux/vmalloc.h>
54f48ad614SDennis Dalessandro #include <linux/module.h>
55f48ad614SDennis Dalessandro #include <linux/prefetch.h>
56f48ad614SDennis Dalessandro #include <rdma/ib_verbs.h>
57f48ad614SDennis Dalessandro 
58f48ad614SDennis Dalessandro #include "hfi.h"
59f48ad614SDennis Dalessandro #include "trace.h"
60f48ad614SDennis Dalessandro #include "qp.h"
61f48ad614SDennis Dalessandro #include "sdma.h"
620181ce31SDon Hiatt #include "debugfs.h"
63d4829ea6SVishwanathapura, Niranjana #include "vnic.h"
64f48ad614SDennis Dalessandro 
65f48ad614SDennis Dalessandro #undef pr_fmt
66f48ad614SDennis Dalessandro #define pr_fmt(fmt) DRIVER_NAME ": " fmt
67f48ad614SDennis Dalessandro 
68f48ad614SDennis Dalessandro /*
69f48ad614SDennis Dalessandro  * The size has to be longer than this string, so we can append
70f48ad614SDennis Dalessandro  * board/chip information to it in the initialization code.
71f48ad614SDennis Dalessandro  */
72f48ad614SDennis Dalessandro const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
73f48ad614SDennis Dalessandro 
74f48ad614SDennis Dalessandro DEFINE_SPINLOCK(hfi1_devs_lock);
75f48ad614SDennis Dalessandro LIST_HEAD(hfi1_dev_list);
76f48ad614SDennis Dalessandro DEFINE_MUTEX(hfi1_mutex);	/* general driver use */
77f48ad614SDennis Dalessandro 
78f48ad614SDennis Dalessandro unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
79f48ad614SDennis Dalessandro module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
80f48ad614SDennis Dalessandro MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
81f48ad614SDennis Dalessandro 		 HFI1_DEFAULT_MAX_MTU));
82f48ad614SDennis Dalessandro 
83f48ad614SDennis Dalessandro unsigned int hfi1_cu = 1;
84f48ad614SDennis Dalessandro module_param_named(cu, hfi1_cu, uint, S_IRUGO);
85f48ad614SDennis Dalessandro MODULE_PARM_DESC(cu, "Credit return units");
86f48ad614SDennis Dalessandro 
87f48ad614SDennis Dalessandro unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
88f4cd8765SMichael J. Ruhl static int hfi1_caps_set(const char *val, const struct kernel_param *kp);
89f4cd8765SMichael J. Ruhl static int hfi1_caps_get(char *buffer, const struct kernel_param *kp);
90f48ad614SDennis Dalessandro static const struct kernel_param_ops cap_ops = {
91f48ad614SDennis Dalessandro 	.set = hfi1_caps_set,
92f48ad614SDennis Dalessandro 	.get = hfi1_caps_get
93f48ad614SDennis Dalessandro };
94f48ad614SDennis Dalessandro module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
95f48ad614SDennis Dalessandro MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
96f48ad614SDennis Dalessandro 
97f48ad614SDennis Dalessandro MODULE_LICENSE("Dual BSD/GPL");
98f48ad614SDennis Dalessandro MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
99f48ad614SDennis Dalessandro 
100f48ad614SDennis Dalessandro /*
101f48ad614SDennis Dalessandro  * MAX_PKT_RCV is the max # if packets processed per receive interrupt.
102f48ad614SDennis Dalessandro  */
103f48ad614SDennis Dalessandro #define MAX_PKT_RECV 64
104a82a7fcdSMike Marciniszyn /*
105a82a7fcdSMike Marciniszyn  * MAX_PKT_THREAD_RCV is the max # of packets processed before
106a82a7fcdSMike Marciniszyn  * the qp_wait_list queue is flushed.
107a82a7fcdSMike Marciniszyn  */
108a82a7fcdSMike Marciniszyn #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4)
109f48ad614SDennis Dalessandro #define EGR_HEAD_UPDATE_THRESHOLD 16
110f48ad614SDennis Dalessandro 
111f48ad614SDennis Dalessandro struct hfi1_ib_stats hfi1_stats;
112f48ad614SDennis Dalessandro 
113f48ad614SDennis Dalessandro static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
114f48ad614SDennis Dalessandro {
115f48ad614SDennis Dalessandro 	int ret = 0;
116f48ad614SDennis Dalessandro 	unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
117f48ad614SDennis Dalessandro 		cap_mask = *cap_mask_ptr, value, diff,
118f48ad614SDennis Dalessandro 		write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
119f48ad614SDennis Dalessandro 			      HFI1_CAP_WRITABLE_MASK);
120f48ad614SDennis Dalessandro 
121f48ad614SDennis Dalessandro 	ret = kstrtoul(val, 0, &value);
122f48ad614SDennis Dalessandro 	if (ret) {
123f48ad614SDennis Dalessandro 		pr_warn("Invalid module parameter value for 'cap_mask'\n");
124f48ad614SDennis Dalessandro 		goto done;
125f48ad614SDennis Dalessandro 	}
126f48ad614SDennis Dalessandro 	/* Get the changed bits (except the locked bit) */
127f48ad614SDennis Dalessandro 	diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);
128f48ad614SDennis Dalessandro 
129f48ad614SDennis Dalessandro 	/* Remove any bits that are not allowed to change after driver load */
130f48ad614SDennis Dalessandro 	if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
131f48ad614SDennis Dalessandro 		pr_warn("Ignoring non-writable capability bits %#lx\n",
132f48ad614SDennis Dalessandro 			diff & ~write_mask);
133f48ad614SDennis Dalessandro 		diff &= write_mask;
134f48ad614SDennis Dalessandro 	}
135f48ad614SDennis Dalessandro 
136f48ad614SDennis Dalessandro 	/* Mask off any reserved bits */
137f48ad614SDennis Dalessandro 	diff &= ~HFI1_CAP_RESERVED_MASK;
138f48ad614SDennis Dalessandro 	/* Clear any previously set and changing bits */
139f48ad614SDennis Dalessandro 	cap_mask &= ~diff;
140f48ad614SDennis Dalessandro 	/* Update the bits with the new capability */
141f48ad614SDennis Dalessandro 	cap_mask |= (value & diff);
142f48ad614SDennis Dalessandro 	/* Check for any kernel/user restrictions */
143f48ad614SDennis Dalessandro 	diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
144f48ad614SDennis Dalessandro 		((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
145f48ad614SDennis Dalessandro 	cap_mask &= ~diff;
146f48ad614SDennis Dalessandro 	/* Set the bitmask to the final set */
147f48ad614SDennis Dalessandro 	*cap_mask_ptr = cap_mask;
148f48ad614SDennis Dalessandro done:
149f48ad614SDennis Dalessandro 	return ret;
150f48ad614SDennis Dalessandro }
151f48ad614SDennis Dalessandro 
152f48ad614SDennis Dalessandro static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
153f48ad614SDennis Dalessandro {
154f48ad614SDennis Dalessandro 	unsigned long cap_mask = *(unsigned long *)kp->arg;
155f48ad614SDennis Dalessandro 
156f48ad614SDennis Dalessandro 	cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
157f48ad614SDennis Dalessandro 	cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
158f48ad614SDennis Dalessandro 
159f48ad614SDennis Dalessandro 	return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
160f48ad614SDennis Dalessandro }
161f48ad614SDennis Dalessandro 
162f48ad614SDennis Dalessandro struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
163f48ad614SDennis Dalessandro {
164f48ad614SDennis Dalessandro 	struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
165f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = container_of(ibdev,
166f48ad614SDennis Dalessandro 					       struct hfi1_devdata, verbs_dev);
167f48ad614SDennis Dalessandro 	return dd->pcidev;
168f48ad614SDennis Dalessandro }
169f48ad614SDennis Dalessandro 
170f48ad614SDennis Dalessandro /*
171f48ad614SDennis Dalessandro  * Return count of units with at least one port ACTIVE.
172f48ad614SDennis Dalessandro  */
173f48ad614SDennis Dalessandro int hfi1_count_active_units(void)
174f48ad614SDennis Dalessandro {
175f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd;
176f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd;
177f48ad614SDennis Dalessandro 	unsigned long flags;
178f48ad614SDennis Dalessandro 	int pidx, nunits_active = 0;
179f48ad614SDennis Dalessandro 
180f48ad614SDennis Dalessandro 	spin_lock_irqsave(&hfi1_devs_lock, flags);
181f48ad614SDennis Dalessandro 	list_for_each_entry(dd, &hfi1_dev_list, list) {
182cb51c5d2SMike Marciniszyn 		if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1)
183f48ad614SDennis Dalessandro 			continue;
184f48ad614SDennis Dalessandro 		for (pidx = 0; pidx < dd->num_pports; ++pidx) {
185f48ad614SDennis Dalessandro 			ppd = dd->pport + pidx;
186f48ad614SDennis Dalessandro 			if (ppd->lid && ppd->linkup) {
187f48ad614SDennis Dalessandro 				nunits_active++;
188f48ad614SDennis Dalessandro 				break;
189f48ad614SDennis Dalessandro 			}
190f48ad614SDennis Dalessandro 		}
191f48ad614SDennis Dalessandro 	}
192f48ad614SDennis Dalessandro 	spin_unlock_irqrestore(&hfi1_devs_lock, flags);
193f48ad614SDennis Dalessandro 	return nunits_active;
194f48ad614SDennis Dalessandro }
195f48ad614SDennis Dalessandro 
196f48ad614SDennis Dalessandro /*
197f48ad614SDennis Dalessandro  * Get address of eager buffer from it's index (allocated in chunks, not
198f48ad614SDennis Dalessandro  * contiguous).
199f48ad614SDennis Dalessandro  */
200f48ad614SDennis Dalessandro static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
201f48ad614SDennis Dalessandro 			       u8 *update)
202f48ad614SDennis Dalessandro {
203f48ad614SDennis Dalessandro 	u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);
204f48ad614SDennis Dalessandro 
205f48ad614SDennis Dalessandro 	*update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
206f48ad614SDennis Dalessandro 	return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
207f48ad614SDennis Dalessandro 			(offset * RCV_BUF_BLOCK_SIZE));
208f48ad614SDennis Dalessandro }
209f48ad614SDennis Dalessandro 
2109039746cSDon Hiatt static inline void *hfi1_get_header(struct hfi1_devdata *dd,
2119039746cSDon Hiatt 				    __le32 *rhf_addr)
2129039746cSDon Hiatt {
2139039746cSDon Hiatt 	u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
2149039746cSDon Hiatt 
2159039746cSDon Hiatt 	return (void *)(rhf_addr - dd->rhf_offset + offset);
2169039746cSDon Hiatt }
2179039746cSDon Hiatt 
2189039746cSDon Hiatt static inline struct ib_header *hfi1_get_msgheader(struct hfi1_devdata *dd,
2199039746cSDon Hiatt 						   __le32 *rhf_addr)
2209039746cSDon Hiatt {
2219039746cSDon Hiatt 	return (struct ib_header *)hfi1_get_header(dd, rhf_addr);
2229039746cSDon Hiatt }
2239039746cSDon Hiatt 
22472c07e2bSDon Hiatt static inline struct hfi1_16b_header
22572c07e2bSDon Hiatt 		*hfi1_get_16B_header(struct hfi1_devdata *dd,
22672c07e2bSDon Hiatt 				     __le32 *rhf_addr)
22772c07e2bSDon Hiatt {
22872c07e2bSDon Hiatt 	return (struct hfi1_16b_header *)hfi1_get_header(dd, rhf_addr);
22972c07e2bSDon Hiatt }
23072c07e2bSDon Hiatt 
231f48ad614SDennis Dalessandro /*
232f48ad614SDennis Dalessandro  * Validate and encode the a given RcvArray Buffer size.
233f48ad614SDennis Dalessandro  * The function will check whether the given size falls within
234f48ad614SDennis Dalessandro  * allowed size ranges for the respective type and, optionally,
235f48ad614SDennis Dalessandro  * return the proper encoding.
236f48ad614SDennis Dalessandro  */
237a82a7fcdSMike Marciniszyn int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
238f48ad614SDennis Dalessandro {
239f48ad614SDennis Dalessandro 	if (unlikely(!PAGE_ALIGNED(size)))
240f48ad614SDennis Dalessandro 		return 0;
241f48ad614SDennis Dalessandro 	if (unlikely(size < MIN_EAGER_BUFFER))
242f48ad614SDennis Dalessandro 		return 0;
243f48ad614SDennis Dalessandro 	if (size >
244f48ad614SDennis Dalessandro 	    (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
245f48ad614SDennis Dalessandro 		return 0;
246f48ad614SDennis Dalessandro 	if (encoded)
247f48ad614SDennis Dalessandro 		*encoded = ilog2(size / PAGE_SIZE) + 1;
248f48ad614SDennis Dalessandro 	return 1;
249f48ad614SDennis Dalessandro }
250f48ad614SDennis Dalessandro 
251f48ad614SDennis Dalessandro static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
252f48ad614SDennis Dalessandro 		       struct hfi1_packet *packet)
253f48ad614SDennis Dalessandro {
254261a4351SMike Marciniszyn 	struct ib_header *rhdr = packet->hdr;
255f48ad614SDennis Dalessandro 	u32 rte = rhf_rcv_type_err(packet->rhf);
2565786adf3SDon Hiatt 	u32 mlid_base;
257f3e862cbSSebastian Sanchez 	struct hfi1_ibport *ibp = rcd_to_iport(rcd);
258f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
259aca7f4fcSSebastian Sanchez 	struct hfi1_ibdev *verbs_dev = &dd->verbs_dev;
260aca7f4fcSSebastian Sanchez 	struct rvt_dev_info *rdi = &verbs_dev->rdi;
261aca7f4fcSSebastian Sanchez 
262aca7f4fcSSebastian Sanchez 	if ((packet->rhf & RHF_DC_ERR) &&
263aca7f4fcSSebastian Sanchez 	    hfi1_dbg_fault_suppress_err(verbs_dev))
264aca7f4fcSSebastian Sanchez 		return;
265f48ad614SDennis Dalessandro 
266f48ad614SDennis Dalessandro 	if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR))
267f48ad614SDennis Dalessandro 		return;
268f48ad614SDennis Dalessandro 
2695786adf3SDon Hiatt 	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
2705786adf3SDon Hiatt 		goto drop;
2715786adf3SDon Hiatt 	} else {
2725786adf3SDon Hiatt 		u8 lnh = ib_get_lnh(rhdr);
2735786adf3SDon Hiatt 
2745786adf3SDon Hiatt 		mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE);
2759039746cSDon Hiatt 		if (lnh == HFI1_LRH_BTH) {
2769039746cSDon Hiatt 			packet->ohdr = &rhdr->u.oth;
2779039746cSDon Hiatt 		} else if (lnh == HFI1_LRH_GRH) {
2789039746cSDon Hiatt 			packet->ohdr = &rhdr->u.l.oth;
2799039746cSDon Hiatt 			packet->grh = &rhdr->u.l.grh;
2809039746cSDon Hiatt 		} else {
2819039746cSDon Hiatt 			goto drop;
2829039746cSDon Hiatt 		}
2835786adf3SDon Hiatt 	}
2849039746cSDon Hiatt 
285f48ad614SDennis Dalessandro 	if (packet->rhf & RHF_TID_ERR) {
286f48ad614SDennis Dalessandro 		/* For TIDERR and RC QPs preemptively schedule a NAK */
287f48ad614SDennis Dalessandro 		u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */
2889039746cSDon Hiatt 		u32 dlid = ib_get_dlid(rhdr);
289f48ad614SDennis Dalessandro 		u32 qp_num;
290f48ad614SDennis Dalessandro 
291f48ad614SDennis Dalessandro 		/* Sanity check packet */
292f48ad614SDennis Dalessandro 		if (tlen < 24)
293f48ad614SDennis Dalessandro 			goto drop;
294f48ad614SDennis Dalessandro 
295f48ad614SDennis Dalessandro 		/* Check for GRH */
2965786adf3SDon Hiatt 		if (packet->grh) {
297f48ad614SDennis Dalessandro 			u32 vtf;
2989039746cSDon Hiatt 			struct ib_grh *grh = packet->grh;
299f48ad614SDennis Dalessandro 
3009039746cSDon Hiatt 			if (grh->next_hdr != IB_GRH_NEXT_HDR)
301f48ad614SDennis Dalessandro 				goto drop;
3029039746cSDon Hiatt 			vtf = be32_to_cpu(grh->version_tclass_flow);
303f48ad614SDennis Dalessandro 			if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
304f48ad614SDennis Dalessandro 				goto drop;
305f48ad614SDennis Dalessandro 		}
3069039746cSDon Hiatt 
307f48ad614SDennis Dalessandro 		/* Get the destination QP number. */
3089039746cSDon Hiatt 		qp_num = ib_bth_get_qpn(packet->ohdr);
3099039746cSDon Hiatt 		if (dlid < mlid_base) {
310f48ad614SDennis Dalessandro 			struct rvt_qp *qp;
311f48ad614SDennis Dalessandro 			unsigned long flags;
312f48ad614SDennis Dalessandro 
313f48ad614SDennis Dalessandro 			rcu_read_lock();
314f48ad614SDennis Dalessandro 			qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
315f48ad614SDennis Dalessandro 			if (!qp) {
316f48ad614SDennis Dalessandro 				rcu_read_unlock();
317f48ad614SDennis Dalessandro 				goto drop;
318f48ad614SDennis Dalessandro 			}
319f48ad614SDennis Dalessandro 
320f48ad614SDennis Dalessandro 			/*
321f48ad614SDennis Dalessandro 			 * Handle only RC QPs - for other QP types drop error
322f48ad614SDennis Dalessandro 			 * packet.
323f48ad614SDennis Dalessandro 			 */
324f48ad614SDennis Dalessandro 			spin_lock_irqsave(&qp->r_lock, flags);
325f48ad614SDennis Dalessandro 
326f48ad614SDennis Dalessandro 			/* Check for valid receive state. */
327f48ad614SDennis Dalessandro 			if (!(ib_rvt_state_ops[qp->state] &
328f48ad614SDennis Dalessandro 			      RVT_PROCESS_RECV_OK)) {
329f48ad614SDennis Dalessandro 				ibp->rvp.n_pkt_drops++;
330f48ad614SDennis Dalessandro 			}
331f48ad614SDennis Dalessandro 
332f48ad614SDennis Dalessandro 			switch (qp->ibqp.qp_type) {
333f48ad614SDennis Dalessandro 			case IB_QPT_RC:
3349039746cSDon Hiatt 				hfi1_rc_hdrerr(rcd, packet, qp);
335f48ad614SDennis Dalessandro 				break;
336f48ad614SDennis Dalessandro 			default:
337f48ad614SDennis Dalessandro 				/* For now don't handle any other QP types */
338f48ad614SDennis Dalessandro 				break;
339f48ad614SDennis Dalessandro 			}
340f48ad614SDennis Dalessandro 
341f48ad614SDennis Dalessandro 			spin_unlock_irqrestore(&qp->r_lock, flags);
342f48ad614SDennis Dalessandro 			rcu_read_unlock();
343f48ad614SDennis Dalessandro 		} /* Unicast QP */
344f48ad614SDennis Dalessandro 	} /* Valid packet with TIDErr */
345f48ad614SDennis Dalessandro 
346f48ad614SDennis Dalessandro 	/* handle "RcvTypeErr" flags */
347f48ad614SDennis Dalessandro 	switch (rte) {
348f48ad614SDennis Dalessandro 	case RHF_RTE_ERROR_OP_CODE_ERR:
349f48ad614SDennis Dalessandro 	{
350f48ad614SDennis Dalessandro 		void *ebuf = NULL;
3519039746cSDon Hiatt 		u8 opcode;
352f48ad614SDennis Dalessandro 
353f48ad614SDennis Dalessandro 		if (rhf_use_egr_bfr(packet->rhf))
354f48ad614SDennis Dalessandro 			ebuf = packet->ebuf;
355f48ad614SDennis Dalessandro 
356f48ad614SDennis Dalessandro 		if (!ebuf)
357f48ad614SDennis Dalessandro 			goto drop; /* this should never happen */
358f48ad614SDennis Dalessandro 
3599039746cSDon Hiatt 		opcode = ib_bth_get_opcode(packet->ohdr);
360f48ad614SDennis Dalessandro 		if (opcode == IB_OPCODE_CNP) {
361f48ad614SDennis Dalessandro 			/*
362f48ad614SDennis Dalessandro 			 * Only in pre-B0 h/w is the CNP_OPCODE handled
363f48ad614SDennis Dalessandro 			 * via this code path.
364f48ad614SDennis Dalessandro 			 */
365f48ad614SDennis Dalessandro 			struct rvt_qp *qp = NULL;
366f48ad614SDennis Dalessandro 			u32 lqpn, rqpn;
367f48ad614SDennis Dalessandro 			u16 rlid;
368f48ad614SDennis Dalessandro 			u8 svc_type, sl, sc5;
369f48ad614SDennis Dalessandro 
370aad559c2SDasaratharaman Chandramouli 			sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf);
371f48ad614SDennis Dalessandro 			sl = ibp->sc_to_sl[sc5];
372f48ad614SDennis Dalessandro 
3739039746cSDon Hiatt 			lqpn = ib_bth_get_qpn(packet->ohdr);
374f48ad614SDennis Dalessandro 			rcu_read_lock();
375f48ad614SDennis Dalessandro 			qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
376f48ad614SDennis Dalessandro 			if (!qp) {
377f48ad614SDennis Dalessandro 				rcu_read_unlock();
378f48ad614SDennis Dalessandro 				goto drop;
379f48ad614SDennis Dalessandro 			}
380f48ad614SDennis Dalessandro 
381f48ad614SDennis Dalessandro 			switch (qp->ibqp.qp_type) {
382f48ad614SDennis Dalessandro 			case IB_QPT_UD:
383f48ad614SDennis Dalessandro 				rlid = 0;
384f48ad614SDennis Dalessandro 				rqpn = 0;
385f48ad614SDennis Dalessandro 				svc_type = IB_CC_SVCTYPE_UD;
386f48ad614SDennis Dalessandro 				break;
387f48ad614SDennis Dalessandro 			case IB_QPT_UC:
388cb427057SDon Hiatt 				rlid = ib_get_slid(rhdr);
389f48ad614SDennis Dalessandro 				rqpn = qp->remote_qpn;
390f48ad614SDennis Dalessandro 				svc_type = IB_CC_SVCTYPE_UC;
391f48ad614SDennis Dalessandro 				break;
392f48ad614SDennis Dalessandro 			default:
393f48ad614SDennis Dalessandro 				goto drop;
394f48ad614SDennis Dalessandro 			}
395f48ad614SDennis Dalessandro 
396f48ad614SDennis Dalessandro 			process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
397f48ad614SDennis Dalessandro 			rcu_read_unlock();
398f48ad614SDennis Dalessandro 		}
399f48ad614SDennis Dalessandro 
400f48ad614SDennis Dalessandro 		packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
401f48ad614SDennis Dalessandro 		break;
402f48ad614SDennis Dalessandro 	}
403f48ad614SDennis Dalessandro 	default:
404f48ad614SDennis Dalessandro 		break;
405f48ad614SDennis Dalessandro 	}
406f48ad614SDennis Dalessandro 
407f48ad614SDennis Dalessandro drop:
408f48ad614SDennis Dalessandro 	return;
409f48ad614SDennis Dalessandro }
410f48ad614SDennis Dalessandro 
411f48ad614SDennis Dalessandro static inline void init_packet(struct hfi1_ctxtdata *rcd,
412f48ad614SDennis Dalessandro 			       struct hfi1_packet *packet)
413f48ad614SDennis Dalessandro {
414f48ad614SDennis Dalessandro 	packet->rsize = rcd->rcvhdrqentsize; /* words */
415f48ad614SDennis Dalessandro 	packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */
416f48ad614SDennis Dalessandro 	packet->rcd = rcd;
417f48ad614SDennis Dalessandro 	packet->updegr = 0;
418f48ad614SDennis Dalessandro 	packet->etail = -1;
419f48ad614SDennis Dalessandro 	packet->rhf_addr = get_rhf_addr(rcd);
420f48ad614SDennis Dalessandro 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
421f48ad614SDennis Dalessandro 	packet->rhqoff = rcd->head;
422f48ad614SDennis Dalessandro 	packet->numpkt = 0;
423f48ad614SDennis Dalessandro }
424f48ad614SDennis Dalessandro 
425e2fdbc23SBart Van Assche /* We support only two types - 9B and 16B for now */
426e2fdbc23SBart Van Assche static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = {
427e2fdbc23SBart Van Assche 	[HFI1_PKT_TYPE_9B] = &return_cnp,
428e2fdbc23SBart Van Assche 	[HFI1_PKT_TYPE_16B] = &return_cnp_16B
429e2fdbc23SBart Van Assche };
430e2fdbc23SBart Van Assche 
4315fd2b562SMitko Haralanov void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
4325fd2b562SMitko Haralanov 			       bool do_cnp)
433f48ad614SDennis Dalessandro {
434f48ad614SDennis Dalessandro 	struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
435261a4351SMike Marciniszyn 	struct ib_other_headers *ohdr = pkt->ohdr;
4369039746cSDon Hiatt 	struct ib_grh *grh = pkt->grh;
4375fd2b562SMitko Haralanov 	u32 rqpn = 0, bth1;
43888733e3bSDon Hiatt 	u16 pkey, rlid, dlid = ib_get_dlid(pkt->hdr);
43988733e3bSDon Hiatt 	u8 hdr_type, sc, svc_type;
4405fd2b562SMitko Haralanov 	bool is_mcast = false;
4415fd2b562SMitko Haralanov 
44288733e3bSDon Hiatt 	if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
44388733e3bSDon Hiatt 		is_mcast = hfi1_is_16B_mcast(dlid);
44488733e3bSDon Hiatt 		pkey = hfi1_16B_get_pkey(pkt->hdr);
44588733e3bSDon Hiatt 		sc = hfi1_16B_get_sc(pkt->hdr);
44688733e3bSDon Hiatt 		hdr_type = HFI1_PKT_TYPE_16B;
44788733e3bSDon Hiatt 	} else {
44888733e3bSDon Hiatt 		is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
44988733e3bSDon Hiatt 			   (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
45088733e3bSDon Hiatt 		pkey = ib_bth_get_pkey(ohdr);
45188733e3bSDon Hiatt 		sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf);
45288733e3bSDon Hiatt 		hdr_type = HFI1_PKT_TYPE_9B;
45388733e3bSDon Hiatt 	}
45488733e3bSDon Hiatt 
455f48ad614SDennis Dalessandro 	switch (qp->ibqp.qp_type) {
456f48ad614SDennis Dalessandro 	case IB_QPT_SMI:
457f48ad614SDennis Dalessandro 	case IB_QPT_GSI:
458f48ad614SDennis Dalessandro 	case IB_QPT_UD:
45988733e3bSDon Hiatt 		rlid = ib_get_slid(pkt->hdr);
46088733e3bSDon Hiatt 		rqpn = ib_get_sqpn(pkt->ohdr);
461f48ad614SDennis Dalessandro 		svc_type = IB_CC_SVCTYPE_UD;
462f48ad614SDennis Dalessandro 		break;
463f48ad614SDennis Dalessandro 	case IB_QPT_UC:
464d8966fcdSDasaratharaman Chandramouli 		rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
465f48ad614SDennis Dalessandro 		rqpn = qp->remote_qpn;
466f48ad614SDennis Dalessandro 		svc_type = IB_CC_SVCTYPE_UC;
467f48ad614SDennis Dalessandro 		break;
468f48ad614SDennis Dalessandro 	case IB_QPT_RC:
469d8966fcdSDasaratharaman Chandramouli 		rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
470f48ad614SDennis Dalessandro 		rqpn = qp->remote_qpn;
471f48ad614SDennis Dalessandro 		svc_type = IB_CC_SVCTYPE_RC;
472f48ad614SDennis Dalessandro 		break;
473f48ad614SDennis Dalessandro 	default:
474f48ad614SDennis Dalessandro 		return;
475f48ad614SDennis Dalessandro 	}
476f48ad614SDennis Dalessandro 
4775fd2b562SMitko Haralanov 	bth1 = be32_to_cpu(ohdr->bth[1]);
47888733e3bSDon Hiatt 	/* Call appropriate CNP handler */
47988733e3bSDon Hiatt 	if (do_cnp && (bth1 & IB_FECN_SMASK))
48088733e3bSDon Hiatt 		hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey,
48188733e3bSDon Hiatt 					      dlid, rlid, sc, grh);
482f48ad614SDennis Dalessandro 
4833d591099SDon Hiatt 	if (!is_mcast && (bth1 & IB_BECN_SMASK)) {
484f48ad614SDennis Dalessandro 		struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
485f48ad614SDennis Dalessandro 		u32 lqpn = bth1 & RVT_QPN_MASK;
4865fd2b562SMitko Haralanov 		u8 sl = ibp->sc_to_sl[sc];
487f48ad614SDennis Dalessandro 
488f48ad614SDennis Dalessandro 		process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
489f48ad614SDennis Dalessandro 	}
4905fd2b562SMitko Haralanov 
491f48ad614SDennis Dalessandro }
492f48ad614SDennis Dalessandro 
493f48ad614SDennis Dalessandro struct ps_mdata {
494f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd;
495f48ad614SDennis Dalessandro 	u32 rsize;
496f48ad614SDennis Dalessandro 	u32 maxcnt;
497f48ad614SDennis Dalessandro 	u32 ps_head;
498f48ad614SDennis Dalessandro 	u32 ps_tail;
499f48ad614SDennis Dalessandro 	u32 ps_seq;
500f48ad614SDennis Dalessandro };
501f48ad614SDennis Dalessandro 
502f48ad614SDennis Dalessandro static inline void init_ps_mdata(struct ps_mdata *mdata,
503f48ad614SDennis Dalessandro 				 struct hfi1_packet *packet)
504f48ad614SDennis Dalessandro {
505f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd = packet->rcd;
506f48ad614SDennis Dalessandro 
507f48ad614SDennis Dalessandro 	mdata->rcd = rcd;
508f48ad614SDennis Dalessandro 	mdata->rsize = packet->rsize;
509f48ad614SDennis Dalessandro 	mdata->maxcnt = packet->maxcnt;
510f48ad614SDennis Dalessandro 	mdata->ps_head = packet->rhqoff;
511f48ad614SDennis Dalessandro 
512f48ad614SDennis Dalessandro 	if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
513f48ad614SDennis Dalessandro 		mdata->ps_tail = get_rcvhdrtail(rcd);
514f48ad614SDennis Dalessandro 		if (rcd->ctxt == HFI1_CTRL_CTXT)
515f48ad614SDennis Dalessandro 			mdata->ps_seq = rcd->seq_cnt;
516f48ad614SDennis Dalessandro 		else
517f48ad614SDennis Dalessandro 			mdata->ps_seq = 0; /* not used with DMA_RTAIL */
518f48ad614SDennis Dalessandro 	} else {
519f48ad614SDennis Dalessandro 		mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
520f48ad614SDennis Dalessandro 		mdata->ps_seq = rcd->seq_cnt;
521f48ad614SDennis Dalessandro 	}
522f48ad614SDennis Dalessandro }
523f48ad614SDennis Dalessandro 
524f48ad614SDennis Dalessandro static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
525f48ad614SDennis Dalessandro 			  struct hfi1_ctxtdata *rcd)
526f48ad614SDennis Dalessandro {
527f48ad614SDennis Dalessandro 	if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
528f48ad614SDennis Dalessandro 		return mdata->ps_head == mdata->ps_tail;
529f48ad614SDennis Dalessandro 	return mdata->ps_seq != rhf_rcv_seq(rhf);
530f48ad614SDennis Dalessandro }
531f48ad614SDennis Dalessandro 
532f48ad614SDennis Dalessandro static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
533f48ad614SDennis Dalessandro 			  struct hfi1_ctxtdata *rcd)
534f48ad614SDennis Dalessandro {
535f48ad614SDennis Dalessandro 	/*
536f48ad614SDennis Dalessandro 	 * Control context can potentially receive an invalid rhf.
537f48ad614SDennis Dalessandro 	 * Drop such packets.
538f48ad614SDennis Dalessandro 	 */
539f48ad614SDennis Dalessandro 	if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
540f48ad614SDennis Dalessandro 		return mdata->ps_seq != rhf_rcv_seq(rhf);
541f48ad614SDennis Dalessandro 
542f48ad614SDennis Dalessandro 	return 0;
543f48ad614SDennis Dalessandro }
544f48ad614SDennis Dalessandro 
545f48ad614SDennis Dalessandro static inline void update_ps_mdata(struct ps_mdata *mdata,
546f48ad614SDennis Dalessandro 				   struct hfi1_ctxtdata *rcd)
547f48ad614SDennis Dalessandro {
548f48ad614SDennis Dalessandro 	mdata->ps_head += mdata->rsize;
549f48ad614SDennis Dalessandro 	if (mdata->ps_head >= mdata->maxcnt)
550f48ad614SDennis Dalessandro 		mdata->ps_head = 0;
551f48ad614SDennis Dalessandro 
552f48ad614SDennis Dalessandro 	/* Control context must do seq counting */
553f48ad614SDennis Dalessandro 	if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
554f48ad614SDennis Dalessandro 	    (rcd->ctxt == HFI1_CTRL_CTXT)) {
555f48ad614SDennis Dalessandro 		if (++mdata->ps_seq > 13)
556f48ad614SDennis Dalessandro 			mdata->ps_seq = 1;
557f48ad614SDennis Dalessandro 	}
558f48ad614SDennis Dalessandro }
559f48ad614SDennis Dalessandro 
560f48ad614SDennis Dalessandro /*
561f48ad614SDennis Dalessandro  * prescan_rxq - search through the receive queue looking for packets
562f48ad614SDennis Dalessandro  * containing Excplicit Congestion Notifications (FECNs, or BECNs).
563f48ad614SDennis Dalessandro  * When an ECN is found, process the Congestion Notification, and toggle
564f48ad614SDennis Dalessandro  * it off.
565f48ad614SDennis Dalessandro  * This is declared as a macro to allow quick checking of the port to avoid
566f48ad614SDennis Dalessandro  * the overhead of a function call if not enabled.
567f48ad614SDennis Dalessandro  */
568f48ad614SDennis Dalessandro #define prescan_rxq(rcd, packet) \
569f48ad614SDennis Dalessandro 	do { \
570f48ad614SDennis Dalessandro 		if (rcd->ppd->cc_prescan) \
571f48ad614SDennis Dalessandro 			__prescan_rxq(packet); \
572f48ad614SDennis Dalessandro 	} while (0)
573f48ad614SDennis Dalessandro static void __prescan_rxq(struct hfi1_packet *packet)
574f48ad614SDennis Dalessandro {
575f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd = packet->rcd;
576f48ad614SDennis Dalessandro 	struct ps_mdata mdata;
577f48ad614SDennis Dalessandro 
578f48ad614SDennis Dalessandro 	init_ps_mdata(&mdata, packet);
579f48ad614SDennis Dalessandro 
580f48ad614SDennis Dalessandro 	while (1) {
581f48ad614SDennis Dalessandro 		struct hfi1_devdata *dd = rcd->dd;
582f3e862cbSSebastian Sanchez 		struct hfi1_ibport *ibp = rcd_to_iport(rcd);
583f48ad614SDennis Dalessandro 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
584f48ad614SDennis Dalessandro 					 dd->rhf_offset;
585f48ad614SDennis Dalessandro 		struct rvt_qp *qp;
586261a4351SMike Marciniszyn 		struct ib_header *hdr;
587f48ad614SDennis Dalessandro 		struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
588f48ad614SDennis Dalessandro 		u64 rhf = rhf_to_cpu(rhf_addr);
589f48ad614SDennis Dalessandro 		u32 etype = rhf_rcv_type(rhf), qpn, bth1;
590f48ad614SDennis Dalessandro 		int is_ecn = 0;
591f48ad614SDennis Dalessandro 		u8 lnh;
592f48ad614SDennis Dalessandro 
593f48ad614SDennis Dalessandro 		if (ps_done(&mdata, rhf, rcd))
594f48ad614SDennis Dalessandro 			break;
595f48ad614SDennis Dalessandro 
596f48ad614SDennis Dalessandro 		if (ps_skip(&mdata, rhf, rcd))
597f48ad614SDennis Dalessandro 			goto next;
598f48ad614SDennis Dalessandro 
599f48ad614SDennis Dalessandro 		if (etype != RHF_RCV_TYPE_IB)
600f48ad614SDennis Dalessandro 			goto next;
601f48ad614SDennis Dalessandro 
602f2d8a0b3SDasaratharaman Chandramouli 		packet->hdr = hfi1_get_msgheader(dd, rhf_addr);
603f2d8a0b3SDasaratharaman Chandramouli 		hdr = packet->hdr;
604cb427057SDon Hiatt 		lnh = ib_get_lnh(hdr);
605f48ad614SDennis Dalessandro 
606f2d8a0b3SDasaratharaman Chandramouli 		if (lnh == HFI1_LRH_BTH) {
607f2d8a0b3SDasaratharaman Chandramouli 			packet->ohdr = &hdr->u.oth;
6089039746cSDon Hiatt 			packet->grh = NULL;
609f2d8a0b3SDasaratharaman Chandramouli 		} else if (lnh == HFI1_LRH_GRH) {
610f2d8a0b3SDasaratharaman Chandramouli 			packet->ohdr = &hdr->u.l.oth;
6119039746cSDon Hiatt 			packet->grh = &hdr->u.l.grh;
612f2d8a0b3SDasaratharaman Chandramouli 		} else {
613f48ad614SDennis Dalessandro 			goto next; /* just in case */
614f2d8a0b3SDasaratharaman Chandramouli 		}
6155fd2b562SMitko Haralanov 
616f2d8a0b3SDasaratharaman Chandramouli 		bth1 = be32_to_cpu(packet->ohdr->bth[1]);
6173d591099SDon Hiatt 		is_ecn = !!(bth1 & (IB_FECN_SMASK | IB_BECN_SMASK));
618f48ad614SDennis Dalessandro 
619f48ad614SDennis Dalessandro 		if (!is_ecn)
620f48ad614SDennis Dalessandro 			goto next;
621f48ad614SDennis Dalessandro 
622f48ad614SDennis Dalessandro 		qpn = bth1 & RVT_QPN_MASK;
623f48ad614SDennis Dalessandro 		rcu_read_lock();
624f48ad614SDennis Dalessandro 		qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
625f48ad614SDennis Dalessandro 
626f48ad614SDennis Dalessandro 		if (!qp) {
627f48ad614SDennis Dalessandro 			rcu_read_unlock();
628f48ad614SDennis Dalessandro 			goto next;
629f48ad614SDennis Dalessandro 		}
630f48ad614SDennis Dalessandro 
6315fd2b562SMitko Haralanov 		process_ecn(qp, packet, true);
632f48ad614SDennis Dalessandro 		rcu_read_unlock();
633f48ad614SDennis Dalessandro 
634f48ad614SDennis Dalessandro 		/* turn off BECN, FECN */
6353d591099SDon Hiatt 		bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK);
636f2d8a0b3SDasaratharaman Chandramouli 		packet->ohdr->bth[1] = cpu_to_be32(bth1);
637f48ad614SDennis Dalessandro next:
638f48ad614SDennis Dalessandro 		update_ps_mdata(&mdata, rcd);
639f48ad614SDennis Dalessandro 	}
640f48ad614SDennis Dalessandro }
641f48ad614SDennis Dalessandro 
642bdaf96f6SSebastian Sanchez static void process_rcv_qp_work(struct hfi1_packet *packet)
643a82a7fcdSMike Marciniszyn {
644a82a7fcdSMike Marciniszyn 	struct rvt_qp *qp, *nqp;
645bdaf96f6SSebastian Sanchez 	struct hfi1_ctxtdata *rcd = packet->rcd;
646a82a7fcdSMike Marciniszyn 
647a82a7fcdSMike Marciniszyn 	/*
648a82a7fcdSMike Marciniszyn 	 * Iterate over all QPs waiting to respond.
649a82a7fcdSMike Marciniszyn 	 * The list won't change since the IRQ is only run on one CPU.
650a82a7fcdSMike Marciniszyn 	 */
651a82a7fcdSMike Marciniszyn 	list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
652a82a7fcdSMike Marciniszyn 		list_del_init(&qp->rspwait);
653a82a7fcdSMike Marciniszyn 		if (qp->r_flags & RVT_R_RSP_NAK) {
654a82a7fcdSMike Marciniszyn 			qp->r_flags &= ~RVT_R_RSP_NAK;
655bdaf96f6SSebastian Sanchez 			packet->qp = qp;
656bdaf96f6SSebastian Sanchez 			hfi1_send_rc_ack(packet, 0);
657a82a7fcdSMike Marciniszyn 		}
658a82a7fcdSMike Marciniszyn 		if (qp->r_flags & RVT_R_RSP_SEND) {
659a82a7fcdSMike Marciniszyn 			unsigned long flags;
660a82a7fcdSMike Marciniszyn 
661a82a7fcdSMike Marciniszyn 			qp->r_flags &= ~RVT_R_RSP_SEND;
662a82a7fcdSMike Marciniszyn 			spin_lock_irqsave(&qp->s_lock, flags);
663a82a7fcdSMike Marciniszyn 			if (ib_rvt_state_ops[qp->state] &
664a82a7fcdSMike Marciniszyn 					RVT_PROCESS_OR_FLUSH_SEND)
665a82a7fcdSMike Marciniszyn 				hfi1_schedule_send(qp);
666a82a7fcdSMike Marciniszyn 			spin_unlock_irqrestore(&qp->s_lock, flags);
667a82a7fcdSMike Marciniszyn 		}
668a82a7fcdSMike Marciniszyn 		rvt_put_qp(qp);
669a82a7fcdSMike Marciniszyn 	}
670a82a7fcdSMike Marciniszyn }
671a82a7fcdSMike Marciniszyn 
672a82a7fcdSMike Marciniszyn static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread)
673a82a7fcdSMike Marciniszyn {
674a82a7fcdSMike Marciniszyn 	if (thread) {
675a82a7fcdSMike Marciniszyn 		if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
676a82a7fcdSMike Marciniszyn 			/* allow defered processing */
677bdaf96f6SSebastian Sanchez 			process_rcv_qp_work(packet);
678a82a7fcdSMike Marciniszyn 		cond_resched();
679a82a7fcdSMike Marciniszyn 		return RCV_PKT_OK;
680a82a7fcdSMike Marciniszyn 	} else {
681a82a7fcdSMike Marciniszyn 		this_cpu_inc(*packet->rcd->dd->rcv_limit);
682a82a7fcdSMike Marciniszyn 		return RCV_PKT_LIMIT;
683a82a7fcdSMike Marciniszyn 	}
684a82a7fcdSMike Marciniszyn }
685a82a7fcdSMike Marciniszyn 
686a82a7fcdSMike Marciniszyn static inline int check_max_packet(struct hfi1_packet *packet, int thread)
687f48ad614SDennis Dalessandro {
688f48ad614SDennis Dalessandro 	int ret = RCV_PKT_OK;
689f48ad614SDennis Dalessandro 
690a82a7fcdSMike Marciniszyn 	if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0))
691a82a7fcdSMike Marciniszyn 		ret = max_packet_exceeded(packet, thread);
692a82a7fcdSMike Marciniszyn 	return ret;
693a82a7fcdSMike Marciniszyn }
694a82a7fcdSMike Marciniszyn 
695a82a7fcdSMike Marciniszyn static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
696a82a7fcdSMike Marciniszyn {
697a82a7fcdSMike Marciniszyn 	int ret;
698a82a7fcdSMike Marciniszyn 
699f48ad614SDennis Dalessandro 	/* Set up for the next packet */
700f48ad614SDennis Dalessandro 	packet->rhqoff += packet->rsize;
701f48ad614SDennis Dalessandro 	if (packet->rhqoff >= packet->maxcnt)
702f48ad614SDennis Dalessandro 		packet->rhqoff = 0;
703f48ad614SDennis Dalessandro 
704f48ad614SDennis Dalessandro 	packet->numpkt++;
705a82a7fcdSMike Marciniszyn 	ret = check_max_packet(packet, thread);
706f48ad614SDennis Dalessandro 
707f48ad614SDennis Dalessandro 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
708f48ad614SDennis Dalessandro 				     packet->rcd->dd->rhf_offset;
709f48ad614SDennis Dalessandro 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
710f48ad614SDennis Dalessandro 
711f48ad614SDennis Dalessandro 	return ret;
712f48ad614SDennis Dalessandro }
713f48ad614SDennis Dalessandro 
714f48ad614SDennis Dalessandro static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
715f48ad614SDennis Dalessandro {
716a82a7fcdSMike Marciniszyn 	int ret;
717f48ad614SDennis Dalessandro 
718f48ad614SDennis Dalessandro 	packet->etype = rhf_rcv_type(packet->rhf);
7199039746cSDon Hiatt 
720f48ad614SDennis Dalessandro 	/* total length */
721f48ad614SDennis Dalessandro 	packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
722f48ad614SDennis Dalessandro 	/* retrieve eager buffer details */
723f48ad614SDennis Dalessandro 	packet->ebuf = NULL;
724f48ad614SDennis Dalessandro 	if (rhf_use_egr_bfr(packet->rhf)) {
725f48ad614SDennis Dalessandro 		packet->etail = rhf_egr_index(packet->rhf);
726f48ad614SDennis Dalessandro 		packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
727f48ad614SDennis Dalessandro 				 &packet->updegr);
728f48ad614SDennis Dalessandro 		/*
729f48ad614SDennis Dalessandro 		 * Prefetch the contents of the eager buffer.  It is
730f48ad614SDennis Dalessandro 		 * OK to send a negative length to prefetch_range().
731f48ad614SDennis Dalessandro 		 * The +2 is the size of the RHF.
732f48ad614SDennis Dalessandro 		 */
733f48ad614SDennis Dalessandro 		prefetch_range(packet->ebuf,
734f48ad614SDennis Dalessandro 			       packet->tlen - ((packet->rcd->rcvhdrqentsize -
735f48ad614SDennis Dalessandro 					       (rhf_hdrq_offset(packet->rhf)
736f48ad614SDennis Dalessandro 						+ 2)) * 4));
737f48ad614SDennis Dalessandro 	}
738f48ad614SDennis Dalessandro 
739f48ad614SDennis Dalessandro 	/*
740f48ad614SDennis Dalessandro 	 * Call a type specific handler for the packet. We
741f48ad614SDennis Dalessandro 	 * should be able to trust that etype won't be beyond
742f48ad614SDennis Dalessandro 	 * the range of valid indexes. If so something is really
743f48ad614SDennis Dalessandro 	 * wrong and we can probably just let things come
744f48ad614SDennis Dalessandro 	 * crashing down. There is no need to eat another
745f48ad614SDennis Dalessandro 	 * comparison in this performance critical code.
746f48ad614SDennis Dalessandro 	 */
747f48ad614SDennis Dalessandro 	packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet);
748f48ad614SDennis Dalessandro 	packet->numpkt++;
749f48ad614SDennis Dalessandro 
750f48ad614SDennis Dalessandro 	/* Set up for the next packet */
751f48ad614SDennis Dalessandro 	packet->rhqoff += packet->rsize;
752f48ad614SDennis Dalessandro 	if (packet->rhqoff >= packet->maxcnt)
753f48ad614SDennis Dalessandro 		packet->rhqoff = 0;
754f48ad614SDennis Dalessandro 
755a82a7fcdSMike Marciniszyn 	ret = check_max_packet(packet, thread);
756f48ad614SDennis Dalessandro 
757f48ad614SDennis Dalessandro 	packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
758f48ad614SDennis Dalessandro 				      packet->rcd->dd->rhf_offset;
759f48ad614SDennis Dalessandro 	packet->rhf = rhf_to_cpu(packet->rhf_addr);
760f48ad614SDennis Dalessandro 
761f48ad614SDennis Dalessandro 	return ret;
762f48ad614SDennis Dalessandro }
763f48ad614SDennis Dalessandro 
764f48ad614SDennis Dalessandro static inline void process_rcv_update(int last, struct hfi1_packet *packet)
765f48ad614SDennis Dalessandro {
766f48ad614SDennis Dalessandro 	/*
767f48ad614SDennis Dalessandro 	 * Update head regs etc., every 16 packets, if not last pkt,
768f48ad614SDennis Dalessandro 	 * to help prevent rcvhdrq overflows, when many packets
769f48ad614SDennis Dalessandro 	 * are processed and queue is nearly full.
770f48ad614SDennis Dalessandro 	 * Don't request an interrupt for intermediate updates.
771f48ad614SDennis Dalessandro 	 */
772f48ad614SDennis Dalessandro 	if (!last && !(packet->numpkt & 0xf)) {
773f48ad614SDennis Dalessandro 		update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
774f48ad614SDennis Dalessandro 			       packet->etail, 0, 0);
775f48ad614SDennis Dalessandro 		packet->updegr = 0;
776f48ad614SDennis Dalessandro 	}
7779039746cSDon Hiatt 	packet->grh = NULL;
778f48ad614SDennis Dalessandro }
779f48ad614SDennis Dalessandro 
780f48ad614SDennis Dalessandro static inline void finish_packet(struct hfi1_packet *packet)
781f48ad614SDennis Dalessandro {
782f48ad614SDennis Dalessandro 	/*
783f48ad614SDennis Dalessandro 	 * Nothing we need to free for the packet.
784f48ad614SDennis Dalessandro 	 *
785f48ad614SDennis Dalessandro 	 * The only thing we need to do is a final update and call for an
786f48ad614SDennis Dalessandro 	 * interrupt
787f48ad614SDennis Dalessandro 	 */
788f48ad614SDennis Dalessandro 	update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
789f48ad614SDennis Dalessandro 		       packet->etail, rcv_intr_dynamic, packet->numpkt);
790f48ad614SDennis Dalessandro }
791f48ad614SDennis Dalessandro 
792f48ad614SDennis Dalessandro /*
793f48ad614SDennis Dalessandro  * Handle receive interrupts when using the no dma rtail option.
794f48ad614SDennis Dalessandro  */
795f48ad614SDennis Dalessandro int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
796f48ad614SDennis Dalessandro {
797f48ad614SDennis Dalessandro 	u32 seq;
798f48ad614SDennis Dalessandro 	int last = RCV_PKT_OK;
799f48ad614SDennis Dalessandro 	struct hfi1_packet packet;
800f48ad614SDennis Dalessandro 
801f48ad614SDennis Dalessandro 	init_packet(rcd, &packet);
802f48ad614SDennis Dalessandro 	seq = rhf_rcv_seq(packet.rhf);
803f48ad614SDennis Dalessandro 	if (seq != rcd->seq_cnt) {
804f48ad614SDennis Dalessandro 		last = RCV_PKT_DONE;
805f48ad614SDennis Dalessandro 		goto bail;
806f48ad614SDennis Dalessandro 	}
807f48ad614SDennis Dalessandro 
808f48ad614SDennis Dalessandro 	prescan_rxq(rcd, &packet);
809f48ad614SDennis Dalessandro 
810f48ad614SDennis Dalessandro 	while (last == RCV_PKT_OK) {
811f48ad614SDennis Dalessandro 		last = process_rcv_packet(&packet, thread);
812f48ad614SDennis Dalessandro 		seq = rhf_rcv_seq(packet.rhf);
813f48ad614SDennis Dalessandro 		if (++rcd->seq_cnt > 13)
814f48ad614SDennis Dalessandro 			rcd->seq_cnt = 1;
815f48ad614SDennis Dalessandro 		if (seq != rcd->seq_cnt)
816f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
817f48ad614SDennis Dalessandro 		process_rcv_update(last, &packet);
818f48ad614SDennis Dalessandro 	}
819bdaf96f6SSebastian Sanchez 	process_rcv_qp_work(&packet);
820a82a7fcdSMike Marciniszyn 	rcd->head = packet.rhqoff;
821f48ad614SDennis Dalessandro bail:
822f48ad614SDennis Dalessandro 	finish_packet(&packet);
823f48ad614SDennis Dalessandro 	return last;
824f48ad614SDennis Dalessandro }
825f48ad614SDennis Dalessandro 
826f48ad614SDennis Dalessandro int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
827f48ad614SDennis Dalessandro {
828f48ad614SDennis Dalessandro 	u32 hdrqtail;
829f48ad614SDennis Dalessandro 	int last = RCV_PKT_OK;
830f48ad614SDennis Dalessandro 	struct hfi1_packet packet;
831f48ad614SDennis Dalessandro 
832f48ad614SDennis Dalessandro 	init_packet(rcd, &packet);
833f48ad614SDennis Dalessandro 	hdrqtail = get_rcvhdrtail(rcd);
834f48ad614SDennis Dalessandro 	if (packet.rhqoff == hdrqtail) {
835f48ad614SDennis Dalessandro 		last = RCV_PKT_DONE;
836f48ad614SDennis Dalessandro 		goto bail;
837f48ad614SDennis Dalessandro 	}
838f48ad614SDennis Dalessandro 	smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
839f48ad614SDennis Dalessandro 
840f48ad614SDennis Dalessandro 	prescan_rxq(rcd, &packet);
841f48ad614SDennis Dalessandro 
842f48ad614SDennis Dalessandro 	while (last == RCV_PKT_OK) {
843f48ad614SDennis Dalessandro 		last = process_rcv_packet(&packet, thread);
844f48ad614SDennis Dalessandro 		if (packet.rhqoff == hdrqtail)
845f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
846f48ad614SDennis Dalessandro 		process_rcv_update(last, &packet);
847f48ad614SDennis Dalessandro 	}
848bdaf96f6SSebastian Sanchez 	process_rcv_qp_work(&packet);
849a82a7fcdSMike Marciniszyn 	rcd->head = packet.rhqoff;
850f48ad614SDennis Dalessandro bail:
851f48ad614SDennis Dalessandro 	finish_packet(&packet);
852f48ad614SDennis Dalessandro 	return last;
853f48ad614SDennis Dalessandro }
854f48ad614SDennis Dalessandro 
855e6f7622dSMichael J. Ruhl static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)
856f48ad614SDennis Dalessandro {
857d295dbebSMichael J. Ruhl 	struct hfi1_ctxtdata *rcd;
858e6f7622dSMichael J. Ruhl 	u16 i;
859f48ad614SDennis Dalessandro 
8602280740fSVishwanathapura, Niranjana 	/*
8612280740fSVishwanathapura, Niranjana 	 * For dynamically allocated kernel contexts (like vnic) switch
8622280740fSVishwanathapura, Niranjana 	 * interrupt handler only for that context. Otherwise, switch
8632280740fSVishwanathapura, Niranjana 	 * interrupt handler for all statically allocated kernel contexts.
8642280740fSVishwanathapura, Niranjana 	 */
8652280740fSVishwanathapura, Niranjana 	if (ctxt >= dd->first_dyn_alloc_ctxt) {
866d59075adSMichael J. Ruhl 		rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
867d295dbebSMichael J. Ruhl 		if (rcd) {
868d295dbebSMichael J. Ruhl 			rcd->do_interrupt =
8692280740fSVishwanathapura, Niranjana 				&handle_receive_interrupt_nodma_rtail;
870d295dbebSMichael J. Ruhl 			hfi1_rcd_put(rcd);
871d295dbebSMichael J. Ruhl 		}
8722280740fSVishwanathapura, Niranjana 		return;
8732280740fSVishwanathapura, Niranjana 	}
8742280740fSVishwanathapura, Niranjana 
875d295dbebSMichael J. Ruhl 	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
876d295dbebSMichael J. Ruhl 		rcd = hfi1_rcd_get_by_index(dd, i);
877d295dbebSMichael J. Ruhl 		if (rcd)
878d295dbebSMichael J. Ruhl 			rcd->do_interrupt =
879f48ad614SDennis Dalessandro 				&handle_receive_interrupt_nodma_rtail;
880d295dbebSMichael J. Ruhl 		hfi1_rcd_put(rcd);
881d295dbebSMichael J. Ruhl 	}
882f48ad614SDennis Dalessandro }
883f48ad614SDennis Dalessandro 
884e6f7622dSMichael J. Ruhl static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt)
885f48ad614SDennis Dalessandro {
886d295dbebSMichael J. Ruhl 	struct hfi1_ctxtdata *rcd;
887e6f7622dSMichael J. Ruhl 	u16 i;
888f48ad614SDennis Dalessandro 
8892280740fSVishwanathapura, Niranjana 	/*
8902280740fSVishwanathapura, Niranjana 	 * For dynamically allocated kernel contexts (like vnic) switch
8912280740fSVishwanathapura, Niranjana 	 * interrupt handler only for that context. Otherwise, switch
8922280740fSVishwanathapura, Niranjana 	 * interrupt handler for all statically allocated kernel contexts.
8932280740fSVishwanathapura, Niranjana 	 */
8942280740fSVishwanathapura, Niranjana 	if (ctxt >= dd->first_dyn_alloc_ctxt) {
895d59075adSMichael J. Ruhl 		rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
896d295dbebSMichael J. Ruhl 		if (rcd) {
897d295dbebSMichael J. Ruhl 			rcd->do_interrupt =
8982280740fSVishwanathapura, Niranjana 				&handle_receive_interrupt_dma_rtail;
899d295dbebSMichael J. Ruhl 			hfi1_rcd_put(rcd);
900d295dbebSMichael J. Ruhl 		}
9012280740fSVishwanathapura, Niranjana 		return;
9022280740fSVishwanathapura, Niranjana 	}
9032280740fSVishwanathapura, Niranjana 
904d295dbebSMichael J. Ruhl 	for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
905d295dbebSMichael J. Ruhl 		rcd = hfi1_rcd_get_by_index(dd, i);
906d295dbebSMichael J. Ruhl 		if (rcd)
907d295dbebSMichael J. Ruhl 			rcd->do_interrupt =
908f48ad614SDennis Dalessandro 				&handle_receive_interrupt_dma_rtail;
909d295dbebSMichael J. Ruhl 		hfi1_rcd_put(rcd);
910d295dbebSMichael J. Ruhl 	}
911f48ad614SDennis Dalessandro }
912f48ad614SDennis Dalessandro 
913f48ad614SDennis Dalessandro void set_all_slowpath(struct hfi1_devdata *dd)
914f48ad614SDennis Dalessandro {
915d295dbebSMichael J. Ruhl 	struct hfi1_ctxtdata *rcd;
916e6f7622dSMichael J. Ruhl 	u16 i;
917f48ad614SDennis Dalessandro 
918f48ad614SDennis Dalessandro 	/* HFI1_CTRL_CTXT must always use the slow path interrupt handler */
9192280740fSVishwanathapura, Niranjana 	for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
920d295dbebSMichael J. Ruhl 		rcd = hfi1_rcd_get_by_index(dd, i);
921d295dbebSMichael J. Ruhl 		if (!rcd)
922d295dbebSMichael J. Ruhl 			continue;
923cc9a97eaSNiranjana Vishwanathapura 		if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)
9242280740fSVishwanathapura, Niranjana 			rcd->do_interrupt = &handle_receive_interrupt;
925cc9a97eaSNiranjana Vishwanathapura 
926d295dbebSMichael J. Ruhl 		hfi1_rcd_put(rcd);
927d295dbebSMichael J. Ruhl 	}
928f48ad614SDennis Dalessandro }
929f48ad614SDennis Dalessandro 
930f48ad614SDennis Dalessandro static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
931c867caafSMike Marciniszyn 				      struct hfi1_packet *packet,
932f48ad614SDennis Dalessandro 				      struct hfi1_devdata *dd)
933f48ad614SDennis Dalessandro {
934f48ad614SDennis Dalessandro 	struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
9359039746cSDon Hiatt 	u8 etype = rhf_rcv_type(packet->rhf);
9369039746cSDon Hiatt 	u8 sc = SC15_PACKET;
9379039746cSDon Hiatt 
9389039746cSDon Hiatt 	if (etype == RHF_RCV_TYPE_IB) {
939261a4351SMike Marciniszyn 		struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd,
940c867caafSMike Marciniszyn 							   packet->rhf_addr);
9419039746cSDon Hiatt 		sc = hfi1_9B_get_sc5(hdr, packet->rhf);
94272c07e2bSDon Hiatt 	} else if (etype == RHF_RCV_TYPE_BYPASS) {
94372c07e2bSDon Hiatt 		struct hfi1_16b_header *hdr = hfi1_get_16B_header(
94472c07e2bSDon Hiatt 						packet->rcd->dd,
94572c07e2bSDon Hiatt 						packet->rhf_addr);
94672c07e2bSDon Hiatt 		sc = hfi1_16B_get_sc(hdr);
9479039746cSDon Hiatt 	}
9489039746cSDon Hiatt 	if (sc != SC15_PACKET) {
949bec7c79cSByczkowski, Jakub 		int hwstate = driver_lstate(rcd->ppd);
950f48ad614SDennis Dalessandro 
951bec7c79cSByczkowski, Jakub 		if (hwstate != IB_PORT_ACTIVE) {
952bec7c79cSByczkowski, Jakub 			dd_dev_info(dd,
953bec7c79cSByczkowski, Jakub 				    "Unexpected link state %s\n",
954bec7c79cSByczkowski, Jakub 				    opa_lstate_name(hwstate));
955f48ad614SDennis Dalessandro 			return 0;
956f48ad614SDennis Dalessandro 		}
957f48ad614SDennis Dalessandro 
95871d47008SSebastian Sanchez 		queue_work(rcd->ppd->link_wq, lsaw);
959f48ad614SDennis Dalessandro 		return 1;
960f48ad614SDennis Dalessandro 	}
961f48ad614SDennis Dalessandro 	return 0;
962f48ad614SDennis Dalessandro }
963f48ad614SDennis Dalessandro 
964f48ad614SDennis Dalessandro /*
965f48ad614SDennis Dalessandro  * handle_receive_interrupt - receive a packet
966f48ad614SDennis Dalessandro  * @rcd: the context
967f48ad614SDennis Dalessandro  *
968f48ad614SDennis Dalessandro  * Called from interrupt handler for errors or receive interrupt.
969f48ad614SDennis Dalessandro  * This is the slow path interrupt handler.
970f48ad614SDennis Dalessandro  */
971f48ad614SDennis Dalessandro int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
972f48ad614SDennis Dalessandro {
973f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = rcd->dd;
974f48ad614SDennis Dalessandro 	u32 hdrqtail;
975f48ad614SDennis Dalessandro 	int needset, last = RCV_PKT_OK;
976f48ad614SDennis Dalessandro 	struct hfi1_packet packet;
977f48ad614SDennis Dalessandro 	int skip_pkt = 0;
978f48ad614SDennis Dalessandro 
979f48ad614SDennis Dalessandro 	/* Control context will always use the slow path interrupt handler */
980f48ad614SDennis Dalessandro 	needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
981f48ad614SDennis Dalessandro 
982f48ad614SDennis Dalessandro 	init_packet(rcd, &packet);
983f48ad614SDennis Dalessandro 
984f48ad614SDennis Dalessandro 	if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
985f48ad614SDennis Dalessandro 		u32 seq = rhf_rcv_seq(packet.rhf);
986f48ad614SDennis Dalessandro 
987f48ad614SDennis Dalessandro 		if (seq != rcd->seq_cnt) {
988f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
989f48ad614SDennis Dalessandro 			goto bail;
990f48ad614SDennis Dalessandro 		}
991f48ad614SDennis Dalessandro 		hdrqtail = 0;
992f48ad614SDennis Dalessandro 	} else {
993f48ad614SDennis Dalessandro 		hdrqtail = get_rcvhdrtail(rcd);
994f48ad614SDennis Dalessandro 		if (packet.rhqoff == hdrqtail) {
995f48ad614SDennis Dalessandro 			last = RCV_PKT_DONE;
996f48ad614SDennis Dalessandro 			goto bail;
997f48ad614SDennis Dalessandro 		}
998f48ad614SDennis Dalessandro 		smp_rmb();  /* prevent speculative reads of dma'ed hdrq */
999f48ad614SDennis Dalessandro 
1000f48ad614SDennis Dalessandro 		/*
1001f48ad614SDennis Dalessandro 		 * Control context can potentially receive an invalid
1002f48ad614SDennis Dalessandro 		 * rhf. Drop such packets.
1003f48ad614SDennis Dalessandro 		 */
1004f48ad614SDennis Dalessandro 		if (rcd->ctxt == HFI1_CTRL_CTXT) {
1005f48ad614SDennis Dalessandro 			u32 seq = rhf_rcv_seq(packet.rhf);
1006f48ad614SDennis Dalessandro 
1007f48ad614SDennis Dalessandro 			if (seq != rcd->seq_cnt)
1008f48ad614SDennis Dalessandro 				skip_pkt = 1;
1009f48ad614SDennis Dalessandro 		}
1010f48ad614SDennis Dalessandro 	}
1011f48ad614SDennis Dalessandro 
1012f48ad614SDennis Dalessandro 	prescan_rxq(rcd, &packet);
1013f48ad614SDennis Dalessandro 
1014f48ad614SDennis Dalessandro 	while (last == RCV_PKT_OK) {
1015f48ad614SDennis Dalessandro 		if (unlikely(dd->do_drop &&
1016f48ad614SDennis Dalessandro 			     atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
1017f48ad614SDennis Dalessandro 			     DROP_PACKET_ON)) {
1018f48ad614SDennis Dalessandro 			dd->do_drop = 0;
1019f48ad614SDennis Dalessandro 
1020f48ad614SDennis Dalessandro 			/* On to the next packet */
1021f48ad614SDennis Dalessandro 			packet.rhqoff += packet.rsize;
1022f48ad614SDennis Dalessandro 			packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
1023f48ad614SDennis Dalessandro 					  packet.rhqoff +
1024f48ad614SDennis Dalessandro 					  dd->rhf_offset;
1025f48ad614SDennis Dalessandro 			packet.rhf = rhf_to_cpu(packet.rhf_addr);
1026f48ad614SDennis Dalessandro 
1027f48ad614SDennis Dalessandro 		} else if (skip_pkt) {
1028f48ad614SDennis Dalessandro 			last = skip_rcv_packet(&packet, thread);
1029f48ad614SDennis Dalessandro 			skip_pkt = 0;
1030f48ad614SDennis Dalessandro 		} else {
1031f48ad614SDennis Dalessandro 			/* Auto activate link on non-SC15 packet receive */
1032f48ad614SDennis Dalessandro 			if (unlikely(rcd->ppd->host_link_state ==
1033f48ad614SDennis Dalessandro 				     HLS_UP_ARMED) &&
1034c867caafSMike Marciniszyn 			    set_armed_to_active(rcd, &packet, dd))
1035f48ad614SDennis Dalessandro 				goto bail;
1036f48ad614SDennis Dalessandro 			last = process_rcv_packet(&packet, thread);
1037f48ad614SDennis Dalessandro 		}
1038f48ad614SDennis Dalessandro 
1039f48ad614SDennis Dalessandro 		if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1040f48ad614SDennis Dalessandro 			u32 seq = rhf_rcv_seq(packet.rhf);
1041f48ad614SDennis Dalessandro 
1042f48ad614SDennis Dalessandro 			if (++rcd->seq_cnt > 13)
1043f48ad614SDennis Dalessandro 				rcd->seq_cnt = 1;
1044f48ad614SDennis Dalessandro 			if (seq != rcd->seq_cnt)
1045f48ad614SDennis Dalessandro 				last = RCV_PKT_DONE;
1046f48ad614SDennis Dalessandro 			if (needset) {
1047f48ad614SDennis Dalessandro 				dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
10482280740fSVishwanathapura, Niranjana 				set_nodma_rtail(dd, rcd->ctxt);
1049f48ad614SDennis Dalessandro 				needset = 0;
1050f48ad614SDennis Dalessandro 			}
1051f48ad614SDennis Dalessandro 		} else {
1052f48ad614SDennis Dalessandro 			if (packet.rhqoff == hdrqtail)
1053f48ad614SDennis Dalessandro 				last = RCV_PKT_DONE;
1054f48ad614SDennis Dalessandro 			/*
1055f48ad614SDennis Dalessandro 			 * Control context can potentially receive an invalid
1056f48ad614SDennis Dalessandro 			 * rhf. Drop such packets.
1057f48ad614SDennis Dalessandro 			 */
1058f48ad614SDennis Dalessandro 			if (rcd->ctxt == HFI1_CTRL_CTXT) {
1059f48ad614SDennis Dalessandro 				u32 seq = rhf_rcv_seq(packet.rhf);
1060f48ad614SDennis Dalessandro 
1061f48ad614SDennis Dalessandro 				if (++rcd->seq_cnt > 13)
1062f48ad614SDennis Dalessandro 					rcd->seq_cnt = 1;
1063f48ad614SDennis Dalessandro 				if (!last && (seq != rcd->seq_cnt))
1064f48ad614SDennis Dalessandro 					skip_pkt = 1;
1065f48ad614SDennis Dalessandro 			}
1066f48ad614SDennis Dalessandro 
1067f48ad614SDennis Dalessandro 			if (needset) {
1068f48ad614SDennis Dalessandro 				dd_dev_info(dd,
1069f48ad614SDennis Dalessandro 					    "Switching to DMA_RTAIL\n");
10702280740fSVishwanathapura, Niranjana 				set_dma_rtail(dd, rcd->ctxt);
1071f48ad614SDennis Dalessandro 				needset = 0;
1072f48ad614SDennis Dalessandro 			}
1073f48ad614SDennis Dalessandro 		}
1074f48ad614SDennis Dalessandro 
1075f48ad614SDennis Dalessandro 		process_rcv_update(last, &packet);
1076f48ad614SDennis Dalessandro 	}
1077f48ad614SDennis Dalessandro 
1078bdaf96f6SSebastian Sanchez 	process_rcv_qp_work(&packet);
1079a82a7fcdSMike Marciniszyn 	rcd->head = packet.rhqoff;
1080f48ad614SDennis Dalessandro 
1081f48ad614SDennis Dalessandro bail:
1082f48ad614SDennis Dalessandro 	/*
1083f48ad614SDennis Dalessandro 	 * Always write head at end, and setup rcv interrupt, even
1084f48ad614SDennis Dalessandro 	 * if no packets were processed.
1085f48ad614SDennis Dalessandro 	 */
1086f48ad614SDennis Dalessandro 	finish_packet(&packet);
1087f48ad614SDennis Dalessandro 	return last;
1088f48ad614SDennis Dalessandro }
1089f48ad614SDennis Dalessandro 
1090f48ad614SDennis Dalessandro /*
1091f48ad614SDennis Dalessandro  * We may discover in the interrupt that the hardware link state has
1092f48ad614SDennis Dalessandro  * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet),
1093f48ad614SDennis Dalessandro  * and we need to update the driver's notion of the link state.  We cannot
1094f48ad614SDennis Dalessandro  * run set_link_state from interrupt context, so we queue this function on
1095f48ad614SDennis Dalessandro  * a workqueue.
1096f48ad614SDennis Dalessandro  *
1097f48ad614SDennis Dalessandro  * We delay the regular interrupt processing until after the state changes
1098f48ad614SDennis Dalessandro  * so that the link will be in the correct state by the time any application
1099f48ad614SDennis Dalessandro  * we wake up attempts to send a reply to any message it received.
1100f48ad614SDennis Dalessandro  * (Subsequent receive interrupts may possibly force the wakeup before we
1101f48ad614SDennis Dalessandro  * update the link state.)
1102f48ad614SDennis Dalessandro  *
1103f48ad614SDennis Dalessandro  * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes
1104f48ad614SDennis Dalessandro  * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues,
1105f48ad614SDennis Dalessandro  * so we're safe from use-after-free of the rcd.
1106f48ad614SDennis Dalessandro  */
1107f48ad614SDennis Dalessandro void receive_interrupt_work(struct work_struct *work)
1108f48ad614SDennis Dalessandro {
1109f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
1110f48ad614SDennis Dalessandro 						  linkstate_active_work);
1111f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1112d295dbebSMichael J. Ruhl 	struct hfi1_ctxtdata *rcd;
1113e6f7622dSMichael J. Ruhl 	u16 i;
1114f48ad614SDennis Dalessandro 
1115f48ad614SDennis Dalessandro 	/* Received non-SC15 packet implies neighbor_normal */
1116f48ad614SDennis Dalessandro 	ppd->neighbor_normal = 1;
1117f48ad614SDennis Dalessandro 	set_link_state(ppd, HLS_UP_ACTIVE);
1118f48ad614SDennis Dalessandro 
1119f48ad614SDennis Dalessandro 	/*
11202280740fSVishwanathapura, Niranjana 	 * Interrupt all statically allocated kernel contexts that could
11212280740fSVishwanathapura, Niranjana 	 * have had an interrupt during auto activation.
1122f48ad614SDennis Dalessandro 	 */
1123d295dbebSMichael J. Ruhl 	for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) {
1124d295dbebSMichael J. Ruhl 		rcd = hfi1_rcd_get_by_index(dd, i);
1125d295dbebSMichael J. Ruhl 		if (rcd)
1126d295dbebSMichael J. Ruhl 			force_recv_intr(rcd);
1127d295dbebSMichael J. Ruhl 		hfi1_rcd_put(rcd);
1128d295dbebSMichael J. Ruhl 	}
1129f48ad614SDennis Dalessandro }
1130f48ad614SDennis Dalessandro 
1131f48ad614SDennis Dalessandro /*
1132f48ad614SDennis Dalessandro  * Convert a given MTU size to the on-wire MAD packet enumeration.
1133f48ad614SDennis Dalessandro  * Return -1 if the size is invalid.
1134f48ad614SDennis Dalessandro  */
1135f48ad614SDennis Dalessandro int mtu_to_enum(u32 mtu, int default_if_bad)
1136f48ad614SDennis Dalessandro {
1137f48ad614SDennis Dalessandro 	switch (mtu) {
1138f48ad614SDennis Dalessandro 	case     0: return OPA_MTU_0;
1139f48ad614SDennis Dalessandro 	case   256: return OPA_MTU_256;
1140f48ad614SDennis Dalessandro 	case   512: return OPA_MTU_512;
1141f48ad614SDennis Dalessandro 	case  1024: return OPA_MTU_1024;
1142f48ad614SDennis Dalessandro 	case  2048: return OPA_MTU_2048;
1143f48ad614SDennis Dalessandro 	case  4096: return OPA_MTU_4096;
1144f48ad614SDennis Dalessandro 	case  8192: return OPA_MTU_8192;
1145f48ad614SDennis Dalessandro 	case 10240: return OPA_MTU_10240;
1146f48ad614SDennis Dalessandro 	}
1147f48ad614SDennis Dalessandro 	return default_if_bad;
1148f48ad614SDennis Dalessandro }
1149f48ad614SDennis Dalessandro 
1150f48ad614SDennis Dalessandro u16 enum_to_mtu(int mtu)
1151f48ad614SDennis Dalessandro {
1152f48ad614SDennis Dalessandro 	switch (mtu) {
1153f48ad614SDennis Dalessandro 	case OPA_MTU_0:     return 0;
1154f48ad614SDennis Dalessandro 	case OPA_MTU_256:   return 256;
1155f48ad614SDennis Dalessandro 	case OPA_MTU_512:   return 512;
1156f48ad614SDennis Dalessandro 	case OPA_MTU_1024:  return 1024;
1157f48ad614SDennis Dalessandro 	case OPA_MTU_2048:  return 2048;
1158f48ad614SDennis Dalessandro 	case OPA_MTU_4096:  return 4096;
1159f48ad614SDennis Dalessandro 	case OPA_MTU_8192:  return 8192;
1160f48ad614SDennis Dalessandro 	case OPA_MTU_10240: return 10240;
1161f48ad614SDennis Dalessandro 	default: return 0xffff;
1162f48ad614SDennis Dalessandro 	}
1163f48ad614SDennis Dalessandro }
1164f48ad614SDennis Dalessandro 
1165f48ad614SDennis Dalessandro /*
1166f48ad614SDennis Dalessandro  * set_mtu - set the MTU
1167f48ad614SDennis Dalessandro  * @ppd: the per port data
1168f48ad614SDennis Dalessandro  *
1169f48ad614SDennis Dalessandro  * We can handle "any" incoming size, the issue here is whether we
1170f48ad614SDennis Dalessandro  * need to restrict our outgoing size.  We do not deal with what happens
1171f48ad614SDennis Dalessandro  * to programs that are already running when the size changes.
1172f48ad614SDennis Dalessandro  */
1173f48ad614SDennis Dalessandro int set_mtu(struct hfi1_pportdata *ppd)
1174f48ad614SDennis Dalessandro {
1175f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1176f48ad614SDennis Dalessandro 	int i, drain, ret = 0, is_up = 0;
1177f48ad614SDennis Dalessandro 
1178f48ad614SDennis Dalessandro 	ppd->ibmtu = 0;
1179f48ad614SDennis Dalessandro 	for (i = 0; i < ppd->vls_supported; i++)
1180f48ad614SDennis Dalessandro 		if (ppd->ibmtu < dd->vld[i].mtu)
1181f48ad614SDennis Dalessandro 			ppd->ibmtu = dd->vld[i].mtu;
1182f48ad614SDennis Dalessandro 	ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);
1183f48ad614SDennis Dalessandro 
1184f48ad614SDennis Dalessandro 	mutex_lock(&ppd->hls_lock);
1185f48ad614SDennis Dalessandro 	if (ppd->host_link_state == HLS_UP_INIT ||
1186f48ad614SDennis Dalessandro 	    ppd->host_link_state == HLS_UP_ARMED ||
1187f48ad614SDennis Dalessandro 	    ppd->host_link_state == HLS_UP_ACTIVE)
1188f48ad614SDennis Dalessandro 		is_up = 1;
1189f48ad614SDennis Dalessandro 
1190f48ad614SDennis Dalessandro 	drain = !is_ax(dd) && is_up;
1191f48ad614SDennis Dalessandro 
1192f48ad614SDennis Dalessandro 	if (drain)
1193f48ad614SDennis Dalessandro 		/*
1194f48ad614SDennis Dalessandro 		 * MTU is specified per-VL. To ensure that no packet gets
1195f48ad614SDennis Dalessandro 		 * stuck (due, e.g., to the MTU for the packet's VL being
1196f48ad614SDennis Dalessandro 		 * reduced), empty the per-VL FIFOs before adjusting MTU.
1197f48ad614SDennis Dalessandro 		 */
1198f48ad614SDennis Dalessandro 		ret = stop_drain_data_vls(dd);
1199f48ad614SDennis Dalessandro 
1200f48ad614SDennis Dalessandro 	if (ret) {
1201f48ad614SDennis Dalessandro 		dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
1202f48ad614SDennis Dalessandro 			   __func__);
1203f48ad614SDennis Dalessandro 		goto err;
1204f48ad614SDennis Dalessandro 	}
1205f48ad614SDennis Dalessandro 
1206f48ad614SDennis Dalessandro 	hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);
1207f48ad614SDennis Dalessandro 
1208f48ad614SDennis Dalessandro 	if (drain)
1209f48ad614SDennis Dalessandro 		open_fill_data_vls(dd); /* reopen all VLs */
1210f48ad614SDennis Dalessandro 
1211f48ad614SDennis Dalessandro err:
1212f48ad614SDennis Dalessandro 	mutex_unlock(&ppd->hls_lock);
1213f48ad614SDennis Dalessandro 
1214f48ad614SDennis Dalessandro 	return ret;
1215f48ad614SDennis Dalessandro }
1216f48ad614SDennis Dalessandro 
1217f48ad614SDennis Dalessandro int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
1218f48ad614SDennis Dalessandro {
1219f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1220f48ad614SDennis Dalessandro 
1221f48ad614SDennis Dalessandro 	ppd->lid = lid;
1222f48ad614SDennis Dalessandro 	ppd->lmc = lmc;
1223f48ad614SDennis Dalessandro 	hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);
1224f48ad614SDennis Dalessandro 
1225f48ad614SDennis Dalessandro 	dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
1226f48ad614SDennis Dalessandro 
1227f48ad614SDennis Dalessandro 	return 0;
1228f48ad614SDennis Dalessandro }
1229f48ad614SDennis Dalessandro 
1230f48ad614SDennis Dalessandro void shutdown_led_override(struct hfi1_pportdata *ppd)
1231f48ad614SDennis Dalessandro {
1232f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1233f48ad614SDennis Dalessandro 
1234f48ad614SDennis Dalessandro 	/*
1235f48ad614SDennis Dalessandro 	 * This pairs with the memory barrier in hfi1_start_led_override to
1236f48ad614SDennis Dalessandro 	 * ensure that we read the correct state of LED beaconing represented
1237f48ad614SDennis Dalessandro 	 * by led_override_timer_active
1238f48ad614SDennis Dalessandro 	 */
1239f48ad614SDennis Dalessandro 	smp_rmb();
1240f48ad614SDennis Dalessandro 	if (atomic_read(&ppd->led_override_timer_active)) {
1241f48ad614SDennis Dalessandro 		del_timer_sync(&ppd->led_override_timer);
1242f48ad614SDennis Dalessandro 		atomic_set(&ppd->led_override_timer_active, 0);
1243f48ad614SDennis Dalessandro 		/* Ensure the atomic_set is visible to all CPUs */
1244f48ad614SDennis Dalessandro 		smp_wmb();
1245f48ad614SDennis Dalessandro 	}
1246f48ad614SDennis Dalessandro 
1247f48ad614SDennis Dalessandro 	/* Hand control of the LED to the DC for normal operation */
1248f48ad614SDennis Dalessandro 	write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1249f48ad614SDennis Dalessandro }
1250f48ad614SDennis Dalessandro 
12518064135eSKees Cook static void run_led_override(struct timer_list *t)
1252f48ad614SDennis Dalessandro {
12538064135eSKees Cook 	struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer);
1254f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = ppd->dd;
1255f48ad614SDennis Dalessandro 	unsigned long timeout;
1256f48ad614SDennis Dalessandro 	int phase_idx;
1257f48ad614SDennis Dalessandro 
1258f48ad614SDennis Dalessandro 	if (!(dd->flags & HFI1_INITTED))
1259f48ad614SDennis Dalessandro 		return;
1260f48ad614SDennis Dalessandro 
1261f48ad614SDennis Dalessandro 	phase_idx = ppd->led_override_phase & 1;
1262f48ad614SDennis Dalessandro 
1263f48ad614SDennis Dalessandro 	setextled(dd, phase_idx);
1264f48ad614SDennis Dalessandro 
1265f48ad614SDennis Dalessandro 	timeout = ppd->led_override_vals[phase_idx];
1266f48ad614SDennis Dalessandro 
1267f48ad614SDennis Dalessandro 	/* Set up for next phase */
1268f48ad614SDennis Dalessandro 	ppd->led_override_phase = !ppd->led_override_phase;
1269f48ad614SDennis Dalessandro 
1270f48ad614SDennis Dalessandro 	mod_timer(&ppd->led_override_timer, jiffies + timeout);
1271f48ad614SDennis Dalessandro }
1272f48ad614SDennis Dalessandro 
1273f48ad614SDennis Dalessandro /*
1274f48ad614SDennis Dalessandro  * To have the LED blink in a particular pattern, provide timeon and timeoff
1275f48ad614SDennis Dalessandro  * in milliseconds.
1276f48ad614SDennis Dalessandro  * To turn off custom blinking and return to normal operation, use
1277f48ad614SDennis Dalessandro  * shutdown_led_override()
1278f48ad614SDennis Dalessandro  */
1279f48ad614SDennis Dalessandro void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
1280f48ad614SDennis Dalessandro 			     unsigned int timeoff)
1281f48ad614SDennis Dalessandro {
1282f48ad614SDennis Dalessandro 	if (!(ppd->dd->flags & HFI1_INITTED))
1283f48ad614SDennis Dalessandro 		return;
1284f48ad614SDennis Dalessandro 
1285f48ad614SDennis Dalessandro 	/* Convert to jiffies for direct use in timer */
1286f48ad614SDennis Dalessandro 	ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
1287f48ad614SDennis Dalessandro 	ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
1288f48ad614SDennis Dalessandro 
1289f48ad614SDennis Dalessandro 	/* Arbitrarily start from LED on phase */
1290f48ad614SDennis Dalessandro 	ppd->led_override_phase = 1;
1291f48ad614SDennis Dalessandro 
1292f48ad614SDennis Dalessandro 	/*
1293f48ad614SDennis Dalessandro 	 * If the timer has not already been started, do so. Use a "quick"
1294f48ad614SDennis Dalessandro 	 * timeout so the handler will be called soon to look at our request.
1295f48ad614SDennis Dalessandro 	 */
1296f48ad614SDennis Dalessandro 	if (!timer_pending(&ppd->led_override_timer)) {
12978064135eSKees Cook 		timer_setup(&ppd->led_override_timer, run_led_override, 0);
1298f48ad614SDennis Dalessandro 		ppd->led_override_timer.expires = jiffies + 1;
1299f48ad614SDennis Dalessandro 		add_timer(&ppd->led_override_timer);
1300f48ad614SDennis Dalessandro 		atomic_set(&ppd->led_override_timer_active, 1);
1301f48ad614SDennis Dalessandro 		/* Ensure the atomic_set is visible to all CPUs */
1302f48ad614SDennis Dalessandro 		smp_wmb();
1303f48ad614SDennis Dalessandro 	}
1304f48ad614SDennis Dalessandro }
1305f48ad614SDennis Dalessandro 
1306f48ad614SDennis Dalessandro /**
1307f48ad614SDennis Dalessandro  * hfi1_reset_device - reset the chip if possible
1308f48ad614SDennis Dalessandro  * @unit: the device to reset
1309f48ad614SDennis Dalessandro  *
1310f48ad614SDennis Dalessandro  * Whether or not reset is successful, we attempt to re-initialize the chip
1311f48ad614SDennis Dalessandro  * (that is, much like a driver unload/reload).  We clear the INITTED flag
1312f48ad614SDennis Dalessandro  * so that the various entry points will fail until we reinitialize.  For
1313f48ad614SDennis Dalessandro  * now, we only allow this if no user contexts are open that use chip resources
1314f48ad614SDennis Dalessandro  */
1315f48ad614SDennis Dalessandro int hfi1_reset_device(int unit)
1316f48ad614SDennis Dalessandro {
1317e6f7622dSMichael J. Ruhl 	int ret;
1318f48ad614SDennis Dalessandro 	struct hfi1_devdata *dd = hfi1_lookup(unit);
1319f48ad614SDennis Dalessandro 	struct hfi1_pportdata *ppd;
1320f48ad614SDennis Dalessandro 	int pidx;
1321f48ad614SDennis Dalessandro 
1322f48ad614SDennis Dalessandro 	if (!dd) {
1323f48ad614SDennis Dalessandro 		ret = -ENODEV;
1324f48ad614SDennis Dalessandro 		goto bail;
1325f48ad614SDennis Dalessandro 	}
1326f48ad614SDennis Dalessandro 
1327f48ad614SDennis Dalessandro 	dd_dev_info(dd, "Reset on unit %u requested\n", unit);
1328f48ad614SDennis Dalessandro 
1329cb51c5d2SMike Marciniszyn 	if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) {
1330f48ad614SDennis Dalessandro 		dd_dev_info(dd,
1331f48ad614SDennis Dalessandro 			    "Invalid unit number %u or not initialized or not present\n",
1332f48ad614SDennis Dalessandro 			    unit);
1333f48ad614SDennis Dalessandro 		ret = -ENXIO;
1334f48ad614SDennis Dalessandro 		goto bail;
1335f48ad614SDennis Dalessandro 	}
1336f48ad614SDennis Dalessandro 
1337d295dbebSMichael J. Ruhl 	/* If there are any user/vnic contexts, we cannot reset */
1338d295dbebSMichael J. Ruhl 	mutex_lock(&hfi1_mutex);
1339f48ad614SDennis Dalessandro 	if (dd->rcd)
1340d295dbebSMichael J. Ruhl 		if (hfi1_stats.sps_ctxts) {
1341d295dbebSMichael J. Ruhl 			mutex_unlock(&hfi1_mutex);
1342f48ad614SDennis Dalessandro 			ret = -EBUSY;
1343f48ad614SDennis Dalessandro 			goto bail;
1344f48ad614SDennis Dalessandro 		}
1345d295dbebSMichael J. Ruhl 	mutex_unlock(&hfi1_mutex);
1346f48ad614SDennis Dalessandro 
1347f48ad614SDennis Dalessandro 	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1348f48ad614SDennis Dalessandro 		ppd = dd->pport + pidx;
1349f48ad614SDennis Dalessandro 
1350f48ad614SDennis Dalessandro 		shutdown_led_override(ppd);
1351f48ad614SDennis Dalessandro 	}
1352f48ad614SDennis Dalessandro 	if (dd->flags & HFI1_HAS_SEND_DMA)
1353f48ad614SDennis Dalessandro 		sdma_exit(dd);
1354f48ad614SDennis Dalessandro 
1355f48ad614SDennis Dalessandro 	hfi1_reset_cpu_counters(dd);
1356f48ad614SDennis Dalessandro 
1357f48ad614SDennis Dalessandro 	ret = hfi1_init(dd, 1);
1358f48ad614SDennis Dalessandro 
1359f48ad614SDennis Dalessandro 	if (ret)
1360f48ad614SDennis Dalessandro 		dd_dev_err(dd,
1361f48ad614SDennis Dalessandro 			   "Reinitialize unit %u after reset failed with %d\n",
1362f48ad614SDennis Dalessandro 			   unit, ret);
1363f48ad614SDennis Dalessandro 	else
1364f48ad614SDennis Dalessandro 		dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1365f48ad614SDennis Dalessandro 			    unit);
1366f48ad614SDennis Dalessandro 
1367f48ad614SDennis Dalessandro bail:
1368f48ad614SDennis Dalessandro 	return ret;
1369f48ad614SDennis Dalessandro }
1370f48ad614SDennis Dalessandro 
13719039746cSDon Hiatt static inline void hfi1_setup_ib_header(struct hfi1_packet *packet)
13729039746cSDon Hiatt {
13739039746cSDon Hiatt 	packet->hdr = (struct hfi1_ib_message_header *)
13749039746cSDon Hiatt 			hfi1_get_msgheader(packet->rcd->dd,
13759039746cSDon Hiatt 					   packet->rhf_addr);
13769039746cSDon Hiatt 	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
13779039746cSDon Hiatt }
13789039746cSDon Hiatt 
13795786adf3SDon Hiatt static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet)
13805786adf3SDon Hiatt {
13815786adf3SDon Hiatt 	struct hfi1_pportdata *ppd = packet->rcd->ppd;
13825786adf3SDon Hiatt 
13835786adf3SDon Hiatt 	/* slid and dlid cannot be 0 */
13845786adf3SDon Hiatt 	if ((!packet->slid) || (!packet->dlid))
13855786adf3SDon Hiatt 		return -EINVAL;
13865786adf3SDon Hiatt 
13875786adf3SDon Hiatt 	/* Compare port lid with incoming packet dlid */
13885786adf3SDon Hiatt 	if ((!(hfi1_is_16B_mcast(packet->dlid))) &&
13895786adf3SDon Hiatt 	    (packet->dlid !=
13905786adf3SDon Hiatt 		opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) {
13915786adf3SDon Hiatt 		if (packet->dlid != ppd->lid)
13925786adf3SDon Hiatt 			return -EINVAL;
13935786adf3SDon Hiatt 	}
13945786adf3SDon Hiatt 
13955786adf3SDon Hiatt 	/* No multicast packets with SC15 */
13965786adf3SDon Hiatt 	if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF))
13975786adf3SDon Hiatt 		return -EINVAL;
13985786adf3SDon Hiatt 
13995786adf3SDon Hiatt 	/* Packets with permissive DLID always on SC15 */
14005786adf3SDon Hiatt 	if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE),
14015786adf3SDon Hiatt 					 16B)) &&
14025786adf3SDon Hiatt 	    (packet->sc != 0xF))
14035786adf3SDon Hiatt 		return -EINVAL;
14045786adf3SDon Hiatt 
14055786adf3SDon Hiatt 	return 0;
14065786adf3SDon Hiatt }
14075786adf3SDon Hiatt 
14089039746cSDon Hiatt static int hfi1_setup_9B_packet(struct hfi1_packet *packet)
14099039746cSDon Hiatt {
14109039746cSDon Hiatt 	struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
14119039746cSDon Hiatt 	struct ib_header *hdr;
14129039746cSDon Hiatt 	u8 lnh;
14139039746cSDon Hiatt 
14149039746cSDon Hiatt 	hfi1_setup_ib_header(packet);
14159039746cSDon Hiatt 	hdr = packet->hdr;
14169039746cSDon Hiatt 
14179039746cSDon Hiatt 	lnh = ib_get_lnh(hdr);
14189039746cSDon Hiatt 	if (lnh == HFI1_LRH_BTH) {
14199039746cSDon Hiatt 		packet->ohdr = &hdr->u.oth;
14209039746cSDon Hiatt 		packet->grh = NULL;
14219039746cSDon Hiatt 	} else if (lnh == HFI1_LRH_GRH) {
14229039746cSDon Hiatt 		u32 vtf;
14239039746cSDon Hiatt 
14249039746cSDon Hiatt 		packet->ohdr = &hdr->u.l.oth;
14259039746cSDon Hiatt 		packet->grh = &hdr->u.l.grh;
14269039746cSDon Hiatt 		if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
14279039746cSDon Hiatt 			goto drop;
14289039746cSDon Hiatt 		vtf = be32_to_cpu(packet->grh->version_tclass_flow);
14299039746cSDon Hiatt 		if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
14309039746cSDon Hiatt 			goto drop;
14319039746cSDon Hiatt 	} else {
14329039746cSDon Hiatt 		goto drop;
14339039746cSDon Hiatt 	}
14349039746cSDon Hiatt 
14359039746cSDon Hiatt 	/* Query commonly used fields from packet header */
143672c07e2bSDon Hiatt 	packet->payload = packet->ebuf;
14379039746cSDon Hiatt 	packet->opcode = ib_bth_get_opcode(packet->ohdr);
14389039746cSDon Hiatt 	packet->slid = ib_get_slid(hdr);
14399039746cSDon Hiatt 	packet->dlid = ib_get_dlid(hdr);
144072c07e2bSDon Hiatt 	if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
144172c07e2bSDon Hiatt 		     (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE))))
144272c07e2bSDon Hiatt 		packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
144372c07e2bSDon Hiatt 				be16_to_cpu(IB_MULTICAST_LID_BASE);
14449039746cSDon Hiatt 	packet->sl = ib_get_sl(hdr);
14459039746cSDon Hiatt 	packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf);
14469039746cSDon Hiatt 	packet->pad = ib_bth_get_pad(packet->ohdr);
14479039746cSDon Hiatt 	packet->extra_byte = 0;
14486d6b8848SSebastian Sanchez 	packet->pkey = ib_bth_get_pkey(packet->ohdr);
14496d6b8848SSebastian Sanchez 	packet->migrated = ib_bth_is_migration(packet->ohdr);
14509039746cSDon Hiatt 
14519039746cSDon Hiatt 	return 0;
14529039746cSDon Hiatt drop:
14539039746cSDon Hiatt 	ibp->rvp.n_pkt_drops++;
14549039746cSDon Hiatt 	return -EINVAL;
14559039746cSDon Hiatt }
14569039746cSDon Hiatt 
145772c07e2bSDon Hiatt static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
145872c07e2bSDon Hiatt {
145972c07e2bSDon Hiatt 	/*
146072c07e2bSDon Hiatt 	 * Bypass packets have a different header/payload split
146172c07e2bSDon Hiatt 	 * compared to an IB packet.
146272c07e2bSDon Hiatt 	 * Current split is set such that 16 bytes of the actual
146372c07e2bSDon Hiatt 	 * header is in the header buffer and the remining is in
146472c07e2bSDon Hiatt 	 * the eager buffer. We chose 16 since hfi1 driver only
146572c07e2bSDon Hiatt 	 * supports 16B bypass packets and we will be able to
146672c07e2bSDon Hiatt 	 * receive the entire LRH with such a split.
146772c07e2bSDon Hiatt 	 */
146872c07e2bSDon Hiatt 
146972c07e2bSDon Hiatt 	struct hfi1_ctxtdata *rcd = packet->rcd;
147072c07e2bSDon Hiatt 	struct hfi1_pportdata *ppd = rcd->ppd;
147172c07e2bSDon Hiatt 	struct hfi1_ibport *ibp = &ppd->ibport_data;
147272c07e2bSDon Hiatt 	u8 l4;
147372c07e2bSDon Hiatt 	u8 grh_len;
147472c07e2bSDon Hiatt 
147572c07e2bSDon Hiatt 	packet->hdr = (struct hfi1_16b_header *)
147672c07e2bSDon Hiatt 			hfi1_get_16B_header(packet->rcd->dd,
147772c07e2bSDon Hiatt 					    packet->rhf_addr);
147872c07e2bSDon Hiatt 	packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
147972c07e2bSDon Hiatt 
148072c07e2bSDon Hiatt 	l4 = hfi1_16B_get_l4(packet->hdr);
148172c07e2bSDon Hiatt 	if (l4 == OPA_16B_L4_IB_LOCAL) {
148272c07e2bSDon Hiatt 		grh_len = 0;
148372c07e2bSDon Hiatt 		packet->ohdr = packet->ebuf;
148472c07e2bSDon Hiatt 		packet->grh = NULL;
148572c07e2bSDon Hiatt 	} else if (l4 == OPA_16B_L4_IB_GLOBAL) {
148672c07e2bSDon Hiatt 		u32 vtf;
148772c07e2bSDon Hiatt 
148872c07e2bSDon Hiatt 		grh_len = sizeof(struct ib_grh);
148972c07e2bSDon Hiatt 		packet->ohdr = packet->ebuf + grh_len;
149072c07e2bSDon Hiatt 		packet->grh = packet->ebuf;
149172c07e2bSDon Hiatt 		if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
149272c07e2bSDon Hiatt 			goto drop;
149372c07e2bSDon Hiatt 		vtf = be32_to_cpu(packet->grh->version_tclass_flow);
149472c07e2bSDon Hiatt 		if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
149572c07e2bSDon Hiatt 			goto drop;
149672c07e2bSDon Hiatt 	} else {
149772c07e2bSDon Hiatt 		goto drop;
149872c07e2bSDon Hiatt 	}
149972c07e2bSDon Hiatt 
150072c07e2bSDon Hiatt 	/* Query commonly used fields from packet header */
150172c07e2bSDon Hiatt 	packet->opcode = ib_bth_get_opcode(packet->ohdr);
150278d3633bSMike Marciniszyn 	/* hdr_len_by_opcode already has an IB LRH factored in */
150378d3633bSMike Marciniszyn 	packet->hlen = hdr_len_by_opcode[packet->opcode] +
150478d3633bSMike Marciniszyn 		(LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
150578d3633bSMike Marciniszyn 	packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES;
150672c07e2bSDon Hiatt 	packet->slid = hfi1_16B_get_slid(packet->hdr);
150772c07e2bSDon Hiatt 	packet->dlid = hfi1_16B_get_dlid(packet->hdr);
150872c07e2bSDon Hiatt 	if (unlikely(hfi1_is_16B_mcast(packet->dlid)))
150972c07e2bSDon Hiatt 		packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
151072c07e2bSDon Hiatt 				opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR),
151172c07e2bSDon Hiatt 					    16B);
151272c07e2bSDon Hiatt 	packet->sc = hfi1_16B_get_sc(packet->hdr);
151372c07e2bSDon Hiatt 	packet->sl = ibp->sc_to_sl[packet->sc];
151472c07e2bSDon Hiatt 	packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
151572c07e2bSDon Hiatt 	packet->extra_byte = SIZE_OF_LT;
15166d6b8848SSebastian Sanchez 	packet->pkey = hfi1_16B_get_pkey(packet->hdr);
15176d6b8848SSebastian Sanchez 	packet->migrated = opa_bth_is_migration(packet->ohdr);
151872c07e2bSDon Hiatt 
15195786adf3SDon Hiatt 	if (hfi1_bypass_ingress_pkt_check(packet))
15205786adf3SDon Hiatt 		goto drop;
15215786adf3SDon Hiatt 
152272c07e2bSDon Hiatt 	return 0;
152372c07e2bSDon Hiatt drop:
152472c07e2bSDon Hiatt 	hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);
152572c07e2bSDon Hiatt 	ibp->rvp.n_pkt_drops++;
152672c07e2bSDon Hiatt 	return -EINVAL;
152772c07e2bSDon Hiatt }
152872c07e2bSDon Hiatt 
1529f48ad614SDennis Dalessandro void handle_eflags(struct hfi1_packet *packet)
1530f48ad614SDennis Dalessandro {
1531f48ad614SDennis Dalessandro 	struct hfi1_ctxtdata *rcd = packet->rcd;
1532f48ad614SDennis Dalessandro 	u32 rte = rhf_rcv_type_err(packet->rhf);
1533f48ad614SDennis Dalessandro 
1534f48ad614SDennis Dalessandro 	rcv_hdrerr(rcd, rcd->ppd, packet);
1535f48ad614SDennis Dalessandro 	if (rhf_err_flags(packet->rhf))
1536f48ad614SDennis Dalessandro 		dd_dev_err(rcd->dd,
1537f48ad614SDennis Dalessandro 			   "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
1538f48ad614SDennis Dalessandro 			   rcd->ctxt, packet->rhf,
1539f48ad614SDennis Dalessandro 			   packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1540f48ad614SDennis Dalessandro 			   packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1541f48ad614SDennis Dalessandro 			   packet->rhf & RHF_DC_ERR ? "dc " : "",
1542f48ad614SDennis Dalessandro 			   packet->rhf & RHF_TID_ERR ? "tid " : "",
1543f48ad614SDennis Dalessandro 			   packet->rhf & RHF_LEN_ERR ? "len " : "",
1544f48ad614SDennis Dalessandro 			   packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1545f48ad614SDennis Dalessandro 			   packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
1546f48ad614SDennis Dalessandro 			   packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1547f48ad614SDennis Dalessandro 			   rte);
1548f48ad614SDennis Dalessandro }
1549f48ad614SDennis Dalessandro 
1550f48ad614SDennis Dalessandro /*
1551f48ad614SDennis Dalessandro  * The following functions are called by the interrupt handler. They are type
1552f48ad614SDennis Dalessandro  * specific handlers for each packet type.
1553f48ad614SDennis Dalessandro  */
1554f48ad614SDennis Dalessandro int process_receive_ib(struct hfi1_packet *packet)
1555f48ad614SDennis Dalessandro {
15560181ce31SDon Hiatt 	if (unlikely(hfi1_dbg_fault_packet(packet)))
15570181ce31SDon Hiatt 		return RHF_RCV_CONTINUE;
15580181ce31SDon Hiatt 
15599039746cSDon Hiatt 	if (hfi1_setup_9B_packet(packet))
15609039746cSDon Hiatt 		return RHF_RCV_CONTINUE;
15619039746cSDon Hiatt 
1562aca7f4fcSSebastian Sanchez 	trace_hfi1_rcvhdr(packet, RHF_RCV_TYPE_IB);
1563243d9f43SDon Hiatt 
1564f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf))) {
1565f48ad614SDennis Dalessandro 		handle_eflags(packet);
1566f48ad614SDennis Dalessandro 		return RHF_RCV_CONTINUE;
1567f48ad614SDennis Dalessandro 	}
1568f48ad614SDennis Dalessandro 
1569f48ad614SDennis Dalessandro 	hfi1_ib_rcv(packet);
1570f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1571f48ad614SDennis Dalessandro }
1572f48ad614SDennis Dalessandro 
1573d4829ea6SVishwanathapura, Niranjana static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)
1574d4829ea6SVishwanathapura, Niranjana {
1575d4829ea6SVishwanathapura, Niranjana 	/* Packet received in VNIC context via RSM */
1576d4829ea6SVishwanathapura, Niranjana 	if (packet->rcd->is_vnic)
1577d4829ea6SVishwanathapura, Niranjana 		return true;
1578d4829ea6SVishwanathapura, Niranjana 
157972c07e2bSDon Hiatt 	if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) &&
158072c07e2bSDon Hiatt 	    (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR))
1581d4829ea6SVishwanathapura, Niranjana 		return true;
1582d4829ea6SVishwanathapura, Niranjana 
1583d4829ea6SVishwanathapura, Niranjana 	return false;
1584d4829ea6SVishwanathapura, Niranjana }
1585d4829ea6SVishwanathapura, Niranjana 
1586f48ad614SDennis Dalessandro int process_receive_bypass(struct hfi1_packet *packet)
1587f48ad614SDennis Dalessandro {
1588505efe3eSJakub Pawlak 	struct hfi1_devdata *dd = packet->rcd->dd;
1589505efe3eSJakub Pawlak 
159072c07e2bSDon Hiatt 	if (hfi1_is_vnic_packet(packet)) {
1591d4829ea6SVishwanathapura, Niranjana 		hfi1_vnic_bypass_rcv(packet);
1592d4829ea6SVishwanathapura, Niranjana 		return RHF_RCV_CONTINUE;
1593d4829ea6SVishwanathapura, Niranjana 	}
1594f48ad614SDennis Dalessandro 
159572c07e2bSDon Hiatt 	if (hfi1_setup_bypass_packet(packet))
159672c07e2bSDon Hiatt 		return RHF_RCV_CONTINUE;
159772c07e2bSDon Hiatt 
159872c07e2bSDon Hiatt 	if (unlikely(rhf_err_flags(packet->rhf))) {
159972c07e2bSDon Hiatt 		handle_eflags(packet);
160072c07e2bSDon Hiatt 		return RHF_RCV_CONTINUE;
160172c07e2bSDon Hiatt 	}
160272c07e2bSDon Hiatt 
160372c07e2bSDon Hiatt 	if (hfi1_16B_get_l2(packet->hdr) == 0x2) {
160472c07e2bSDon Hiatt 		hfi1_16B_rcv(packet);
160572c07e2bSDon Hiatt 	} else {
160672c07e2bSDon Hiatt 		dd_dev_err(dd,
160772c07e2bSDon Hiatt 			   "Bypass packets other than 16B are not supported in normal operation. Dropping\n");
1608505efe3eSJakub Pawlak 		incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
160972c07e2bSDon Hiatt 		if (!(dd->err_info_rcvport.status_and_code &
161072c07e2bSDon Hiatt 		      OPA_EI_STATUS_SMASK)) {
1611505efe3eSJakub Pawlak 			u64 *flits = packet->ebuf;
1612505efe3eSJakub Pawlak 
1613505efe3eSJakub Pawlak 			if (flits && !(packet->rhf & RHF_LEN_ERR)) {
1614505efe3eSJakub Pawlak 				dd->err_info_rcvport.packet_flit1 = flits[0];
1615505efe3eSJakub Pawlak 				dd->err_info_rcvport.packet_flit2 =
161672c07e2bSDon Hiatt 					packet->tlen > sizeof(flits[0]) ?
161772c07e2bSDon Hiatt 					flits[1] : 0;
1618505efe3eSJakub Pawlak 			}
1619505efe3eSJakub Pawlak 			dd->err_info_rcvport.status_and_code |=
1620505efe3eSJakub Pawlak 				(OPA_EI_STATUS_SMASK | BAD_L2_ERR);
1621505efe3eSJakub Pawlak 		}
162272c07e2bSDon Hiatt 	}
1623f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1624f48ad614SDennis Dalessandro }
1625f48ad614SDennis Dalessandro 
1626f48ad614SDennis Dalessandro int process_receive_error(struct hfi1_packet *packet)
1627f48ad614SDennis Dalessandro {
1628243d9f43SDon Hiatt 	/* KHdrHCRCErr -- KDETH packet with a bad HCRC */
1629243d9f43SDon Hiatt 	if (unlikely(
1630243d9f43SDon Hiatt 		 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1631243d9f43SDon Hiatt 		 rhf_rcv_type_err(packet->rhf) == 3))
1632243d9f43SDon Hiatt 		return RHF_RCV_CONTINUE;
1633243d9f43SDon Hiatt 
16349039746cSDon Hiatt 	hfi1_setup_ib_header(packet);
1635f48ad614SDennis Dalessandro 	handle_eflags(packet);
1636f48ad614SDennis Dalessandro 
1637f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1638f48ad614SDennis Dalessandro 		dd_dev_err(packet->rcd->dd,
1639f48ad614SDennis Dalessandro 			   "Unhandled error packet received. Dropping.\n");
1640f48ad614SDennis Dalessandro 
1641f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1642f48ad614SDennis Dalessandro }
1643f48ad614SDennis Dalessandro 
1644f48ad614SDennis Dalessandro int kdeth_process_expected(struct hfi1_packet *packet)
1645f48ad614SDennis Dalessandro {
16460181ce31SDon Hiatt 	if (unlikely(hfi1_dbg_fault_packet(packet)))
16470181ce31SDon Hiatt 		return RHF_RCV_CONTINUE;
16489039746cSDon Hiatt 
16499039746cSDon Hiatt 	hfi1_setup_ib_header(packet);
1650f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1651f48ad614SDennis Dalessandro 		handle_eflags(packet);
1652f48ad614SDennis Dalessandro 
1653f48ad614SDennis Dalessandro 	dd_dev_err(packet->rcd->dd,
1654f48ad614SDennis Dalessandro 		   "Unhandled expected packet received. Dropping.\n");
1655f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1656f48ad614SDennis Dalessandro }
1657f48ad614SDennis Dalessandro 
1658f48ad614SDennis Dalessandro int kdeth_process_eager(struct hfi1_packet *packet)
1659f48ad614SDennis Dalessandro {
16609039746cSDon Hiatt 	hfi1_setup_ib_header(packet);
1661f48ad614SDennis Dalessandro 	if (unlikely(rhf_err_flags(packet->rhf)))
1662f48ad614SDennis Dalessandro 		handle_eflags(packet);
16630181ce31SDon Hiatt 	if (unlikely(hfi1_dbg_fault_packet(packet)))
16640181ce31SDon Hiatt 		return RHF_RCV_CONTINUE;
1665f48ad614SDennis Dalessandro 
1666f48ad614SDennis Dalessandro 	dd_dev_err(packet->rcd->dd,
1667f48ad614SDennis Dalessandro 		   "Unhandled eager packet received. Dropping.\n");
1668f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1669f48ad614SDennis Dalessandro }
1670f48ad614SDennis Dalessandro 
1671f48ad614SDennis Dalessandro int process_receive_invalid(struct hfi1_packet *packet)
1672f48ad614SDennis Dalessandro {
1673f48ad614SDennis Dalessandro 	dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1674f48ad614SDennis Dalessandro 		   rhf_rcv_type(packet->rhf));
1675f48ad614SDennis Dalessandro 	return RHF_RCV_CONTINUE;
1676f48ad614SDennis Dalessandro }
1677bf808b50SKaike Wan 
1678bf808b50SKaike Wan void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
1679bf808b50SKaike Wan {
1680bf808b50SKaike Wan 	struct hfi1_packet packet;
1681bf808b50SKaike Wan 	struct ps_mdata mdata;
1682bf808b50SKaike Wan 
1683bf808b50SKaike Wan 	seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",
1684bf808b50SKaike Wan 		   rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize,
1685bf808b50SKaike Wan 		   HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
1686bf808b50SKaike Wan 		   "dma_rtail" : "nodma_rtail",
1687bf808b50SKaike Wan 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
1688bf808b50SKaike Wan 		   RCV_HDR_HEAD_HEAD_MASK,
1689bf808b50SKaike Wan 		   read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL));
1690bf808b50SKaike Wan 
1691bf808b50SKaike Wan 	init_packet(rcd, &packet);
1692bf808b50SKaike Wan 	init_ps_mdata(&mdata, &packet);
1693bf808b50SKaike Wan 
1694bf808b50SKaike Wan 	while (1) {
1695bf808b50SKaike Wan 		struct hfi1_devdata *dd = rcd->dd;
1696bf808b50SKaike Wan 		__le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
1697bf808b50SKaike Wan 					 dd->rhf_offset;
1698bf808b50SKaike Wan 		struct ib_header *hdr;
1699bf808b50SKaike Wan 		u64 rhf = rhf_to_cpu(rhf_addr);
1700bf808b50SKaike Wan 		u32 etype = rhf_rcv_type(rhf), qpn;
1701bf808b50SKaike Wan 		u8 opcode;
1702bf808b50SKaike Wan 		u32 psn;
1703bf808b50SKaike Wan 		u8 lnh;
1704bf808b50SKaike Wan 
1705bf808b50SKaike Wan 		if (ps_done(&mdata, rhf, rcd))
1706bf808b50SKaike Wan 			break;
1707bf808b50SKaike Wan 
1708bf808b50SKaike Wan 		if (ps_skip(&mdata, rhf, rcd))
1709bf808b50SKaike Wan 			goto next;
1710bf808b50SKaike Wan 
1711bf808b50SKaike Wan 		if (etype > RHF_RCV_TYPE_IB)
1712bf808b50SKaike Wan 			goto next;
1713bf808b50SKaike Wan 
1714bf808b50SKaike Wan 		packet.hdr = hfi1_get_msgheader(dd, rhf_addr);
1715bf808b50SKaike Wan 		hdr = packet.hdr;
1716bf808b50SKaike Wan 
1717bf808b50SKaike Wan 		lnh = be16_to_cpu(hdr->lrh[0]) & 3;
1718bf808b50SKaike Wan 
1719bf808b50SKaike Wan 		if (lnh == HFI1_LRH_BTH)
1720bf808b50SKaike Wan 			packet.ohdr = &hdr->u.oth;
1721bf808b50SKaike Wan 		else if (lnh == HFI1_LRH_GRH)
1722bf808b50SKaike Wan 			packet.ohdr = &hdr->u.l.oth;
1723bf808b50SKaike Wan 		else
1724bf808b50SKaike Wan 			goto next; /* just in case */
1725bf808b50SKaike Wan 
1726bf808b50SKaike Wan 		opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24);
1727bf808b50SKaike Wan 		qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK;
1728bf808b50SKaike Wan 		psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2]));
1729bf808b50SKaike Wan 
1730bf808b50SKaike Wan 		seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n",
1731bf808b50SKaike Wan 			   mdata.ps_head, opcode, qpn, psn);
1732bf808b50SKaike Wan next:
1733bf808b50SKaike Wan 		update_ps_mdata(&mdata, rcd);
1734bf808b50SKaike Wan 	}
1735bf808b50SKaike Wan }
1736