1f48ad614SDennis Dalessandro /* 2f48ad614SDennis Dalessandro * Copyright(c) 2015, 2016 Intel Corporation. 3f48ad614SDennis Dalessandro * 4f48ad614SDennis Dalessandro * This file is provided under a dual BSD/GPLv2 license. When using or 5f48ad614SDennis Dalessandro * redistributing this file, you may do so under either license. 6f48ad614SDennis Dalessandro * 7f48ad614SDennis Dalessandro * GPL LICENSE SUMMARY 8f48ad614SDennis Dalessandro * 9f48ad614SDennis Dalessandro * This program is free software; you can redistribute it and/or modify 10f48ad614SDennis Dalessandro * it under the terms of version 2 of the GNU General Public License as 11f48ad614SDennis Dalessandro * published by the Free Software Foundation. 12f48ad614SDennis Dalessandro * 13f48ad614SDennis Dalessandro * This program is distributed in the hope that it will be useful, but 14f48ad614SDennis Dalessandro * WITHOUT ANY WARRANTY; without even the implied warranty of 15f48ad614SDennis Dalessandro * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16f48ad614SDennis Dalessandro * General Public License for more details. 17f48ad614SDennis Dalessandro * 18f48ad614SDennis Dalessandro * BSD LICENSE 19f48ad614SDennis Dalessandro * 20f48ad614SDennis Dalessandro * Redistribution and use in source and binary forms, with or without 21f48ad614SDennis Dalessandro * modification, are permitted provided that the following conditions 22f48ad614SDennis Dalessandro * are met: 23f48ad614SDennis Dalessandro * 24f48ad614SDennis Dalessandro * - Redistributions of source code must retain the above copyright 25f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer. 26f48ad614SDennis Dalessandro * - Redistributions in binary form must reproduce the above copyright 27f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer in 28f48ad614SDennis Dalessandro * the documentation and/or other materials provided with the 29f48ad614SDennis Dalessandro * distribution. 30f48ad614SDennis Dalessandro * - Neither the name of Intel Corporation nor the names of its 31f48ad614SDennis Dalessandro * contributors may be used to endorse or promote products derived 32f48ad614SDennis Dalessandro * from this software without specific prior written permission. 33f48ad614SDennis Dalessandro * 34f48ad614SDennis Dalessandro * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35f48ad614SDennis Dalessandro * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36f48ad614SDennis Dalessandro * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37f48ad614SDennis Dalessandro * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38f48ad614SDennis Dalessandro * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39f48ad614SDennis Dalessandro * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40f48ad614SDennis Dalessandro * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41f48ad614SDennis Dalessandro * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42f48ad614SDennis Dalessandro * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43f48ad614SDennis Dalessandro * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44f48ad614SDennis Dalessandro * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45f48ad614SDennis Dalessandro * 46f48ad614SDennis Dalessandro */ 47f48ad614SDennis Dalessandro 48f48ad614SDennis Dalessandro #include <linux/spinlock.h> 49f48ad614SDennis Dalessandro #include <linux/pci.h> 50f48ad614SDennis Dalessandro #include <linux/io.h> 51f48ad614SDennis Dalessandro #include <linux/delay.h> 52f48ad614SDennis Dalessandro #include <linux/netdevice.h> 53f48ad614SDennis Dalessandro #include <linux/vmalloc.h> 54f48ad614SDennis Dalessandro #include <linux/module.h> 55f48ad614SDennis Dalessandro #include <linux/prefetch.h> 56f48ad614SDennis Dalessandro #include <rdma/ib_verbs.h> 57f48ad614SDennis Dalessandro 58f48ad614SDennis Dalessandro #include "hfi.h" 59f48ad614SDennis Dalessandro #include "trace.h" 60f48ad614SDennis Dalessandro #include "qp.h" 61f48ad614SDennis Dalessandro #include "sdma.h" 62f48ad614SDennis Dalessandro 63f48ad614SDennis Dalessandro #undef pr_fmt 64f48ad614SDennis Dalessandro #define pr_fmt(fmt) DRIVER_NAME ": " fmt 65f48ad614SDennis Dalessandro 66f48ad614SDennis Dalessandro /* 67f48ad614SDennis Dalessandro * The size has to be longer than this string, so we can append 68f48ad614SDennis Dalessandro * board/chip information to it in the initialization code. 69f48ad614SDennis Dalessandro */ 70f48ad614SDennis Dalessandro const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n"; 71f48ad614SDennis Dalessandro 72f48ad614SDennis Dalessandro DEFINE_SPINLOCK(hfi1_devs_lock); 73f48ad614SDennis Dalessandro LIST_HEAD(hfi1_dev_list); 74f48ad614SDennis Dalessandro DEFINE_MUTEX(hfi1_mutex); /* general driver use */ 75f48ad614SDennis Dalessandro 76f48ad614SDennis Dalessandro unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU; 77f48ad614SDennis Dalessandro module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO); 78f48ad614SDennis Dalessandro MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify( 79f48ad614SDennis Dalessandro HFI1_DEFAULT_MAX_MTU)); 80f48ad614SDennis Dalessandro 81f48ad614SDennis Dalessandro unsigned int hfi1_cu = 1; 82f48ad614SDennis Dalessandro module_param_named(cu, hfi1_cu, uint, S_IRUGO); 83f48ad614SDennis Dalessandro MODULE_PARM_DESC(cu, "Credit return units"); 84f48ad614SDennis Dalessandro 85f48ad614SDennis Dalessandro unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT; 86f48ad614SDennis Dalessandro static int hfi1_caps_set(const char *, const struct kernel_param *); 87f48ad614SDennis Dalessandro static int hfi1_caps_get(char *, const struct kernel_param *); 88f48ad614SDennis Dalessandro static const struct kernel_param_ops cap_ops = { 89f48ad614SDennis Dalessandro .set = hfi1_caps_set, 90f48ad614SDennis Dalessandro .get = hfi1_caps_get 91f48ad614SDennis Dalessandro }; 92f48ad614SDennis Dalessandro module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO); 93f48ad614SDennis Dalessandro MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features"); 94f48ad614SDennis Dalessandro 95f48ad614SDennis Dalessandro MODULE_LICENSE("Dual BSD/GPL"); 96f48ad614SDennis Dalessandro MODULE_DESCRIPTION("Intel Omni-Path Architecture driver"); 97f48ad614SDennis Dalessandro MODULE_VERSION(HFI1_DRIVER_VERSION); 98f48ad614SDennis Dalessandro 99f48ad614SDennis Dalessandro /* 100f48ad614SDennis Dalessandro * MAX_PKT_RCV is the max # if packets processed per receive interrupt. 101f48ad614SDennis Dalessandro */ 102f48ad614SDennis Dalessandro #define MAX_PKT_RECV 64 103f48ad614SDennis Dalessandro #define EGR_HEAD_UPDATE_THRESHOLD 16 104f48ad614SDennis Dalessandro 105f48ad614SDennis Dalessandro struct hfi1_ib_stats hfi1_stats; 106f48ad614SDennis Dalessandro 107f48ad614SDennis Dalessandro static int hfi1_caps_set(const char *val, const struct kernel_param *kp) 108f48ad614SDennis Dalessandro { 109f48ad614SDennis Dalessandro int ret = 0; 110f48ad614SDennis Dalessandro unsigned long *cap_mask_ptr = (unsigned long *)kp->arg, 111f48ad614SDennis Dalessandro cap_mask = *cap_mask_ptr, value, diff, 112f48ad614SDennis Dalessandro write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) | 113f48ad614SDennis Dalessandro HFI1_CAP_WRITABLE_MASK); 114f48ad614SDennis Dalessandro 115f48ad614SDennis Dalessandro ret = kstrtoul(val, 0, &value); 116f48ad614SDennis Dalessandro if (ret) { 117f48ad614SDennis Dalessandro pr_warn("Invalid module parameter value for 'cap_mask'\n"); 118f48ad614SDennis Dalessandro goto done; 119f48ad614SDennis Dalessandro } 120f48ad614SDennis Dalessandro /* Get the changed bits (except the locked bit) */ 121f48ad614SDennis Dalessandro diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK); 122f48ad614SDennis Dalessandro 123f48ad614SDennis Dalessandro /* Remove any bits that are not allowed to change after driver load */ 124f48ad614SDennis Dalessandro if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) { 125f48ad614SDennis Dalessandro pr_warn("Ignoring non-writable capability bits %#lx\n", 126f48ad614SDennis Dalessandro diff & ~write_mask); 127f48ad614SDennis Dalessandro diff &= write_mask; 128f48ad614SDennis Dalessandro } 129f48ad614SDennis Dalessandro 130f48ad614SDennis Dalessandro /* Mask off any reserved bits */ 131f48ad614SDennis Dalessandro diff &= ~HFI1_CAP_RESERVED_MASK; 132f48ad614SDennis Dalessandro /* Clear any previously set and changing bits */ 133f48ad614SDennis Dalessandro cap_mask &= ~diff; 134f48ad614SDennis Dalessandro /* Update the bits with the new capability */ 135f48ad614SDennis Dalessandro cap_mask |= (value & diff); 136f48ad614SDennis Dalessandro /* Check for any kernel/user restrictions */ 137f48ad614SDennis Dalessandro diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^ 138f48ad614SDennis Dalessandro ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT); 139f48ad614SDennis Dalessandro cap_mask &= ~diff; 140f48ad614SDennis Dalessandro /* Set the bitmask to the final set */ 141f48ad614SDennis Dalessandro *cap_mask_ptr = cap_mask; 142f48ad614SDennis Dalessandro done: 143f48ad614SDennis Dalessandro return ret; 144f48ad614SDennis Dalessandro } 145f48ad614SDennis Dalessandro 146f48ad614SDennis Dalessandro static int hfi1_caps_get(char *buffer, const struct kernel_param *kp) 147f48ad614SDennis Dalessandro { 148f48ad614SDennis Dalessandro unsigned long cap_mask = *(unsigned long *)kp->arg; 149f48ad614SDennis Dalessandro 150f48ad614SDennis Dalessandro cap_mask &= ~HFI1_CAP_LOCKED_SMASK; 151f48ad614SDennis Dalessandro cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT); 152f48ad614SDennis Dalessandro 153f48ad614SDennis Dalessandro return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask); 154f48ad614SDennis Dalessandro } 155f48ad614SDennis Dalessandro 156f48ad614SDennis Dalessandro const char *get_unit_name(int unit) 157f48ad614SDennis Dalessandro { 158f48ad614SDennis Dalessandro static char iname[16]; 159f48ad614SDennis Dalessandro 160f48ad614SDennis Dalessandro snprintf(iname, sizeof(iname), DRIVER_NAME "_%u", unit); 161f48ad614SDennis Dalessandro return iname; 162f48ad614SDennis Dalessandro } 163f48ad614SDennis Dalessandro 164f48ad614SDennis Dalessandro const char *get_card_name(struct rvt_dev_info *rdi) 165f48ad614SDennis Dalessandro { 166f48ad614SDennis Dalessandro struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 167f48ad614SDennis Dalessandro struct hfi1_devdata *dd = container_of(ibdev, 168f48ad614SDennis Dalessandro struct hfi1_devdata, verbs_dev); 169f48ad614SDennis Dalessandro return get_unit_name(dd->unit); 170f48ad614SDennis Dalessandro } 171f48ad614SDennis Dalessandro 172f48ad614SDennis Dalessandro struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi) 173f48ad614SDennis Dalessandro { 174f48ad614SDennis Dalessandro struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 175f48ad614SDennis Dalessandro struct hfi1_devdata *dd = container_of(ibdev, 176f48ad614SDennis Dalessandro struct hfi1_devdata, verbs_dev); 177f48ad614SDennis Dalessandro return dd->pcidev; 178f48ad614SDennis Dalessandro } 179f48ad614SDennis Dalessandro 180f48ad614SDennis Dalessandro /* 181f48ad614SDennis Dalessandro * Return count of units with at least one port ACTIVE. 182f48ad614SDennis Dalessandro */ 183f48ad614SDennis Dalessandro int hfi1_count_active_units(void) 184f48ad614SDennis Dalessandro { 185f48ad614SDennis Dalessandro struct hfi1_devdata *dd; 186f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd; 187f48ad614SDennis Dalessandro unsigned long flags; 188f48ad614SDennis Dalessandro int pidx, nunits_active = 0; 189f48ad614SDennis Dalessandro 190f48ad614SDennis Dalessandro spin_lock_irqsave(&hfi1_devs_lock, flags); 191f48ad614SDennis Dalessandro list_for_each_entry(dd, &hfi1_dev_list, list) { 192f48ad614SDennis Dalessandro if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase) 193f48ad614SDennis Dalessandro continue; 194f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) { 195f48ad614SDennis Dalessandro ppd = dd->pport + pidx; 196f48ad614SDennis Dalessandro if (ppd->lid && ppd->linkup) { 197f48ad614SDennis Dalessandro nunits_active++; 198f48ad614SDennis Dalessandro break; 199f48ad614SDennis Dalessandro } 200f48ad614SDennis Dalessandro } 201f48ad614SDennis Dalessandro } 202f48ad614SDennis Dalessandro spin_unlock_irqrestore(&hfi1_devs_lock, flags); 203f48ad614SDennis Dalessandro return nunits_active; 204f48ad614SDennis Dalessandro } 205f48ad614SDennis Dalessandro 206f48ad614SDennis Dalessandro /* 207f48ad614SDennis Dalessandro * Return count of all units, optionally return in arguments 208f48ad614SDennis Dalessandro * the number of usable (present) units, and the number of 209f48ad614SDennis Dalessandro * ports that are up. 210f48ad614SDennis Dalessandro */ 211f48ad614SDennis Dalessandro int hfi1_count_units(int *npresentp, int *nupp) 212f48ad614SDennis Dalessandro { 213f48ad614SDennis Dalessandro int nunits = 0, npresent = 0, nup = 0; 214f48ad614SDennis Dalessandro struct hfi1_devdata *dd; 215f48ad614SDennis Dalessandro unsigned long flags; 216f48ad614SDennis Dalessandro int pidx; 217f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd; 218f48ad614SDennis Dalessandro 219f48ad614SDennis Dalessandro spin_lock_irqsave(&hfi1_devs_lock, flags); 220f48ad614SDennis Dalessandro 221f48ad614SDennis Dalessandro list_for_each_entry(dd, &hfi1_dev_list, list) { 222f48ad614SDennis Dalessandro nunits++; 223f48ad614SDennis Dalessandro if ((dd->flags & HFI1_PRESENT) && dd->kregbase) 224f48ad614SDennis Dalessandro npresent++; 225f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) { 226f48ad614SDennis Dalessandro ppd = dd->pport + pidx; 227f48ad614SDennis Dalessandro if (ppd->lid && ppd->linkup) 228f48ad614SDennis Dalessandro nup++; 229f48ad614SDennis Dalessandro } 230f48ad614SDennis Dalessandro } 231f48ad614SDennis Dalessandro 232f48ad614SDennis Dalessandro spin_unlock_irqrestore(&hfi1_devs_lock, flags); 233f48ad614SDennis Dalessandro 234f48ad614SDennis Dalessandro if (npresentp) 235f48ad614SDennis Dalessandro *npresentp = npresent; 236f48ad614SDennis Dalessandro if (nupp) 237f48ad614SDennis Dalessandro *nupp = nup; 238f48ad614SDennis Dalessandro 239f48ad614SDennis Dalessandro return nunits; 240f48ad614SDennis Dalessandro } 241f48ad614SDennis Dalessandro 242f48ad614SDennis Dalessandro /* 243f48ad614SDennis Dalessandro * Get address of eager buffer from it's index (allocated in chunks, not 244f48ad614SDennis Dalessandro * contiguous). 245f48ad614SDennis Dalessandro */ 246f48ad614SDennis Dalessandro static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf, 247f48ad614SDennis Dalessandro u8 *update) 248f48ad614SDennis Dalessandro { 249f48ad614SDennis Dalessandro u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf); 250f48ad614SDennis Dalessandro 251f48ad614SDennis Dalessandro *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset; 252f48ad614SDennis Dalessandro return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) + 253f48ad614SDennis Dalessandro (offset * RCV_BUF_BLOCK_SIZE)); 254f48ad614SDennis Dalessandro } 255f48ad614SDennis Dalessandro 256f48ad614SDennis Dalessandro /* 257f48ad614SDennis Dalessandro * Validate and encode the a given RcvArray Buffer size. 258f48ad614SDennis Dalessandro * The function will check whether the given size falls within 259f48ad614SDennis Dalessandro * allowed size ranges for the respective type and, optionally, 260f48ad614SDennis Dalessandro * return the proper encoding. 261f48ad614SDennis Dalessandro */ 262f48ad614SDennis Dalessandro inline int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded) 263f48ad614SDennis Dalessandro { 264f48ad614SDennis Dalessandro if (unlikely(!PAGE_ALIGNED(size))) 265f48ad614SDennis Dalessandro return 0; 266f48ad614SDennis Dalessandro if (unlikely(size < MIN_EAGER_BUFFER)) 267f48ad614SDennis Dalessandro return 0; 268f48ad614SDennis Dalessandro if (size > 269f48ad614SDennis Dalessandro (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER)) 270f48ad614SDennis Dalessandro return 0; 271f48ad614SDennis Dalessandro if (encoded) 272f48ad614SDennis Dalessandro *encoded = ilog2(size / PAGE_SIZE) + 1; 273f48ad614SDennis Dalessandro return 1; 274f48ad614SDennis Dalessandro } 275f48ad614SDennis Dalessandro 276f48ad614SDennis Dalessandro static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, 277f48ad614SDennis Dalessandro struct hfi1_packet *packet) 278f48ad614SDennis Dalessandro { 279261a4351SMike Marciniszyn struct ib_header *rhdr = packet->hdr; 280f48ad614SDennis Dalessandro u32 rte = rhf_rcv_type_err(packet->rhf); 281f48ad614SDennis Dalessandro int lnh = be16_to_cpu(rhdr->lrh[0]) & 3; 282f48ad614SDennis Dalessandro struct hfi1_ibport *ibp = &ppd->ibport_data; 283f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 284f48ad614SDennis Dalessandro struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 285f48ad614SDennis Dalessandro 286f48ad614SDennis Dalessandro if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) 287f48ad614SDennis Dalessandro return; 288f48ad614SDennis Dalessandro 289f48ad614SDennis Dalessandro if (packet->rhf & RHF_TID_ERR) { 290f48ad614SDennis Dalessandro /* For TIDERR and RC QPs preemptively schedule a NAK */ 291261a4351SMike Marciniszyn struct ib_other_headers *ohdr = NULL; 292f48ad614SDennis Dalessandro u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 293261a4351SMike Marciniszyn u16 lid = be16_to_cpu(rhdr->lrh[1]); 294f48ad614SDennis Dalessandro u32 qp_num; 295f48ad614SDennis Dalessandro u32 rcv_flags = 0; 296f48ad614SDennis Dalessandro 297f48ad614SDennis Dalessandro /* Sanity check packet */ 298f48ad614SDennis Dalessandro if (tlen < 24) 299f48ad614SDennis Dalessandro goto drop; 300f48ad614SDennis Dalessandro 301f48ad614SDennis Dalessandro /* Check for GRH */ 302f48ad614SDennis Dalessandro if (lnh == HFI1_LRH_BTH) { 303261a4351SMike Marciniszyn ohdr = &rhdr->u.oth; 304f48ad614SDennis Dalessandro } else if (lnh == HFI1_LRH_GRH) { 305f48ad614SDennis Dalessandro u32 vtf; 306f48ad614SDennis Dalessandro 307261a4351SMike Marciniszyn ohdr = &rhdr->u.l.oth; 308261a4351SMike Marciniszyn if (rhdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR) 309f48ad614SDennis Dalessandro goto drop; 310261a4351SMike Marciniszyn vtf = be32_to_cpu(rhdr->u.l.grh.version_tclass_flow); 311f48ad614SDennis Dalessandro if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 312f48ad614SDennis Dalessandro goto drop; 313f48ad614SDennis Dalessandro rcv_flags |= HFI1_HAS_GRH; 314f48ad614SDennis Dalessandro } else { 315f48ad614SDennis Dalessandro goto drop; 316f48ad614SDennis Dalessandro } 317f48ad614SDennis Dalessandro /* Get the destination QP number. */ 318f48ad614SDennis Dalessandro qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK; 319f48ad614SDennis Dalessandro if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) { 320f48ad614SDennis Dalessandro struct rvt_qp *qp; 321f48ad614SDennis Dalessandro unsigned long flags; 322f48ad614SDennis Dalessandro 323f48ad614SDennis Dalessandro rcu_read_lock(); 324f48ad614SDennis Dalessandro qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); 325f48ad614SDennis Dalessandro if (!qp) { 326f48ad614SDennis Dalessandro rcu_read_unlock(); 327f48ad614SDennis Dalessandro goto drop; 328f48ad614SDennis Dalessandro } 329f48ad614SDennis Dalessandro 330f48ad614SDennis Dalessandro /* 331f48ad614SDennis Dalessandro * Handle only RC QPs - for other QP types drop error 332f48ad614SDennis Dalessandro * packet. 333f48ad614SDennis Dalessandro */ 334f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->r_lock, flags); 335f48ad614SDennis Dalessandro 336f48ad614SDennis Dalessandro /* Check for valid receive state. */ 337f48ad614SDennis Dalessandro if (!(ib_rvt_state_ops[qp->state] & 338f48ad614SDennis Dalessandro RVT_PROCESS_RECV_OK)) { 339f48ad614SDennis Dalessandro ibp->rvp.n_pkt_drops++; 340f48ad614SDennis Dalessandro } 341f48ad614SDennis Dalessandro 342f48ad614SDennis Dalessandro switch (qp->ibqp.qp_type) { 343f48ad614SDennis Dalessandro case IB_QPT_RC: 344f48ad614SDennis Dalessandro hfi1_rc_hdrerr( 345f48ad614SDennis Dalessandro rcd, 346261a4351SMike Marciniszyn rhdr, 347f48ad614SDennis Dalessandro rcv_flags, 348f48ad614SDennis Dalessandro qp); 349f48ad614SDennis Dalessandro break; 350f48ad614SDennis Dalessandro default: 351f48ad614SDennis Dalessandro /* For now don't handle any other QP types */ 352f48ad614SDennis Dalessandro break; 353f48ad614SDennis Dalessandro } 354f48ad614SDennis Dalessandro 355f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->r_lock, flags); 356f48ad614SDennis Dalessandro rcu_read_unlock(); 357f48ad614SDennis Dalessandro } /* Unicast QP */ 358f48ad614SDennis Dalessandro } /* Valid packet with TIDErr */ 359f48ad614SDennis Dalessandro 360f48ad614SDennis Dalessandro /* handle "RcvTypeErr" flags */ 361f48ad614SDennis Dalessandro switch (rte) { 362f48ad614SDennis Dalessandro case RHF_RTE_ERROR_OP_CODE_ERR: 363f48ad614SDennis Dalessandro { 364f48ad614SDennis Dalessandro u32 opcode; 365f48ad614SDennis Dalessandro void *ebuf = NULL; 366f48ad614SDennis Dalessandro __be32 *bth = NULL; 367f48ad614SDennis Dalessandro 368f48ad614SDennis Dalessandro if (rhf_use_egr_bfr(packet->rhf)) 369f48ad614SDennis Dalessandro ebuf = packet->ebuf; 370f48ad614SDennis Dalessandro 371f48ad614SDennis Dalessandro if (!ebuf) 372f48ad614SDennis Dalessandro goto drop; /* this should never happen */ 373f48ad614SDennis Dalessandro 374f48ad614SDennis Dalessandro if (lnh == HFI1_LRH_BTH) 375f48ad614SDennis Dalessandro bth = (__be32 *)ebuf; 376f48ad614SDennis Dalessandro else if (lnh == HFI1_LRH_GRH) 377f48ad614SDennis Dalessandro bth = (__be32 *)((char *)ebuf + sizeof(struct ib_grh)); 378f48ad614SDennis Dalessandro else 379f48ad614SDennis Dalessandro goto drop; 380f48ad614SDennis Dalessandro 381f48ad614SDennis Dalessandro opcode = be32_to_cpu(bth[0]) >> 24; 382f48ad614SDennis Dalessandro opcode &= 0xff; 383f48ad614SDennis Dalessandro 384f48ad614SDennis Dalessandro if (opcode == IB_OPCODE_CNP) { 385f48ad614SDennis Dalessandro /* 386f48ad614SDennis Dalessandro * Only in pre-B0 h/w is the CNP_OPCODE handled 387f48ad614SDennis Dalessandro * via this code path. 388f48ad614SDennis Dalessandro */ 389f48ad614SDennis Dalessandro struct rvt_qp *qp = NULL; 390f48ad614SDennis Dalessandro u32 lqpn, rqpn; 391f48ad614SDennis Dalessandro u16 rlid; 392f48ad614SDennis Dalessandro u8 svc_type, sl, sc5; 393f48ad614SDennis Dalessandro 394b736a469SDasaratharaman Chandramouli sc5 = hdr2sc(rhdr, packet->rhf); 395f48ad614SDennis Dalessandro sl = ibp->sc_to_sl[sc5]; 396f48ad614SDennis Dalessandro 397f48ad614SDennis Dalessandro lqpn = be32_to_cpu(bth[1]) & RVT_QPN_MASK; 398f48ad614SDennis Dalessandro rcu_read_lock(); 399f48ad614SDennis Dalessandro qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn); 400f48ad614SDennis Dalessandro if (!qp) { 401f48ad614SDennis Dalessandro rcu_read_unlock(); 402f48ad614SDennis Dalessandro goto drop; 403f48ad614SDennis Dalessandro } 404f48ad614SDennis Dalessandro 405f48ad614SDennis Dalessandro switch (qp->ibqp.qp_type) { 406f48ad614SDennis Dalessandro case IB_QPT_UD: 407f48ad614SDennis Dalessandro rlid = 0; 408f48ad614SDennis Dalessandro rqpn = 0; 409f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UD; 410f48ad614SDennis Dalessandro break; 411f48ad614SDennis Dalessandro case IB_QPT_UC: 412f48ad614SDennis Dalessandro rlid = be16_to_cpu(rhdr->lrh[3]); 413f48ad614SDennis Dalessandro rqpn = qp->remote_qpn; 414f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UC; 415f48ad614SDennis Dalessandro break; 416f48ad614SDennis Dalessandro default: 417f48ad614SDennis Dalessandro goto drop; 418f48ad614SDennis Dalessandro } 419f48ad614SDennis Dalessandro 420f48ad614SDennis Dalessandro process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 421f48ad614SDennis Dalessandro rcu_read_unlock(); 422f48ad614SDennis Dalessandro } 423f48ad614SDennis Dalessandro 424f48ad614SDennis Dalessandro packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; 425f48ad614SDennis Dalessandro break; 426f48ad614SDennis Dalessandro } 427f48ad614SDennis Dalessandro default: 428f48ad614SDennis Dalessandro break; 429f48ad614SDennis Dalessandro } 430f48ad614SDennis Dalessandro 431f48ad614SDennis Dalessandro drop: 432f48ad614SDennis Dalessandro return; 433f48ad614SDennis Dalessandro } 434f48ad614SDennis Dalessandro 435f48ad614SDennis Dalessandro static inline void init_packet(struct hfi1_ctxtdata *rcd, 436f48ad614SDennis Dalessandro struct hfi1_packet *packet) 437f48ad614SDennis Dalessandro { 438f48ad614SDennis Dalessandro packet->rsize = rcd->rcvhdrqentsize; /* words */ 439f48ad614SDennis Dalessandro packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */ 440f48ad614SDennis Dalessandro packet->rcd = rcd; 441f48ad614SDennis Dalessandro packet->updegr = 0; 442f48ad614SDennis Dalessandro packet->etail = -1; 443f48ad614SDennis Dalessandro packet->rhf_addr = get_rhf_addr(rcd); 444f48ad614SDennis Dalessandro packet->rhf = rhf_to_cpu(packet->rhf_addr); 445f48ad614SDennis Dalessandro packet->rhqoff = rcd->head; 446f48ad614SDennis Dalessandro packet->numpkt = 0; 447f48ad614SDennis Dalessandro packet->rcv_flags = 0; 448f48ad614SDennis Dalessandro } 449f48ad614SDennis Dalessandro 4505fd2b562SMitko Haralanov void hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, 4515fd2b562SMitko Haralanov bool do_cnp) 452f48ad614SDennis Dalessandro { 453f48ad614SDennis Dalessandro struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 454261a4351SMike Marciniszyn struct ib_header *hdr = pkt->hdr; 455261a4351SMike Marciniszyn struct ib_other_headers *ohdr = pkt->ohdr; 4565fd2b562SMitko Haralanov struct ib_grh *grh = NULL; 4575fd2b562SMitko Haralanov u32 rqpn = 0, bth1; 4585fd2b562SMitko Haralanov u16 rlid, dlid = be16_to_cpu(hdr->lrh[1]); 4595fd2b562SMitko Haralanov u8 sc, svc_type; 4605fd2b562SMitko Haralanov bool is_mcast = false; 4615fd2b562SMitko Haralanov 4625fd2b562SMitko Haralanov if (pkt->rcv_flags & HFI1_HAS_GRH) 4635fd2b562SMitko Haralanov grh = &hdr->u.l.grh; 464f48ad614SDennis Dalessandro 465f48ad614SDennis Dalessandro switch (qp->ibqp.qp_type) { 466f48ad614SDennis Dalessandro case IB_QPT_SMI: 467f48ad614SDennis Dalessandro case IB_QPT_GSI: 468f48ad614SDennis Dalessandro case IB_QPT_UD: 469f48ad614SDennis Dalessandro rlid = be16_to_cpu(hdr->lrh[3]); 470f48ad614SDennis Dalessandro rqpn = be32_to_cpu(ohdr->u.ud.deth[1]) & RVT_QPN_MASK; 471f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UD; 4725fd2b562SMitko Haralanov is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) && 4735fd2b562SMitko Haralanov (dlid != be16_to_cpu(IB_LID_PERMISSIVE)); 474f48ad614SDennis Dalessandro break; 475f48ad614SDennis Dalessandro case IB_QPT_UC: 476f48ad614SDennis Dalessandro rlid = qp->remote_ah_attr.dlid; 477f48ad614SDennis Dalessandro rqpn = qp->remote_qpn; 478f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UC; 479f48ad614SDennis Dalessandro break; 480f48ad614SDennis Dalessandro case IB_QPT_RC: 481f48ad614SDennis Dalessandro rlid = qp->remote_ah_attr.dlid; 482f48ad614SDennis Dalessandro rqpn = qp->remote_qpn; 483f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_RC; 484f48ad614SDennis Dalessandro break; 485f48ad614SDennis Dalessandro default: 486f48ad614SDennis Dalessandro return; 487f48ad614SDennis Dalessandro } 488f48ad614SDennis Dalessandro 489261a4351SMike Marciniszyn sc = hdr2sc(hdr, pkt->rhf); 490f48ad614SDennis Dalessandro 4915fd2b562SMitko Haralanov bth1 = be32_to_cpu(ohdr->bth[1]); 4925fd2b562SMitko Haralanov if (do_cnp && (bth1 & HFI1_FECN_SMASK)) { 493f48ad614SDennis Dalessandro u16 pkey = (u16)be32_to_cpu(ohdr->bth[0]); 494f48ad614SDennis Dalessandro 4955fd2b562SMitko Haralanov return_cnp(ibp, qp, rqpn, pkey, dlid, rlid, sc, grh); 496f48ad614SDennis Dalessandro } 497f48ad614SDennis Dalessandro 4985fd2b562SMitko Haralanov if (!is_mcast && (bth1 & HFI1_BECN_SMASK)) { 499f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 500f48ad614SDennis Dalessandro u32 lqpn = bth1 & RVT_QPN_MASK; 5015fd2b562SMitko Haralanov u8 sl = ibp->sc_to_sl[sc]; 502f48ad614SDennis Dalessandro 503f48ad614SDennis Dalessandro process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 504f48ad614SDennis Dalessandro } 5055fd2b562SMitko Haralanov 506f48ad614SDennis Dalessandro } 507f48ad614SDennis Dalessandro 508f48ad614SDennis Dalessandro struct ps_mdata { 509f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd; 510f48ad614SDennis Dalessandro u32 rsize; 511f48ad614SDennis Dalessandro u32 maxcnt; 512f48ad614SDennis Dalessandro u32 ps_head; 513f48ad614SDennis Dalessandro u32 ps_tail; 514f48ad614SDennis Dalessandro u32 ps_seq; 515f48ad614SDennis Dalessandro }; 516f48ad614SDennis Dalessandro 517f48ad614SDennis Dalessandro static inline void init_ps_mdata(struct ps_mdata *mdata, 518f48ad614SDennis Dalessandro struct hfi1_packet *packet) 519f48ad614SDennis Dalessandro { 520f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 521f48ad614SDennis Dalessandro 522f48ad614SDennis Dalessandro mdata->rcd = rcd; 523f48ad614SDennis Dalessandro mdata->rsize = packet->rsize; 524f48ad614SDennis Dalessandro mdata->maxcnt = packet->maxcnt; 525f48ad614SDennis Dalessandro mdata->ps_head = packet->rhqoff; 526f48ad614SDennis Dalessandro 527f48ad614SDennis Dalessandro if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 528f48ad614SDennis Dalessandro mdata->ps_tail = get_rcvhdrtail(rcd); 529f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT) 530f48ad614SDennis Dalessandro mdata->ps_seq = rcd->seq_cnt; 531f48ad614SDennis Dalessandro else 532f48ad614SDennis Dalessandro mdata->ps_seq = 0; /* not used with DMA_RTAIL */ 533f48ad614SDennis Dalessandro } else { 534f48ad614SDennis Dalessandro mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ 535f48ad614SDennis Dalessandro mdata->ps_seq = rcd->seq_cnt; 536f48ad614SDennis Dalessandro } 537f48ad614SDennis Dalessandro } 538f48ad614SDennis Dalessandro 539f48ad614SDennis Dalessandro static inline int ps_done(struct ps_mdata *mdata, u64 rhf, 540f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 541f48ad614SDennis Dalessandro { 542f48ad614SDennis Dalessandro if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 543f48ad614SDennis Dalessandro return mdata->ps_head == mdata->ps_tail; 544f48ad614SDennis Dalessandro return mdata->ps_seq != rhf_rcv_seq(rhf); 545f48ad614SDennis Dalessandro } 546f48ad614SDennis Dalessandro 547f48ad614SDennis Dalessandro static inline int ps_skip(struct ps_mdata *mdata, u64 rhf, 548f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 549f48ad614SDennis Dalessandro { 550f48ad614SDennis Dalessandro /* 551f48ad614SDennis Dalessandro * Control context can potentially receive an invalid rhf. 552f48ad614SDennis Dalessandro * Drop such packets. 553f48ad614SDennis Dalessandro */ 554f48ad614SDennis Dalessandro if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail)) 555f48ad614SDennis Dalessandro return mdata->ps_seq != rhf_rcv_seq(rhf); 556f48ad614SDennis Dalessandro 557f48ad614SDennis Dalessandro return 0; 558f48ad614SDennis Dalessandro } 559f48ad614SDennis Dalessandro 560f48ad614SDennis Dalessandro static inline void update_ps_mdata(struct ps_mdata *mdata, 561f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 562f48ad614SDennis Dalessandro { 563f48ad614SDennis Dalessandro mdata->ps_head += mdata->rsize; 564f48ad614SDennis Dalessandro if (mdata->ps_head >= mdata->maxcnt) 565f48ad614SDennis Dalessandro mdata->ps_head = 0; 566f48ad614SDennis Dalessandro 567f48ad614SDennis Dalessandro /* Control context must do seq counting */ 568f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || 569f48ad614SDennis Dalessandro (rcd->ctxt == HFI1_CTRL_CTXT)) { 570f48ad614SDennis Dalessandro if (++mdata->ps_seq > 13) 571f48ad614SDennis Dalessandro mdata->ps_seq = 1; 572f48ad614SDennis Dalessandro } 573f48ad614SDennis Dalessandro } 574f48ad614SDennis Dalessandro 575f48ad614SDennis Dalessandro /* 576f48ad614SDennis Dalessandro * prescan_rxq - search through the receive queue looking for packets 577f48ad614SDennis Dalessandro * containing Excplicit Congestion Notifications (FECNs, or BECNs). 578f48ad614SDennis Dalessandro * When an ECN is found, process the Congestion Notification, and toggle 579f48ad614SDennis Dalessandro * it off. 580f48ad614SDennis Dalessandro * This is declared as a macro to allow quick checking of the port to avoid 581f48ad614SDennis Dalessandro * the overhead of a function call if not enabled. 582f48ad614SDennis Dalessandro */ 583f48ad614SDennis Dalessandro #define prescan_rxq(rcd, packet) \ 584f48ad614SDennis Dalessandro do { \ 585f48ad614SDennis Dalessandro if (rcd->ppd->cc_prescan) \ 586f48ad614SDennis Dalessandro __prescan_rxq(packet); \ 587f48ad614SDennis Dalessandro } while (0) 588f48ad614SDennis Dalessandro static void __prescan_rxq(struct hfi1_packet *packet) 589f48ad614SDennis Dalessandro { 590f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 591f48ad614SDennis Dalessandro struct ps_mdata mdata; 592f48ad614SDennis Dalessandro 593f48ad614SDennis Dalessandro init_ps_mdata(&mdata, packet); 594f48ad614SDennis Dalessandro 595f48ad614SDennis Dalessandro while (1) { 596f48ad614SDennis Dalessandro struct hfi1_devdata *dd = rcd->dd; 597f48ad614SDennis Dalessandro struct hfi1_ibport *ibp = &rcd->ppd->ibport_data; 598f48ad614SDennis Dalessandro __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 599f48ad614SDennis Dalessandro dd->rhf_offset; 600f48ad614SDennis Dalessandro struct rvt_qp *qp; 601261a4351SMike Marciniszyn struct ib_header *hdr; 602f48ad614SDennis Dalessandro struct rvt_dev_info *rdi = &dd->verbs_dev.rdi; 603f48ad614SDennis Dalessandro u64 rhf = rhf_to_cpu(rhf_addr); 604f48ad614SDennis Dalessandro u32 etype = rhf_rcv_type(rhf), qpn, bth1; 605f48ad614SDennis Dalessandro int is_ecn = 0; 606f48ad614SDennis Dalessandro u8 lnh; 607f48ad614SDennis Dalessandro 608f48ad614SDennis Dalessandro if (ps_done(&mdata, rhf, rcd)) 609f48ad614SDennis Dalessandro break; 610f48ad614SDennis Dalessandro 611f48ad614SDennis Dalessandro if (ps_skip(&mdata, rhf, rcd)) 612f48ad614SDennis Dalessandro goto next; 613f48ad614SDennis Dalessandro 614f48ad614SDennis Dalessandro if (etype != RHF_RCV_TYPE_IB) 615f48ad614SDennis Dalessandro goto next; 616f48ad614SDennis Dalessandro 617f2d8a0b3SDasaratharaman Chandramouli packet->hdr = hfi1_get_msgheader(dd, rhf_addr); 618f2d8a0b3SDasaratharaman Chandramouli hdr = packet->hdr; 619261a4351SMike Marciniszyn 620f48ad614SDennis Dalessandro lnh = be16_to_cpu(hdr->lrh[0]) & 3; 621f48ad614SDennis Dalessandro 622f2d8a0b3SDasaratharaman Chandramouli if (lnh == HFI1_LRH_BTH) { 623f2d8a0b3SDasaratharaman Chandramouli packet->ohdr = &hdr->u.oth; 624f2d8a0b3SDasaratharaman Chandramouli } else if (lnh == HFI1_LRH_GRH) { 625f2d8a0b3SDasaratharaman Chandramouli packet->ohdr = &hdr->u.l.oth; 626f2d8a0b3SDasaratharaman Chandramouli packet->rcv_flags |= HFI1_HAS_GRH; 627f2d8a0b3SDasaratharaman Chandramouli } else { 628f48ad614SDennis Dalessandro goto next; /* just in case */ 629f2d8a0b3SDasaratharaman Chandramouli } 6305fd2b562SMitko Haralanov 631f2d8a0b3SDasaratharaman Chandramouli bth1 = be32_to_cpu(packet->ohdr->bth[1]); 632f48ad614SDennis Dalessandro is_ecn = !!(bth1 & (HFI1_FECN_SMASK | HFI1_BECN_SMASK)); 633f48ad614SDennis Dalessandro 634f48ad614SDennis Dalessandro if (!is_ecn) 635f48ad614SDennis Dalessandro goto next; 636f48ad614SDennis Dalessandro 637f48ad614SDennis Dalessandro qpn = bth1 & RVT_QPN_MASK; 638f48ad614SDennis Dalessandro rcu_read_lock(); 639f48ad614SDennis Dalessandro qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn); 640f48ad614SDennis Dalessandro 641f48ad614SDennis Dalessandro if (!qp) { 642f48ad614SDennis Dalessandro rcu_read_unlock(); 643f48ad614SDennis Dalessandro goto next; 644f48ad614SDennis Dalessandro } 645f48ad614SDennis Dalessandro 6465fd2b562SMitko Haralanov process_ecn(qp, packet, true); 647f48ad614SDennis Dalessandro rcu_read_unlock(); 648f48ad614SDennis Dalessandro 649f48ad614SDennis Dalessandro /* turn off BECN, FECN */ 650f48ad614SDennis Dalessandro bth1 &= ~(HFI1_FECN_SMASK | HFI1_BECN_SMASK); 651f2d8a0b3SDasaratharaman Chandramouli packet->ohdr->bth[1] = cpu_to_be32(bth1); 652f48ad614SDennis Dalessandro next: 653f48ad614SDennis Dalessandro update_ps_mdata(&mdata, rcd); 654f48ad614SDennis Dalessandro } 655f48ad614SDennis Dalessandro } 656f48ad614SDennis Dalessandro 657f48ad614SDennis Dalessandro static inline int skip_rcv_packet(struct hfi1_packet *packet, int thread) 658f48ad614SDennis Dalessandro { 659f48ad614SDennis Dalessandro int ret = RCV_PKT_OK; 660f48ad614SDennis Dalessandro 661f48ad614SDennis Dalessandro /* Set up for the next packet */ 662f48ad614SDennis Dalessandro packet->rhqoff += packet->rsize; 663f48ad614SDennis Dalessandro if (packet->rhqoff >= packet->maxcnt) 664f48ad614SDennis Dalessandro packet->rhqoff = 0; 665f48ad614SDennis Dalessandro 666f48ad614SDennis Dalessandro packet->numpkt++; 667f48ad614SDennis Dalessandro if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) { 668f48ad614SDennis Dalessandro if (thread) { 669f48ad614SDennis Dalessandro cond_resched(); 670f48ad614SDennis Dalessandro } else { 671f48ad614SDennis Dalessandro ret = RCV_PKT_LIMIT; 672f48ad614SDennis Dalessandro this_cpu_inc(*packet->rcd->dd->rcv_limit); 673f48ad614SDennis Dalessandro } 674f48ad614SDennis Dalessandro } 675f48ad614SDennis Dalessandro 676f48ad614SDennis Dalessandro packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 677f48ad614SDennis Dalessandro packet->rcd->dd->rhf_offset; 678f48ad614SDennis Dalessandro packet->rhf = rhf_to_cpu(packet->rhf_addr); 679f48ad614SDennis Dalessandro 680f48ad614SDennis Dalessandro return ret; 681f48ad614SDennis Dalessandro } 682f48ad614SDennis Dalessandro 683f48ad614SDennis Dalessandro static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) 684f48ad614SDennis Dalessandro { 685f48ad614SDennis Dalessandro int ret = RCV_PKT_OK; 686f48ad614SDennis Dalessandro 687f48ad614SDennis Dalessandro packet->hdr = hfi1_get_msgheader(packet->rcd->dd, 688f48ad614SDennis Dalessandro packet->rhf_addr); 689f48ad614SDennis Dalessandro packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; 690f48ad614SDennis Dalessandro packet->etype = rhf_rcv_type(packet->rhf); 691f48ad614SDennis Dalessandro /* total length */ 692f48ad614SDennis Dalessandro packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 693f48ad614SDennis Dalessandro /* retrieve eager buffer details */ 694f48ad614SDennis Dalessandro packet->ebuf = NULL; 695f48ad614SDennis Dalessandro if (rhf_use_egr_bfr(packet->rhf)) { 696f48ad614SDennis Dalessandro packet->etail = rhf_egr_index(packet->rhf); 697f48ad614SDennis Dalessandro packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, 698f48ad614SDennis Dalessandro &packet->updegr); 699f48ad614SDennis Dalessandro /* 700f48ad614SDennis Dalessandro * Prefetch the contents of the eager buffer. It is 701f48ad614SDennis Dalessandro * OK to send a negative length to prefetch_range(). 702f48ad614SDennis Dalessandro * The +2 is the size of the RHF. 703f48ad614SDennis Dalessandro */ 704f48ad614SDennis Dalessandro prefetch_range(packet->ebuf, 705f48ad614SDennis Dalessandro packet->tlen - ((packet->rcd->rcvhdrqentsize - 706f48ad614SDennis Dalessandro (rhf_hdrq_offset(packet->rhf) 707f48ad614SDennis Dalessandro + 2)) * 4)); 708f48ad614SDennis Dalessandro } 709f48ad614SDennis Dalessandro 710f48ad614SDennis Dalessandro /* 711f48ad614SDennis Dalessandro * Call a type specific handler for the packet. We 712f48ad614SDennis Dalessandro * should be able to trust that etype won't be beyond 713f48ad614SDennis Dalessandro * the range of valid indexes. If so something is really 714f48ad614SDennis Dalessandro * wrong and we can probably just let things come 715f48ad614SDennis Dalessandro * crashing down. There is no need to eat another 716f48ad614SDennis Dalessandro * comparison in this performance critical code. 717f48ad614SDennis Dalessandro */ 718f48ad614SDennis Dalessandro packet->rcd->dd->rhf_rcv_function_map[packet->etype](packet); 719f48ad614SDennis Dalessandro packet->numpkt++; 720f48ad614SDennis Dalessandro 721f48ad614SDennis Dalessandro /* Set up for the next packet */ 722f48ad614SDennis Dalessandro packet->rhqoff += packet->rsize; 723f48ad614SDennis Dalessandro if (packet->rhqoff >= packet->maxcnt) 724f48ad614SDennis Dalessandro packet->rhqoff = 0; 725f48ad614SDennis Dalessandro 726f48ad614SDennis Dalessandro if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) { 727f48ad614SDennis Dalessandro if (thread) { 728f48ad614SDennis Dalessandro cond_resched(); 729f48ad614SDennis Dalessandro } else { 730f48ad614SDennis Dalessandro ret = RCV_PKT_LIMIT; 731f48ad614SDennis Dalessandro this_cpu_inc(*packet->rcd->dd->rcv_limit); 732f48ad614SDennis Dalessandro } 733f48ad614SDennis Dalessandro } 734f48ad614SDennis Dalessandro 735f48ad614SDennis Dalessandro packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 736f48ad614SDennis Dalessandro packet->rcd->dd->rhf_offset; 737f48ad614SDennis Dalessandro packet->rhf = rhf_to_cpu(packet->rhf_addr); 738f48ad614SDennis Dalessandro 739f48ad614SDennis Dalessandro return ret; 740f48ad614SDennis Dalessandro } 741f48ad614SDennis Dalessandro 742f48ad614SDennis Dalessandro static inline void process_rcv_update(int last, struct hfi1_packet *packet) 743f48ad614SDennis Dalessandro { 744f48ad614SDennis Dalessandro /* 745f48ad614SDennis Dalessandro * Update head regs etc., every 16 packets, if not last pkt, 746f48ad614SDennis Dalessandro * to help prevent rcvhdrq overflows, when many packets 747f48ad614SDennis Dalessandro * are processed and queue is nearly full. 748f48ad614SDennis Dalessandro * Don't request an interrupt for intermediate updates. 749f48ad614SDennis Dalessandro */ 750f48ad614SDennis Dalessandro if (!last && !(packet->numpkt & 0xf)) { 751f48ad614SDennis Dalessandro update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, 752f48ad614SDennis Dalessandro packet->etail, 0, 0); 753f48ad614SDennis Dalessandro packet->updegr = 0; 754f48ad614SDennis Dalessandro } 755f48ad614SDennis Dalessandro packet->rcv_flags = 0; 756f48ad614SDennis Dalessandro } 757f48ad614SDennis Dalessandro 758f48ad614SDennis Dalessandro static inline void finish_packet(struct hfi1_packet *packet) 759f48ad614SDennis Dalessandro { 760f48ad614SDennis Dalessandro /* 761f48ad614SDennis Dalessandro * Nothing we need to free for the packet. 762f48ad614SDennis Dalessandro * 763f48ad614SDennis Dalessandro * The only thing we need to do is a final update and call for an 764f48ad614SDennis Dalessandro * interrupt 765f48ad614SDennis Dalessandro */ 766f48ad614SDennis Dalessandro update_usrhead(packet->rcd, packet->rcd->head, packet->updegr, 767f48ad614SDennis Dalessandro packet->etail, rcv_intr_dynamic, packet->numpkt); 768f48ad614SDennis Dalessandro } 769f48ad614SDennis Dalessandro 770f48ad614SDennis Dalessandro static inline void process_rcv_qp_work(struct hfi1_packet *packet) 771f48ad614SDennis Dalessandro { 772f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd; 773f48ad614SDennis Dalessandro struct rvt_qp *qp, *nqp; 774f48ad614SDennis Dalessandro 775f48ad614SDennis Dalessandro rcd = packet->rcd; 776f48ad614SDennis Dalessandro rcd->head = packet->rhqoff; 777f48ad614SDennis Dalessandro 778f48ad614SDennis Dalessandro /* 779f48ad614SDennis Dalessandro * Iterate over all QPs waiting to respond. 780f48ad614SDennis Dalessandro * The list won't change since the IRQ is only run on one CPU. 781f48ad614SDennis Dalessandro */ 782f48ad614SDennis Dalessandro list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { 783f48ad614SDennis Dalessandro list_del_init(&qp->rspwait); 784f48ad614SDennis Dalessandro if (qp->r_flags & RVT_R_RSP_NAK) { 785f48ad614SDennis Dalessandro qp->r_flags &= ~RVT_R_RSP_NAK; 786f48ad614SDennis Dalessandro hfi1_send_rc_ack(rcd, qp, 0); 787f48ad614SDennis Dalessandro } 788f48ad614SDennis Dalessandro if (qp->r_flags & RVT_R_RSP_SEND) { 789f48ad614SDennis Dalessandro unsigned long flags; 790f48ad614SDennis Dalessandro 791f48ad614SDennis Dalessandro qp->r_flags &= ~RVT_R_RSP_SEND; 792f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->s_lock, flags); 793f48ad614SDennis Dalessandro if (ib_rvt_state_ops[qp->state] & 794f48ad614SDennis Dalessandro RVT_PROCESS_OR_FLUSH_SEND) 795f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 796f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 797f48ad614SDennis Dalessandro } 798f48ad614SDennis Dalessandro if (atomic_dec_and_test(&qp->refcount)) 799f48ad614SDennis Dalessandro wake_up(&qp->wait); 800f48ad614SDennis Dalessandro } 801f48ad614SDennis Dalessandro } 802f48ad614SDennis Dalessandro 803f48ad614SDennis Dalessandro /* 804f48ad614SDennis Dalessandro * Handle receive interrupts when using the no dma rtail option. 805f48ad614SDennis Dalessandro */ 806f48ad614SDennis Dalessandro int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) 807f48ad614SDennis Dalessandro { 808f48ad614SDennis Dalessandro u32 seq; 809f48ad614SDennis Dalessandro int last = RCV_PKT_OK; 810f48ad614SDennis Dalessandro struct hfi1_packet packet; 811f48ad614SDennis Dalessandro 812f48ad614SDennis Dalessandro init_packet(rcd, &packet); 813f48ad614SDennis Dalessandro seq = rhf_rcv_seq(packet.rhf); 814f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) { 815f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 816f48ad614SDennis Dalessandro goto bail; 817f48ad614SDennis Dalessandro } 818f48ad614SDennis Dalessandro 819f48ad614SDennis Dalessandro prescan_rxq(rcd, &packet); 820f48ad614SDennis Dalessandro 821f48ad614SDennis Dalessandro while (last == RCV_PKT_OK) { 822f48ad614SDennis Dalessandro last = process_rcv_packet(&packet, thread); 823f48ad614SDennis Dalessandro seq = rhf_rcv_seq(packet.rhf); 824f48ad614SDennis Dalessandro if (++rcd->seq_cnt > 13) 825f48ad614SDennis Dalessandro rcd->seq_cnt = 1; 826f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) 827f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 828f48ad614SDennis Dalessandro process_rcv_update(last, &packet); 829f48ad614SDennis Dalessandro } 830f48ad614SDennis Dalessandro process_rcv_qp_work(&packet); 831f48ad614SDennis Dalessandro bail: 832f48ad614SDennis Dalessandro finish_packet(&packet); 833f48ad614SDennis Dalessandro return last; 834f48ad614SDennis Dalessandro } 835f48ad614SDennis Dalessandro 836f48ad614SDennis Dalessandro int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) 837f48ad614SDennis Dalessandro { 838f48ad614SDennis Dalessandro u32 hdrqtail; 839f48ad614SDennis Dalessandro int last = RCV_PKT_OK; 840f48ad614SDennis Dalessandro struct hfi1_packet packet; 841f48ad614SDennis Dalessandro 842f48ad614SDennis Dalessandro init_packet(rcd, &packet); 843f48ad614SDennis Dalessandro hdrqtail = get_rcvhdrtail(rcd); 844f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) { 845f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 846f48ad614SDennis Dalessandro goto bail; 847f48ad614SDennis Dalessandro } 848f48ad614SDennis Dalessandro smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 849f48ad614SDennis Dalessandro 850f48ad614SDennis Dalessandro prescan_rxq(rcd, &packet); 851f48ad614SDennis Dalessandro 852f48ad614SDennis Dalessandro while (last == RCV_PKT_OK) { 853f48ad614SDennis Dalessandro last = process_rcv_packet(&packet, thread); 854f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) 855f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 856f48ad614SDennis Dalessandro process_rcv_update(last, &packet); 857f48ad614SDennis Dalessandro } 858f48ad614SDennis Dalessandro process_rcv_qp_work(&packet); 859f48ad614SDennis Dalessandro bail: 860f48ad614SDennis Dalessandro finish_packet(&packet); 861f48ad614SDennis Dalessandro return last; 862f48ad614SDennis Dalessandro } 863f48ad614SDennis Dalessandro 864f48ad614SDennis Dalessandro static inline void set_all_nodma_rtail(struct hfi1_devdata *dd) 865f48ad614SDennis Dalessandro { 866f48ad614SDennis Dalessandro int i; 867f48ad614SDennis Dalessandro 868f48ad614SDennis Dalessandro for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++) 869f48ad614SDennis Dalessandro dd->rcd[i]->do_interrupt = 870f48ad614SDennis Dalessandro &handle_receive_interrupt_nodma_rtail; 871f48ad614SDennis Dalessandro } 872f48ad614SDennis Dalessandro 873f48ad614SDennis Dalessandro static inline void set_all_dma_rtail(struct hfi1_devdata *dd) 874f48ad614SDennis Dalessandro { 875f48ad614SDennis Dalessandro int i; 876f48ad614SDennis Dalessandro 877f48ad614SDennis Dalessandro for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++) 878f48ad614SDennis Dalessandro dd->rcd[i]->do_interrupt = 879f48ad614SDennis Dalessandro &handle_receive_interrupt_dma_rtail; 880f48ad614SDennis Dalessandro } 881f48ad614SDennis Dalessandro 882f48ad614SDennis Dalessandro void set_all_slowpath(struct hfi1_devdata *dd) 883f48ad614SDennis Dalessandro { 884f48ad614SDennis Dalessandro int i; 885f48ad614SDennis Dalessandro 886f48ad614SDennis Dalessandro /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */ 887f48ad614SDennis Dalessandro for (i = HFI1_CTRL_CTXT + 1; i < dd->first_user_ctxt; i++) 888f48ad614SDennis Dalessandro dd->rcd[i]->do_interrupt = &handle_receive_interrupt; 889f48ad614SDennis Dalessandro } 890f48ad614SDennis Dalessandro 891f48ad614SDennis Dalessandro static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, 892c867caafSMike Marciniszyn struct hfi1_packet *packet, 893f48ad614SDennis Dalessandro struct hfi1_devdata *dd) 894f48ad614SDennis Dalessandro { 895f48ad614SDennis Dalessandro struct work_struct *lsaw = &rcd->ppd->linkstate_active_work; 896261a4351SMike Marciniszyn struct ib_header *hdr = hfi1_get_msgheader(packet->rcd->dd, 897c867caafSMike Marciniszyn packet->rhf_addr); 89869b9f4a4SMike Marciniszyn u8 etype = rhf_rcv_type(packet->rhf); 899f48ad614SDennis Dalessandro 90069b9f4a4SMike Marciniszyn if (etype == RHF_RCV_TYPE_IB && hdr2sc(hdr, packet->rhf) != 0xf) { 901f48ad614SDennis Dalessandro int hwstate = read_logical_state(dd); 902f48ad614SDennis Dalessandro 903f48ad614SDennis Dalessandro if (hwstate != LSTATE_ACTIVE) { 904f48ad614SDennis Dalessandro dd_dev_info(dd, "Unexpected link state %d\n", hwstate); 905f48ad614SDennis Dalessandro return 0; 906f48ad614SDennis Dalessandro } 907f48ad614SDennis Dalessandro 908f48ad614SDennis Dalessandro queue_work(rcd->ppd->hfi1_wq, lsaw); 909f48ad614SDennis Dalessandro return 1; 910f48ad614SDennis Dalessandro } 911f48ad614SDennis Dalessandro return 0; 912f48ad614SDennis Dalessandro } 913f48ad614SDennis Dalessandro 914f48ad614SDennis Dalessandro /* 915f48ad614SDennis Dalessandro * handle_receive_interrupt - receive a packet 916f48ad614SDennis Dalessandro * @rcd: the context 917f48ad614SDennis Dalessandro * 918f48ad614SDennis Dalessandro * Called from interrupt handler for errors or receive interrupt. 919f48ad614SDennis Dalessandro * This is the slow path interrupt handler. 920f48ad614SDennis Dalessandro */ 921f48ad614SDennis Dalessandro int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) 922f48ad614SDennis Dalessandro { 923f48ad614SDennis Dalessandro struct hfi1_devdata *dd = rcd->dd; 924f48ad614SDennis Dalessandro u32 hdrqtail; 925f48ad614SDennis Dalessandro int needset, last = RCV_PKT_OK; 926f48ad614SDennis Dalessandro struct hfi1_packet packet; 927f48ad614SDennis Dalessandro int skip_pkt = 0; 928f48ad614SDennis Dalessandro 929f48ad614SDennis Dalessandro /* Control context will always use the slow path interrupt handler */ 930f48ad614SDennis Dalessandro needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1; 931f48ad614SDennis Dalessandro 932f48ad614SDennis Dalessandro init_packet(rcd, &packet); 933f48ad614SDennis Dalessandro 934f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 935f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 936f48ad614SDennis Dalessandro 937f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) { 938f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 939f48ad614SDennis Dalessandro goto bail; 940f48ad614SDennis Dalessandro } 941f48ad614SDennis Dalessandro hdrqtail = 0; 942f48ad614SDennis Dalessandro } else { 943f48ad614SDennis Dalessandro hdrqtail = get_rcvhdrtail(rcd); 944f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) { 945f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 946f48ad614SDennis Dalessandro goto bail; 947f48ad614SDennis Dalessandro } 948f48ad614SDennis Dalessandro smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 949f48ad614SDennis Dalessandro 950f48ad614SDennis Dalessandro /* 951f48ad614SDennis Dalessandro * Control context can potentially receive an invalid 952f48ad614SDennis Dalessandro * rhf. Drop such packets. 953f48ad614SDennis Dalessandro */ 954f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT) { 955f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 956f48ad614SDennis Dalessandro 957f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) 958f48ad614SDennis Dalessandro skip_pkt = 1; 959f48ad614SDennis Dalessandro } 960f48ad614SDennis Dalessandro } 961f48ad614SDennis Dalessandro 962f48ad614SDennis Dalessandro prescan_rxq(rcd, &packet); 963f48ad614SDennis Dalessandro 964f48ad614SDennis Dalessandro while (last == RCV_PKT_OK) { 965f48ad614SDennis Dalessandro if (unlikely(dd->do_drop && 966f48ad614SDennis Dalessandro atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == 967f48ad614SDennis Dalessandro DROP_PACKET_ON)) { 968f48ad614SDennis Dalessandro dd->do_drop = 0; 969f48ad614SDennis Dalessandro 970f48ad614SDennis Dalessandro /* On to the next packet */ 971f48ad614SDennis Dalessandro packet.rhqoff += packet.rsize; 972f48ad614SDennis Dalessandro packet.rhf_addr = (__le32 *)rcd->rcvhdrq + 973f48ad614SDennis Dalessandro packet.rhqoff + 974f48ad614SDennis Dalessandro dd->rhf_offset; 975f48ad614SDennis Dalessandro packet.rhf = rhf_to_cpu(packet.rhf_addr); 976f48ad614SDennis Dalessandro 977f48ad614SDennis Dalessandro } else if (skip_pkt) { 978f48ad614SDennis Dalessandro last = skip_rcv_packet(&packet, thread); 979f48ad614SDennis Dalessandro skip_pkt = 0; 980f48ad614SDennis Dalessandro } else { 981f48ad614SDennis Dalessandro /* Auto activate link on non-SC15 packet receive */ 982f48ad614SDennis Dalessandro if (unlikely(rcd->ppd->host_link_state == 983f48ad614SDennis Dalessandro HLS_UP_ARMED) && 984c867caafSMike Marciniszyn set_armed_to_active(rcd, &packet, dd)) 985f48ad614SDennis Dalessandro goto bail; 986f48ad614SDennis Dalessandro last = process_rcv_packet(&packet, thread); 987f48ad614SDennis Dalessandro } 988f48ad614SDennis Dalessandro 989f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 990f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 991f48ad614SDennis Dalessandro 992f48ad614SDennis Dalessandro if (++rcd->seq_cnt > 13) 993f48ad614SDennis Dalessandro rcd->seq_cnt = 1; 994f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) 995f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 996f48ad614SDennis Dalessandro if (needset) { 997f48ad614SDennis Dalessandro dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n"); 998f48ad614SDennis Dalessandro set_all_nodma_rtail(dd); 999f48ad614SDennis Dalessandro needset = 0; 1000f48ad614SDennis Dalessandro } 1001f48ad614SDennis Dalessandro } else { 1002f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) 1003f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 1004f48ad614SDennis Dalessandro /* 1005f48ad614SDennis Dalessandro * Control context can potentially receive an invalid 1006f48ad614SDennis Dalessandro * rhf. Drop such packets. 1007f48ad614SDennis Dalessandro */ 1008f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT) { 1009f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 1010f48ad614SDennis Dalessandro 1011f48ad614SDennis Dalessandro if (++rcd->seq_cnt > 13) 1012f48ad614SDennis Dalessandro rcd->seq_cnt = 1; 1013f48ad614SDennis Dalessandro if (!last && (seq != rcd->seq_cnt)) 1014f48ad614SDennis Dalessandro skip_pkt = 1; 1015f48ad614SDennis Dalessandro } 1016f48ad614SDennis Dalessandro 1017f48ad614SDennis Dalessandro if (needset) { 1018f48ad614SDennis Dalessandro dd_dev_info(dd, 1019f48ad614SDennis Dalessandro "Switching to DMA_RTAIL\n"); 1020f48ad614SDennis Dalessandro set_all_dma_rtail(dd); 1021f48ad614SDennis Dalessandro needset = 0; 1022f48ad614SDennis Dalessandro } 1023f48ad614SDennis Dalessandro } 1024f48ad614SDennis Dalessandro 1025f48ad614SDennis Dalessandro process_rcv_update(last, &packet); 1026f48ad614SDennis Dalessandro } 1027f48ad614SDennis Dalessandro 1028f48ad614SDennis Dalessandro process_rcv_qp_work(&packet); 1029f48ad614SDennis Dalessandro 1030f48ad614SDennis Dalessandro bail: 1031f48ad614SDennis Dalessandro /* 1032f48ad614SDennis Dalessandro * Always write head at end, and setup rcv interrupt, even 1033f48ad614SDennis Dalessandro * if no packets were processed. 1034f48ad614SDennis Dalessandro */ 1035f48ad614SDennis Dalessandro finish_packet(&packet); 1036f48ad614SDennis Dalessandro return last; 1037f48ad614SDennis Dalessandro } 1038f48ad614SDennis Dalessandro 1039f48ad614SDennis Dalessandro /* 1040f48ad614SDennis Dalessandro * We may discover in the interrupt that the hardware link state has 1041f48ad614SDennis Dalessandro * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet), 1042f48ad614SDennis Dalessandro * and we need to update the driver's notion of the link state. We cannot 1043f48ad614SDennis Dalessandro * run set_link_state from interrupt context, so we queue this function on 1044f48ad614SDennis Dalessandro * a workqueue. 1045f48ad614SDennis Dalessandro * 1046f48ad614SDennis Dalessandro * We delay the regular interrupt processing until after the state changes 1047f48ad614SDennis Dalessandro * so that the link will be in the correct state by the time any application 1048f48ad614SDennis Dalessandro * we wake up attempts to send a reply to any message it received. 1049f48ad614SDennis Dalessandro * (Subsequent receive interrupts may possibly force the wakeup before we 1050f48ad614SDennis Dalessandro * update the link state.) 1051f48ad614SDennis Dalessandro * 1052f48ad614SDennis Dalessandro * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes 1053f48ad614SDennis Dalessandro * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues, 1054f48ad614SDennis Dalessandro * so we're safe from use-after-free of the rcd. 1055f48ad614SDennis Dalessandro */ 1056f48ad614SDennis Dalessandro void receive_interrupt_work(struct work_struct *work) 1057f48ad614SDennis Dalessandro { 1058f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 1059f48ad614SDennis Dalessandro linkstate_active_work); 1060f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1061f48ad614SDennis Dalessandro int i; 1062f48ad614SDennis Dalessandro 1063f48ad614SDennis Dalessandro /* Received non-SC15 packet implies neighbor_normal */ 1064f48ad614SDennis Dalessandro ppd->neighbor_normal = 1; 1065f48ad614SDennis Dalessandro set_link_state(ppd, HLS_UP_ACTIVE); 1066f48ad614SDennis Dalessandro 1067f48ad614SDennis Dalessandro /* 1068f48ad614SDennis Dalessandro * Interrupt all kernel contexts that could have had an 1069f48ad614SDennis Dalessandro * interrupt during auto activation. 1070f48ad614SDennis Dalessandro */ 1071f48ad614SDennis Dalessandro for (i = HFI1_CTRL_CTXT; i < dd->first_user_ctxt; i++) 1072f48ad614SDennis Dalessandro force_recv_intr(dd->rcd[i]); 1073f48ad614SDennis Dalessandro } 1074f48ad614SDennis Dalessandro 1075f48ad614SDennis Dalessandro /* 1076f48ad614SDennis Dalessandro * Convert a given MTU size to the on-wire MAD packet enumeration. 1077f48ad614SDennis Dalessandro * Return -1 if the size is invalid. 1078f48ad614SDennis Dalessandro */ 1079f48ad614SDennis Dalessandro int mtu_to_enum(u32 mtu, int default_if_bad) 1080f48ad614SDennis Dalessandro { 1081f48ad614SDennis Dalessandro switch (mtu) { 1082f48ad614SDennis Dalessandro case 0: return OPA_MTU_0; 1083f48ad614SDennis Dalessandro case 256: return OPA_MTU_256; 1084f48ad614SDennis Dalessandro case 512: return OPA_MTU_512; 1085f48ad614SDennis Dalessandro case 1024: return OPA_MTU_1024; 1086f48ad614SDennis Dalessandro case 2048: return OPA_MTU_2048; 1087f48ad614SDennis Dalessandro case 4096: return OPA_MTU_4096; 1088f48ad614SDennis Dalessandro case 8192: return OPA_MTU_8192; 1089f48ad614SDennis Dalessandro case 10240: return OPA_MTU_10240; 1090f48ad614SDennis Dalessandro } 1091f48ad614SDennis Dalessandro return default_if_bad; 1092f48ad614SDennis Dalessandro } 1093f48ad614SDennis Dalessandro 1094f48ad614SDennis Dalessandro u16 enum_to_mtu(int mtu) 1095f48ad614SDennis Dalessandro { 1096f48ad614SDennis Dalessandro switch (mtu) { 1097f48ad614SDennis Dalessandro case OPA_MTU_0: return 0; 1098f48ad614SDennis Dalessandro case OPA_MTU_256: return 256; 1099f48ad614SDennis Dalessandro case OPA_MTU_512: return 512; 1100f48ad614SDennis Dalessandro case OPA_MTU_1024: return 1024; 1101f48ad614SDennis Dalessandro case OPA_MTU_2048: return 2048; 1102f48ad614SDennis Dalessandro case OPA_MTU_4096: return 4096; 1103f48ad614SDennis Dalessandro case OPA_MTU_8192: return 8192; 1104f48ad614SDennis Dalessandro case OPA_MTU_10240: return 10240; 1105f48ad614SDennis Dalessandro default: return 0xffff; 1106f48ad614SDennis Dalessandro } 1107f48ad614SDennis Dalessandro } 1108f48ad614SDennis Dalessandro 1109f48ad614SDennis Dalessandro /* 1110f48ad614SDennis Dalessandro * set_mtu - set the MTU 1111f48ad614SDennis Dalessandro * @ppd: the per port data 1112f48ad614SDennis Dalessandro * 1113f48ad614SDennis Dalessandro * We can handle "any" incoming size, the issue here is whether we 1114f48ad614SDennis Dalessandro * need to restrict our outgoing size. We do not deal with what happens 1115f48ad614SDennis Dalessandro * to programs that are already running when the size changes. 1116f48ad614SDennis Dalessandro */ 1117f48ad614SDennis Dalessandro int set_mtu(struct hfi1_pportdata *ppd) 1118f48ad614SDennis Dalessandro { 1119f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1120f48ad614SDennis Dalessandro int i, drain, ret = 0, is_up = 0; 1121f48ad614SDennis Dalessandro 1122f48ad614SDennis Dalessandro ppd->ibmtu = 0; 1123f48ad614SDennis Dalessandro for (i = 0; i < ppd->vls_supported; i++) 1124f48ad614SDennis Dalessandro if (ppd->ibmtu < dd->vld[i].mtu) 1125f48ad614SDennis Dalessandro ppd->ibmtu = dd->vld[i].mtu; 1126f48ad614SDennis Dalessandro ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd); 1127f48ad614SDennis Dalessandro 1128f48ad614SDennis Dalessandro mutex_lock(&ppd->hls_lock); 1129f48ad614SDennis Dalessandro if (ppd->host_link_state == HLS_UP_INIT || 1130f48ad614SDennis Dalessandro ppd->host_link_state == HLS_UP_ARMED || 1131f48ad614SDennis Dalessandro ppd->host_link_state == HLS_UP_ACTIVE) 1132f48ad614SDennis Dalessandro is_up = 1; 1133f48ad614SDennis Dalessandro 1134f48ad614SDennis Dalessandro drain = !is_ax(dd) && is_up; 1135f48ad614SDennis Dalessandro 1136f48ad614SDennis Dalessandro if (drain) 1137f48ad614SDennis Dalessandro /* 1138f48ad614SDennis Dalessandro * MTU is specified per-VL. To ensure that no packet gets 1139f48ad614SDennis Dalessandro * stuck (due, e.g., to the MTU for the packet's VL being 1140f48ad614SDennis Dalessandro * reduced), empty the per-VL FIFOs before adjusting MTU. 1141f48ad614SDennis Dalessandro */ 1142f48ad614SDennis Dalessandro ret = stop_drain_data_vls(dd); 1143f48ad614SDennis Dalessandro 1144f48ad614SDennis Dalessandro if (ret) { 1145f48ad614SDennis Dalessandro dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n", 1146f48ad614SDennis Dalessandro __func__); 1147f48ad614SDennis Dalessandro goto err; 1148f48ad614SDennis Dalessandro } 1149f48ad614SDennis Dalessandro 1150f48ad614SDennis Dalessandro hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0); 1151f48ad614SDennis Dalessandro 1152f48ad614SDennis Dalessandro if (drain) 1153f48ad614SDennis Dalessandro open_fill_data_vls(dd); /* reopen all VLs */ 1154f48ad614SDennis Dalessandro 1155f48ad614SDennis Dalessandro err: 1156f48ad614SDennis Dalessandro mutex_unlock(&ppd->hls_lock); 1157f48ad614SDennis Dalessandro 1158f48ad614SDennis Dalessandro return ret; 1159f48ad614SDennis Dalessandro } 1160f48ad614SDennis Dalessandro 1161f48ad614SDennis Dalessandro int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc) 1162f48ad614SDennis Dalessandro { 1163f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1164f48ad614SDennis Dalessandro 1165f48ad614SDennis Dalessandro ppd->lid = lid; 1166f48ad614SDennis Dalessandro ppd->lmc = lmc; 1167f48ad614SDennis Dalessandro hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0); 1168f48ad614SDennis Dalessandro 1169f48ad614SDennis Dalessandro dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid); 1170f48ad614SDennis Dalessandro 1171f48ad614SDennis Dalessandro return 0; 1172f48ad614SDennis Dalessandro } 1173f48ad614SDennis Dalessandro 1174f48ad614SDennis Dalessandro void shutdown_led_override(struct hfi1_pportdata *ppd) 1175f48ad614SDennis Dalessandro { 1176f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1177f48ad614SDennis Dalessandro 1178f48ad614SDennis Dalessandro /* 1179f48ad614SDennis Dalessandro * This pairs with the memory barrier in hfi1_start_led_override to 1180f48ad614SDennis Dalessandro * ensure that we read the correct state of LED beaconing represented 1181f48ad614SDennis Dalessandro * by led_override_timer_active 1182f48ad614SDennis Dalessandro */ 1183f48ad614SDennis Dalessandro smp_rmb(); 1184f48ad614SDennis Dalessandro if (atomic_read(&ppd->led_override_timer_active)) { 1185f48ad614SDennis Dalessandro del_timer_sync(&ppd->led_override_timer); 1186f48ad614SDennis Dalessandro atomic_set(&ppd->led_override_timer_active, 0); 1187f48ad614SDennis Dalessandro /* Ensure the atomic_set is visible to all CPUs */ 1188f48ad614SDennis Dalessandro smp_wmb(); 1189f48ad614SDennis Dalessandro } 1190f48ad614SDennis Dalessandro 1191f48ad614SDennis Dalessandro /* Hand control of the LED to the DC for normal operation */ 1192f48ad614SDennis Dalessandro write_csr(dd, DCC_CFG_LED_CNTRL, 0); 1193f48ad614SDennis Dalessandro } 1194f48ad614SDennis Dalessandro 1195f48ad614SDennis Dalessandro static void run_led_override(unsigned long opaque) 1196f48ad614SDennis Dalessandro { 1197f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)opaque; 1198f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1199f48ad614SDennis Dalessandro unsigned long timeout; 1200f48ad614SDennis Dalessandro int phase_idx; 1201f48ad614SDennis Dalessandro 1202f48ad614SDennis Dalessandro if (!(dd->flags & HFI1_INITTED)) 1203f48ad614SDennis Dalessandro return; 1204f48ad614SDennis Dalessandro 1205f48ad614SDennis Dalessandro phase_idx = ppd->led_override_phase & 1; 1206f48ad614SDennis Dalessandro 1207f48ad614SDennis Dalessandro setextled(dd, phase_idx); 1208f48ad614SDennis Dalessandro 1209f48ad614SDennis Dalessandro timeout = ppd->led_override_vals[phase_idx]; 1210f48ad614SDennis Dalessandro 1211f48ad614SDennis Dalessandro /* Set up for next phase */ 1212f48ad614SDennis Dalessandro ppd->led_override_phase = !ppd->led_override_phase; 1213f48ad614SDennis Dalessandro 1214f48ad614SDennis Dalessandro mod_timer(&ppd->led_override_timer, jiffies + timeout); 1215f48ad614SDennis Dalessandro } 1216f48ad614SDennis Dalessandro 1217f48ad614SDennis Dalessandro /* 1218f48ad614SDennis Dalessandro * To have the LED blink in a particular pattern, provide timeon and timeoff 1219f48ad614SDennis Dalessandro * in milliseconds. 1220f48ad614SDennis Dalessandro * To turn off custom blinking and return to normal operation, use 1221f48ad614SDennis Dalessandro * shutdown_led_override() 1222f48ad614SDennis Dalessandro */ 1223f48ad614SDennis Dalessandro void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, 1224f48ad614SDennis Dalessandro unsigned int timeoff) 1225f48ad614SDennis Dalessandro { 1226f48ad614SDennis Dalessandro if (!(ppd->dd->flags & HFI1_INITTED)) 1227f48ad614SDennis Dalessandro return; 1228f48ad614SDennis Dalessandro 1229f48ad614SDennis Dalessandro /* Convert to jiffies for direct use in timer */ 1230f48ad614SDennis Dalessandro ppd->led_override_vals[0] = msecs_to_jiffies(timeoff); 1231f48ad614SDennis Dalessandro ppd->led_override_vals[1] = msecs_to_jiffies(timeon); 1232f48ad614SDennis Dalessandro 1233f48ad614SDennis Dalessandro /* Arbitrarily start from LED on phase */ 1234f48ad614SDennis Dalessandro ppd->led_override_phase = 1; 1235f48ad614SDennis Dalessandro 1236f48ad614SDennis Dalessandro /* 1237f48ad614SDennis Dalessandro * If the timer has not already been started, do so. Use a "quick" 1238f48ad614SDennis Dalessandro * timeout so the handler will be called soon to look at our request. 1239f48ad614SDennis Dalessandro */ 1240f48ad614SDennis Dalessandro if (!timer_pending(&ppd->led_override_timer)) { 1241f48ad614SDennis Dalessandro setup_timer(&ppd->led_override_timer, run_led_override, 1242f48ad614SDennis Dalessandro (unsigned long)ppd); 1243f48ad614SDennis Dalessandro ppd->led_override_timer.expires = jiffies + 1; 1244f48ad614SDennis Dalessandro add_timer(&ppd->led_override_timer); 1245f48ad614SDennis Dalessandro atomic_set(&ppd->led_override_timer_active, 1); 1246f48ad614SDennis Dalessandro /* Ensure the atomic_set is visible to all CPUs */ 1247f48ad614SDennis Dalessandro smp_wmb(); 1248f48ad614SDennis Dalessandro } 1249f48ad614SDennis Dalessandro } 1250f48ad614SDennis Dalessandro 1251f48ad614SDennis Dalessandro /** 1252f48ad614SDennis Dalessandro * hfi1_reset_device - reset the chip if possible 1253f48ad614SDennis Dalessandro * @unit: the device to reset 1254f48ad614SDennis Dalessandro * 1255f48ad614SDennis Dalessandro * Whether or not reset is successful, we attempt to re-initialize the chip 1256f48ad614SDennis Dalessandro * (that is, much like a driver unload/reload). We clear the INITTED flag 1257f48ad614SDennis Dalessandro * so that the various entry points will fail until we reinitialize. For 1258f48ad614SDennis Dalessandro * now, we only allow this if no user contexts are open that use chip resources 1259f48ad614SDennis Dalessandro */ 1260f48ad614SDennis Dalessandro int hfi1_reset_device(int unit) 1261f48ad614SDennis Dalessandro { 1262f48ad614SDennis Dalessandro int ret, i; 1263f48ad614SDennis Dalessandro struct hfi1_devdata *dd = hfi1_lookup(unit); 1264f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd; 1265f48ad614SDennis Dalessandro unsigned long flags; 1266f48ad614SDennis Dalessandro int pidx; 1267f48ad614SDennis Dalessandro 1268f48ad614SDennis Dalessandro if (!dd) { 1269f48ad614SDennis Dalessandro ret = -ENODEV; 1270f48ad614SDennis Dalessandro goto bail; 1271f48ad614SDennis Dalessandro } 1272f48ad614SDennis Dalessandro 1273f48ad614SDennis Dalessandro dd_dev_info(dd, "Reset on unit %u requested\n", unit); 1274f48ad614SDennis Dalessandro 1275f48ad614SDennis Dalessandro if (!dd->kregbase || !(dd->flags & HFI1_PRESENT)) { 1276f48ad614SDennis Dalessandro dd_dev_info(dd, 1277f48ad614SDennis Dalessandro "Invalid unit number %u or not initialized or not present\n", 1278f48ad614SDennis Dalessandro unit); 1279f48ad614SDennis Dalessandro ret = -ENXIO; 1280f48ad614SDennis Dalessandro goto bail; 1281f48ad614SDennis Dalessandro } 1282f48ad614SDennis Dalessandro 1283f48ad614SDennis Dalessandro spin_lock_irqsave(&dd->uctxt_lock, flags); 1284f48ad614SDennis Dalessandro if (dd->rcd) 1285f48ad614SDennis Dalessandro for (i = dd->first_user_ctxt; i < dd->num_rcv_contexts; i++) { 1286f48ad614SDennis Dalessandro if (!dd->rcd[i] || !dd->rcd[i]->cnt) 1287f48ad614SDennis Dalessandro continue; 1288f48ad614SDennis Dalessandro spin_unlock_irqrestore(&dd->uctxt_lock, flags); 1289f48ad614SDennis Dalessandro ret = -EBUSY; 1290f48ad614SDennis Dalessandro goto bail; 1291f48ad614SDennis Dalessandro } 1292f48ad614SDennis Dalessandro spin_unlock_irqrestore(&dd->uctxt_lock, flags); 1293f48ad614SDennis Dalessandro 1294f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1295f48ad614SDennis Dalessandro ppd = dd->pport + pidx; 1296f48ad614SDennis Dalessandro 1297f48ad614SDennis Dalessandro shutdown_led_override(ppd); 1298f48ad614SDennis Dalessandro } 1299f48ad614SDennis Dalessandro if (dd->flags & HFI1_HAS_SEND_DMA) 1300f48ad614SDennis Dalessandro sdma_exit(dd); 1301f48ad614SDennis Dalessandro 1302f48ad614SDennis Dalessandro hfi1_reset_cpu_counters(dd); 1303f48ad614SDennis Dalessandro 1304f48ad614SDennis Dalessandro ret = hfi1_init(dd, 1); 1305f48ad614SDennis Dalessandro 1306f48ad614SDennis Dalessandro if (ret) 1307f48ad614SDennis Dalessandro dd_dev_err(dd, 1308f48ad614SDennis Dalessandro "Reinitialize unit %u after reset failed with %d\n", 1309f48ad614SDennis Dalessandro unit, ret); 1310f48ad614SDennis Dalessandro else 1311f48ad614SDennis Dalessandro dd_dev_info(dd, "Reinitialized unit %u after resetting\n", 1312f48ad614SDennis Dalessandro unit); 1313f48ad614SDennis Dalessandro 1314f48ad614SDennis Dalessandro bail: 1315f48ad614SDennis Dalessandro return ret; 1316f48ad614SDennis Dalessandro } 1317f48ad614SDennis Dalessandro 1318f48ad614SDennis Dalessandro void handle_eflags(struct hfi1_packet *packet) 1319f48ad614SDennis Dalessandro { 1320f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 1321f48ad614SDennis Dalessandro u32 rte = rhf_rcv_type_err(packet->rhf); 1322f48ad614SDennis Dalessandro 1323f48ad614SDennis Dalessandro rcv_hdrerr(rcd, rcd->ppd, packet); 1324f48ad614SDennis Dalessandro if (rhf_err_flags(packet->rhf)) 1325f48ad614SDennis Dalessandro dd_dev_err(rcd->dd, 1326f48ad614SDennis Dalessandro "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", 1327f48ad614SDennis Dalessandro rcd->ctxt, packet->rhf, 1328f48ad614SDennis Dalessandro packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", 1329f48ad614SDennis Dalessandro packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", 1330f48ad614SDennis Dalessandro packet->rhf & RHF_DC_ERR ? "dc " : "", 1331f48ad614SDennis Dalessandro packet->rhf & RHF_TID_ERR ? "tid " : "", 1332f48ad614SDennis Dalessandro packet->rhf & RHF_LEN_ERR ? "len " : "", 1333f48ad614SDennis Dalessandro packet->rhf & RHF_ECC_ERR ? "ecc " : "", 1334f48ad614SDennis Dalessandro packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", 1335f48ad614SDennis Dalessandro packet->rhf & RHF_ICRC_ERR ? "icrc " : "", 1336f48ad614SDennis Dalessandro rte); 1337f48ad614SDennis Dalessandro } 1338f48ad614SDennis Dalessandro 1339f48ad614SDennis Dalessandro /* 1340f48ad614SDennis Dalessandro * The following functions are called by the interrupt handler. They are type 1341f48ad614SDennis Dalessandro * specific handlers for each packet type. 1342f48ad614SDennis Dalessandro */ 1343f48ad614SDennis Dalessandro int process_receive_ib(struct hfi1_packet *packet) 1344f48ad614SDennis Dalessandro { 1345f48ad614SDennis Dalessandro trace_hfi1_rcvhdr(packet->rcd->ppd->dd, 1346f48ad614SDennis Dalessandro packet->rcd->ctxt, 1347f48ad614SDennis Dalessandro rhf_err_flags(packet->rhf), 1348f48ad614SDennis Dalessandro RHF_RCV_TYPE_IB, 1349f48ad614SDennis Dalessandro packet->hlen, 1350f48ad614SDennis Dalessandro packet->tlen, 1351f48ad614SDennis Dalessandro packet->updegr, 1352f48ad614SDennis Dalessandro rhf_egr_index(packet->rhf)); 1353f48ad614SDennis Dalessandro 1354f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) { 1355f48ad614SDennis Dalessandro handle_eflags(packet); 1356f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1357f48ad614SDennis Dalessandro } 1358f48ad614SDennis Dalessandro 1359f48ad614SDennis Dalessandro hfi1_ib_rcv(packet); 1360f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1361f48ad614SDennis Dalessandro } 1362f48ad614SDennis Dalessandro 1363f48ad614SDennis Dalessandro int process_receive_bypass(struct hfi1_packet *packet) 1364f48ad614SDennis Dalessandro { 1365505efe3eSJakub Pawlak struct hfi1_devdata *dd = packet->rcd->dd; 1366505efe3eSJakub Pawlak 1367f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) 1368f48ad614SDennis Dalessandro handle_eflags(packet); 1369f48ad614SDennis Dalessandro 1370505efe3eSJakub Pawlak dd_dev_err(dd, 1371f48ad614SDennis Dalessandro "Bypass packets are not supported in normal operation. Dropping\n"); 1372505efe3eSJakub Pawlak incr_cntr64(&dd->sw_rcv_bypass_packet_errors); 1373505efe3eSJakub Pawlak if (!(dd->err_info_rcvport.status_and_code & OPA_EI_STATUS_SMASK)) { 1374505efe3eSJakub Pawlak u64 *flits = packet->ebuf; 1375505efe3eSJakub Pawlak 1376505efe3eSJakub Pawlak if (flits && !(packet->rhf & RHF_LEN_ERR)) { 1377505efe3eSJakub Pawlak dd->err_info_rcvport.packet_flit1 = flits[0]; 1378505efe3eSJakub Pawlak dd->err_info_rcvport.packet_flit2 = 1379505efe3eSJakub Pawlak packet->tlen > sizeof(flits[0]) ? flits[1] : 0; 1380505efe3eSJakub Pawlak } 1381505efe3eSJakub Pawlak dd->err_info_rcvport.status_and_code |= 1382505efe3eSJakub Pawlak (OPA_EI_STATUS_SMASK | BAD_L2_ERR); 1383505efe3eSJakub Pawlak } 1384f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1385f48ad614SDennis Dalessandro } 1386f48ad614SDennis Dalessandro 1387f48ad614SDennis Dalessandro int process_receive_error(struct hfi1_packet *packet) 1388f48ad614SDennis Dalessandro { 1389f48ad614SDennis Dalessandro handle_eflags(packet); 1390f48ad614SDennis Dalessandro 1391f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) 1392f48ad614SDennis Dalessandro dd_dev_err(packet->rcd->dd, 1393f48ad614SDennis Dalessandro "Unhandled error packet received. Dropping.\n"); 1394f48ad614SDennis Dalessandro 1395f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1396f48ad614SDennis Dalessandro } 1397f48ad614SDennis Dalessandro 1398f48ad614SDennis Dalessandro int kdeth_process_expected(struct hfi1_packet *packet) 1399f48ad614SDennis Dalessandro { 1400f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) 1401f48ad614SDennis Dalessandro handle_eflags(packet); 1402f48ad614SDennis Dalessandro 1403f48ad614SDennis Dalessandro dd_dev_err(packet->rcd->dd, 1404f48ad614SDennis Dalessandro "Unhandled expected packet received. Dropping.\n"); 1405f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1406f48ad614SDennis Dalessandro } 1407f48ad614SDennis Dalessandro 1408f48ad614SDennis Dalessandro int kdeth_process_eager(struct hfi1_packet *packet) 1409f48ad614SDennis Dalessandro { 1410f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) 1411f48ad614SDennis Dalessandro handle_eflags(packet); 1412f48ad614SDennis Dalessandro 1413f48ad614SDennis Dalessandro dd_dev_err(packet->rcd->dd, 1414f48ad614SDennis Dalessandro "Unhandled eager packet received. Dropping.\n"); 1415f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1416f48ad614SDennis Dalessandro } 1417f48ad614SDennis Dalessandro 1418f48ad614SDennis Dalessandro int process_receive_invalid(struct hfi1_packet *packet) 1419f48ad614SDennis Dalessandro { 1420f48ad614SDennis Dalessandro dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", 1421f48ad614SDennis Dalessandro rhf_rcv_type(packet->rhf)); 1422f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1423f48ad614SDennis Dalessandro } 1424