1f48ad614SDennis Dalessandro /* 2a74d5307SMitko Haralanov * Copyright(c) 2015-2018 Intel Corporation. 3f48ad614SDennis Dalessandro * 4f48ad614SDennis Dalessandro * This file is provided under a dual BSD/GPLv2 license. When using or 5f48ad614SDennis Dalessandro * redistributing this file, you may do so under either license. 6f48ad614SDennis Dalessandro * 7f48ad614SDennis Dalessandro * GPL LICENSE SUMMARY 8f48ad614SDennis Dalessandro * 9f48ad614SDennis Dalessandro * This program is free software; you can redistribute it and/or modify 10f48ad614SDennis Dalessandro * it under the terms of version 2 of the GNU General Public License as 11f48ad614SDennis Dalessandro * published by the Free Software Foundation. 12f48ad614SDennis Dalessandro * 13f48ad614SDennis Dalessandro * This program is distributed in the hope that it will be useful, but 14f48ad614SDennis Dalessandro * WITHOUT ANY WARRANTY; without even the implied warranty of 15f48ad614SDennis Dalessandro * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16f48ad614SDennis Dalessandro * General Public License for more details. 17f48ad614SDennis Dalessandro * 18f48ad614SDennis Dalessandro * BSD LICENSE 19f48ad614SDennis Dalessandro * 20f48ad614SDennis Dalessandro * Redistribution and use in source and binary forms, with or without 21f48ad614SDennis Dalessandro * modification, are permitted provided that the following conditions 22f48ad614SDennis Dalessandro * are met: 23f48ad614SDennis Dalessandro * 24f48ad614SDennis Dalessandro * - Redistributions of source code must retain the above copyright 25f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer. 26f48ad614SDennis Dalessandro * - Redistributions in binary form must reproduce the above copyright 27f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer in 28f48ad614SDennis Dalessandro * the documentation and/or other materials provided with the 29f48ad614SDennis Dalessandro * distribution. 30f48ad614SDennis Dalessandro * - Neither the name of Intel Corporation nor the names of its 31f48ad614SDennis Dalessandro * contributors may be used to endorse or promote products derived 32f48ad614SDennis Dalessandro * from this software without specific prior written permission. 33f48ad614SDennis Dalessandro * 34f48ad614SDennis Dalessandro * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35f48ad614SDennis Dalessandro * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36f48ad614SDennis Dalessandro * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37f48ad614SDennis Dalessandro * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38f48ad614SDennis Dalessandro * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39f48ad614SDennis Dalessandro * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40f48ad614SDennis Dalessandro * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41f48ad614SDennis Dalessandro * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42f48ad614SDennis Dalessandro * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43f48ad614SDennis Dalessandro * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44f48ad614SDennis Dalessandro * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45f48ad614SDennis Dalessandro * 46f48ad614SDennis Dalessandro */ 47f48ad614SDennis Dalessandro 48f48ad614SDennis Dalessandro #include <linux/spinlock.h> 49f48ad614SDennis Dalessandro #include <linux/pci.h> 50f48ad614SDennis Dalessandro #include <linux/io.h> 51f48ad614SDennis Dalessandro #include <linux/delay.h> 52f48ad614SDennis Dalessandro #include <linux/netdevice.h> 53f48ad614SDennis Dalessandro #include <linux/vmalloc.h> 54f48ad614SDennis Dalessandro #include <linux/module.h> 55f48ad614SDennis Dalessandro #include <linux/prefetch.h> 56f48ad614SDennis Dalessandro #include <rdma/ib_verbs.h> 57f48ad614SDennis Dalessandro 58f48ad614SDennis Dalessandro #include "hfi.h" 59f48ad614SDennis Dalessandro #include "trace.h" 60f48ad614SDennis Dalessandro #include "qp.h" 61f48ad614SDennis Dalessandro #include "sdma.h" 620181ce31SDon Hiatt #include "debugfs.h" 63d4829ea6SVishwanathapura, Niranjana #include "vnic.h" 64a74d5307SMitko Haralanov #include "fault.h" 65f48ad614SDennis Dalessandro 66f48ad614SDennis Dalessandro #undef pr_fmt 67f48ad614SDennis Dalessandro #define pr_fmt(fmt) DRIVER_NAME ": " fmt 68f48ad614SDennis Dalessandro 69f48ad614SDennis Dalessandro /* 70f48ad614SDennis Dalessandro * The size has to be longer than this string, so we can append 71f48ad614SDennis Dalessandro * board/chip information to it in the initialization code. 72f48ad614SDennis Dalessandro */ 73f48ad614SDennis Dalessandro const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n"; 74f48ad614SDennis Dalessandro 75f48ad614SDennis Dalessandro DEFINE_SPINLOCK(hfi1_devs_lock); 76f48ad614SDennis Dalessandro LIST_HEAD(hfi1_dev_list); 77f48ad614SDennis Dalessandro DEFINE_MUTEX(hfi1_mutex); /* general driver use */ 78f48ad614SDennis Dalessandro 79f48ad614SDennis Dalessandro unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU; 80f48ad614SDennis Dalessandro module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO); 81f48ad614SDennis Dalessandro MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify( 82f48ad614SDennis Dalessandro HFI1_DEFAULT_MAX_MTU)); 83f48ad614SDennis Dalessandro 84f48ad614SDennis Dalessandro unsigned int hfi1_cu = 1; 85f48ad614SDennis Dalessandro module_param_named(cu, hfi1_cu, uint, S_IRUGO); 86f48ad614SDennis Dalessandro MODULE_PARM_DESC(cu, "Credit return units"); 87f48ad614SDennis Dalessandro 88f48ad614SDennis Dalessandro unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT; 89f4cd8765SMichael J. Ruhl static int hfi1_caps_set(const char *val, const struct kernel_param *kp); 90f4cd8765SMichael J. Ruhl static int hfi1_caps_get(char *buffer, const struct kernel_param *kp); 91f48ad614SDennis Dalessandro static const struct kernel_param_ops cap_ops = { 92f48ad614SDennis Dalessandro .set = hfi1_caps_set, 93f48ad614SDennis Dalessandro .get = hfi1_caps_get 94f48ad614SDennis Dalessandro }; 95f48ad614SDennis Dalessandro module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO); 96f48ad614SDennis Dalessandro MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features"); 97f48ad614SDennis Dalessandro 98f48ad614SDennis Dalessandro MODULE_LICENSE("Dual BSD/GPL"); 99f48ad614SDennis Dalessandro MODULE_DESCRIPTION("Intel Omni-Path Architecture driver"); 100f48ad614SDennis Dalessandro 101f48ad614SDennis Dalessandro /* 102f48ad614SDennis Dalessandro * MAX_PKT_RCV is the max # if packets processed per receive interrupt. 103f48ad614SDennis Dalessandro */ 104f48ad614SDennis Dalessandro #define MAX_PKT_RECV 64 105a82a7fcdSMike Marciniszyn /* 106a82a7fcdSMike Marciniszyn * MAX_PKT_THREAD_RCV is the max # of packets processed before 107a82a7fcdSMike Marciniszyn * the qp_wait_list queue is flushed. 108a82a7fcdSMike Marciniszyn */ 109a82a7fcdSMike Marciniszyn #define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4) 110f48ad614SDennis Dalessandro #define EGR_HEAD_UPDATE_THRESHOLD 16 111f48ad614SDennis Dalessandro 112f48ad614SDennis Dalessandro struct hfi1_ib_stats hfi1_stats; 113f48ad614SDennis Dalessandro 114f48ad614SDennis Dalessandro static int hfi1_caps_set(const char *val, const struct kernel_param *kp) 115f48ad614SDennis Dalessandro { 116f48ad614SDennis Dalessandro int ret = 0; 117f48ad614SDennis Dalessandro unsigned long *cap_mask_ptr = (unsigned long *)kp->arg, 118f48ad614SDennis Dalessandro cap_mask = *cap_mask_ptr, value, diff, 119f48ad614SDennis Dalessandro write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) | 120f48ad614SDennis Dalessandro HFI1_CAP_WRITABLE_MASK); 121f48ad614SDennis Dalessandro 122f48ad614SDennis Dalessandro ret = kstrtoul(val, 0, &value); 123f48ad614SDennis Dalessandro if (ret) { 124f48ad614SDennis Dalessandro pr_warn("Invalid module parameter value for 'cap_mask'\n"); 125f48ad614SDennis Dalessandro goto done; 126f48ad614SDennis Dalessandro } 127f48ad614SDennis Dalessandro /* Get the changed bits (except the locked bit) */ 128f48ad614SDennis Dalessandro diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK); 129f48ad614SDennis Dalessandro 130f48ad614SDennis Dalessandro /* Remove any bits that are not allowed to change after driver load */ 131f48ad614SDennis Dalessandro if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) { 132f48ad614SDennis Dalessandro pr_warn("Ignoring non-writable capability bits %#lx\n", 133f48ad614SDennis Dalessandro diff & ~write_mask); 134f48ad614SDennis Dalessandro diff &= write_mask; 135f48ad614SDennis Dalessandro } 136f48ad614SDennis Dalessandro 137f48ad614SDennis Dalessandro /* Mask off any reserved bits */ 138f48ad614SDennis Dalessandro diff &= ~HFI1_CAP_RESERVED_MASK; 139f48ad614SDennis Dalessandro /* Clear any previously set and changing bits */ 140f48ad614SDennis Dalessandro cap_mask &= ~diff; 141f48ad614SDennis Dalessandro /* Update the bits with the new capability */ 142f48ad614SDennis Dalessandro cap_mask |= (value & diff); 143f48ad614SDennis Dalessandro /* Check for any kernel/user restrictions */ 144f48ad614SDennis Dalessandro diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^ 145f48ad614SDennis Dalessandro ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT); 146f48ad614SDennis Dalessandro cap_mask &= ~diff; 147f48ad614SDennis Dalessandro /* Set the bitmask to the final set */ 148f48ad614SDennis Dalessandro *cap_mask_ptr = cap_mask; 149f48ad614SDennis Dalessandro done: 150f48ad614SDennis Dalessandro return ret; 151f48ad614SDennis Dalessandro } 152f48ad614SDennis Dalessandro 153f48ad614SDennis Dalessandro static int hfi1_caps_get(char *buffer, const struct kernel_param *kp) 154f48ad614SDennis Dalessandro { 155f48ad614SDennis Dalessandro unsigned long cap_mask = *(unsigned long *)kp->arg; 156f48ad614SDennis Dalessandro 157f48ad614SDennis Dalessandro cap_mask &= ~HFI1_CAP_LOCKED_SMASK; 158f48ad614SDennis Dalessandro cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT); 159f48ad614SDennis Dalessandro 160f48ad614SDennis Dalessandro return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask); 161f48ad614SDennis Dalessandro } 162f48ad614SDennis Dalessandro 163f48ad614SDennis Dalessandro struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi) 164f48ad614SDennis Dalessandro { 165f48ad614SDennis Dalessandro struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi); 166f48ad614SDennis Dalessandro struct hfi1_devdata *dd = container_of(ibdev, 167f48ad614SDennis Dalessandro struct hfi1_devdata, verbs_dev); 168f48ad614SDennis Dalessandro return dd->pcidev; 169f48ad614SDennis Dalessandro } 170f48ad614SDennis Dalessandro 171f48ad614SDennis Dalessandro /* 172f48ad614SDennis Dalessandro * Return count of units with at least one port ACTIVE. 173f48ad614SDennis Dalessandro */ 174f48ad614SDennis Dalessandro int hfi1_count_active_units(void) 175f48ad614SDennis Dalessandro { 176f48ad614SDennis Dalessandro struct hfi1_devdata *dd; 177f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd; 178f48ad614SDennis Dalessandro unsigned long flags; 179f48ad614SDennis Dalessandro int pidx, nunits_active = 0; 180f48ad614SDennis Dalessandro 181f48ad614SDennis Dalessandro spin_lock_irqsave(&hfi1_devs_lock, flags); 182f48ad614SDennis Dalessandro list_for_each_entry(dd, &hfi1_dev_list, list) { 183cb51c5d2SMike Marciniszyn if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1) 184f48ad614SDennis Dalessandro continue; 185f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) { 186f48ad614SDennis Dalessandro ppd = dd->pport + pidx; 187f48ad614SDennis Dalessandro if (ppd->lid && ppd->linkup) { 188f48ad614SDennis Dalessandro nunits_active++; 189f48ad614SDennis Dalessandro break; 190f48ad614SDennis Dalessandro } 191f48ad614SDennis Dalessandro } 192f48ad614SDennis Dalessandro } 193f48ad614SDennis Dalessandro spin_unlock_irqrestore(&hfi1_devs_lock, flags); 194f48ad614SDennis Dalessandro return nunits_active; 195f48ad614SDennis Dalessandro } 196f48ad614SDennis Dalessandro 197f48ad614SDennis Dalessandro /* 198f48ad614SDennis Dalessandro * Get address of eager buffer from it's index (allocated in chunks, not 199f48ad614SDennis Dalessandro * contiguous). 200f48ad614SDennis Dalessandro */ 201f48ad614SDennis Dalessandro static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf, 202f48ad614SDennis Dalessandro u8 *update) 203f48ad614SDennis Dalessandro { 204f48ad614SDennis Dalessandro u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf); 205f48ad614SDennis Dalessandro 206f48ad614SDennis Dalessandro *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset; 207f48ad614SDennis Dalessandro return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) + 208f48ad614SDennis Dalessandro (offset * RCV_BUF_BLOCK_SIZE)); 209f48ad614SDennis Dalessandro } 210f48ad614SDennis Dalessandro 21140442b30SMike Marciniszyn static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd, 2129039746cSDon Hiatt __le32 *rhf_addr) 2139039746cSDon Hiatt { 2149039746cSDon Hiatt u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr)); 2159039746cSDon Hiatt 21640442b30SMike Marciniszyn return (void *)(rhf_addr - rcd->rhf_offset + offset); 2179039746cSDon Hiatt } 2189039746cSDon Hiatt 21940442b30SMike Marciniszyn static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd, 2209039746cSDon Hiatt __le32 *rhf_addr) 2219039746cSDon Hiatt { 22240442b30SMike Marciniszyn return (struct ib_header *)hfi1_get_header(rcd, rhf_addr); 2239039746cSDon Hiatt } 2249039746cSDon Hiatt 22572c07e2bSDon Hiatt static inline struct hfi1_16b_header 22640442b30SMike Marciniszyn *hfi1_get_16B_header(struct hfi1_ctxtdata *rcd, 22772c07e2bSDon Hiatt __le32 *rhf_addr) 22872c07e2bSDon Hiatt { 22940442b30SMike Marciniszyn return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr); 23072c07e2bSDon Hiatt } 23172c07e2bSDon Hiatt 232f48ad614SDennis Dalessandro /* 233f48ad614SDennis Dalessandro * Validate and encode the a given RcvArray Buffer size. 234f48ad614SDennis Dalessandro * The function will check whether the given size falls within 235f48ad614SDennis Dalessandro * allowed size ranges for the respective type and, optionally, 236f48ad614SDennis Dalessandro * return the proper encoding. 237f48ad614SDennis Dalessandro */ 238a82a7fcdSMike Marciniszyn int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded) 239f48ad614SDennis Dalessandro { 240f48ad614SDennis Dalessandro if (unlikely(!PAGE_ALIGNED(size))) 241f48ad614SDennis Dalessandro return 0; 242f48ad614SDennis Dalessandro if (unlikely(size < MIN_EAGER_BUFFER)) 243f48ad614SDennis Dalessandro return 0; 244f48ad614SDennis Dalessandro if (size > 245f48ad614SDennis Dalessandro (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER)) 246f48ad614SDennis Dalessandro return 0; 247f48ad614SDennis Dalessandro if (encoded) 248f48ad614SDennis Dalessandro *encoded = ilog2(size / PAGE_SIZE) + 1; 249f48ad614SDennis Dalessandro return 1; 250f48ad614SDennis Dalessandro } 251f48ad614SDennis Dalessandro 252f48ad614SDennis Dalessandro static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd, 253f48ad614SDennis Dalessandro struct hfi1_packet *packet) 254f48ad614SDennis Dalessandro { 255261a4351SMike Marciniszyn struct ib_header *rhdr = packet->hdr; 256f48ad614SDennis Dalessandro u32 rte = rhf_rcv_type_err(packet->rhf); 2575786adf3SDon Hiatt u32 mlid_base; 258f3e862cbSSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(rcd); 259f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 260aca7f4fcSSebastian Sanchez struct hfi1_ibdev *verbs_dev = &dd->verbs_dev; 261aca7f4fcSSebastian Sanchez struct rvt_dev_info *rdi = &verbs_dev->rdi; 262aca7f4fcSSebastian Sanchez 263aca7f4fcSSebastian Sanchez if ((packet->rhf & RHF_DC_ERR) && 264aca7f4fcSSebastian Sanchez hfi1_dbg_fault_suppress_err(verbs_dev)) 265aca7f4fcSSebastian Sanchez return; 266f48ad614SDennis Dalessandro 267f48ad614SDennis Dalessandro if (packet->rhf & (RHF_VCRC_ERR | RHF_ICRC_ERR)) 268f48ad614SDennis Dalessandro return; 269f48ad614SDennis Dalessandro 2705786adf3SDon Hiatt if (packet->etype == RHF_RCV_TYPE_BYPASS) { 2715786adf3SDon Hiatt goto drop; 2725786adf3SDon Hiatt } else { 2735786adf3SDon Hiatt u8 lnh = ib_get_lnh(rhdr); 2745786adf3SDon Hiatt 2755786adf3SDon Hiatt mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE); 2769039746cSDon Hiatt if (lnh == HFI1_LRH_BTH) { 2779039746cSDon Hiatt packet->ohdr = &rhdr->u.oth; 2789039746cSDon Hiatt } else if (lnh == HFI1_LRH_GRH) { 2799039746cSDon Hiatt packet->ohdr = &rhdr->u.l.oth; 2809039746cSDon Hiatt packet->grh = &rhdr->u.l.grh; 2819039746cSDon Hiatt } else { 2829039746cSDon Hiatt goto drop; 2839039746cSDon Hiatt } 2845786adf3SDon Hiatt } 2859039746cSDon Hiatt 286f48ad614SDennis Dalessandro if (packet->rhf & RHF_TID_ERR) { 287f48ad614SDennis Dalessandro /* For TIDERR and RC QPs preemptively schedule a NAK */ 288f48ad614SDennis Dalessandro u32 tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 2899039746cSDon Hiatt u32 dlid = ib_get_dlid(rhdr); 290f48ad614SDennis Dalessandro u32 qp_num; 291f48ad614SDennis Dalessandro 292f48ad614SDennis Dalessandro /* Sanity check packet */ 293f48ad614SDennis Dalessandro if (tlen < 24) 294f48ad614SDennis Dalessandro goto drop; 295f48ad614SDennis Dalessandro 296f48ad614SDennis Dalessandro /* Check for GRH */ 2975786adf3SDon Hiatt if (packet->grh) { 298f48ad614SDennis Dalessandro u32 vtf; 2999039746cSDon Hiatt struct ib_grh *grh = packet->grh; 300f48ad614SDennis Dalessandro 3019039746cSDon Hiatt if (grh->next_hdr != IB_GRH_NEXT_HDR) 302f48ad614SDennis Dalessandro goto drop; 3039039746cSDon Hiatt vtf = be32_to_cpu(grh->version_tclass_flow); 304f48ad614SDennis Dalessandro if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 305f48ad614SDennis Dalessandro goto drop; 306f48ad614SDennis Dalessandro } 3079039746cSDon Hiatt 308f48ad614SDennis Dalessandro /* Get the destination QP number. */ 3099039746cSDon Hiatt qp_num = ib_bth_get_qpn(packet->ohdr); 3109039746cSDon Hiatt if (dlid < mlid_base) { 311f48ad614SDennis Dalessandro struct rvt_qp *qp; 312f48ad614SDennis Dalessandro unsigned long flags; 313f48ad614SDennis Dalessandro 314f48ad614SDennis Dalessandro rcu_read_lock(); 315f48ad614SDennis Dalessandro qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num); 316f48ad614SDennis Dalessandro if (!qp) { 317f48ad614SDennis Dalessandro rcu_read_unlock(); 318f48ad614SDennis Dalessandro goto drop; 319f48ad614SDennis Dalessandro } 320f48ad614SDennis Dalessandro 321f48ad614SDennis Dalessandro /* 322f48ad614SDennis Dalessandro * Handle only RC QPs - for other QP types drop error 323f48ad614SDennis Dalessandro * packet. 324f48ad614SDennis Dalessandro */ 325f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->r_lock, flags); 326f48ad614SDennis Dalessandro 327f48ad614SDennis Dalessandro /* Check for valid receive state. */ 328f48ad614SDennis Dalessandro if (!(ib_rvt_state_ops[qp->state] & 329f48ad614SDennis Dalessandro RVT_PROCESS_RECV_OK)) { 330f48ad614SDennis Dalessandro ibp->rvp.n_pkt_drops++; 331f48ad614SDennis Dalessandro } 332f48ad614SDennis Dalessandro 333f48ad614SDennis Dalessandro switch (qp->ibqp.qp_type) { 334f48ad614SDennis Dalessandro case IB_QPT_RC: 3359039746cSDon Hiatt hfi1_rc_hdrerr(rcd, packet, qp); 336f48ad614SDennis Dalessandro break; 337f48ad614SDennis Dalessandro default: 338f48ad614SDennis Dalessandro /* For now don't handle any other QP types */ 339f48ad614SDennis Dalessandro break; 340f48ad614SDennis Dalessandro } 341f48ad614SDennis Dalessandro 342f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->r_lock, flags); 343f48ad614SDennis Dalessandro rcu_read_unlock(); 344f48ad614SDennis Dalessandro } /* Unicast QP */ 345f48ad614SDennis Dalessandro } /* Valid packet with TIDErr */ 346f48ad614SDennis Dalessandro 347f48ad614SDennis Dalessandro /* handle "RcvTypeErr" flags */ 348f48ad614SDennis Dalessandro switch (rte) { 349f48ad614SDennis Dalessandro case RHF_RTE_ERROR_OP_CODE_ERR: 350f48ad614SDennis Dalessandro { 351f48ad614SDennis Dalessandro void *ebuf = NULL; 3529039746cSDon Hiatt u8 opcode; 353f48ad614SDennis Dalessandro 354f48ad614SDennis Dalessandro if (rhf_use_egr_bfr(packet->rhf)) 355f48ad614SDennis Dalessandro ebuf = packet->ebuf; 356f48ad614SDennis Dalessandro 357f48ad614SDennis Dalessandro if (!ebuf) 358f48ad614SDennis Dalessandro goto drop; /* this should never happen */ 359f48ad614SDennis Dalessandro 3609039746cSDon Hiatt opcode = ib_bth_get_opcode(packet->ohdr); 361f48ad614SDennis Dalessandro if (opcode == IB_OPCODE_CNP) { 362f48ad614SDennis Dalessandro /* 363f48ad614SDennis Dalessandro * Only in pre-B0 h/w is the CNP_OPCODE handled 364f48ad614SDennis Dalessandro * via this code path. 365f48ad614SDennis Dalessandro */ 366f48ad614SDennis Dalessandro struct rvt_qp *qp = NULL; 367f48ad614SDennis Dalessandro u32 lqpn, rqpn; 368f48ad614SDennis Dalessandro u16 rlid; 369f48ad614SDennis Dalessandro u8 svc_type, sl, sc5; 370f48ad614SDennis Dalessandro 371aad559c2SDasaratharaman Chandramouli sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf); 372f48ad614SDennis Dalessandro sl = ibp->sc_to_sl[sc5]; 373f48ad614SDennis Dalessandro 3749039746cSDon Hiatt lqpn = ib_bth_get_qpn(packet->ohdr); 375f48ad614SDennis Dalessandro rcu_read_lock(); 376f48ad614SDennis Dalessandro qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn); 377f48ad614SDennis Dalessandro if (!qp) { 378f48ad614SDennis Dalessandro rcu_read_unlock(); 379f48ad614SDennis Dalessandro goto drop; 380f48ad614SDennis Dalessandro } 381f48ad614SDennis Dalessandro 382f48ad614SDennis Dalessandro switch (qp->ibqp.qp_type) { 383f48ad614SDennis Dalessandro case IB_QPT_UD: 384f48ad614SDennis Dalessandro rlid = 0; 385f48ad614SDennis Dalessandro rqpn = 0; 386f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UD; 387f48ad614SDennis Dalessandro break; 388f48ad614SDennis Dalessandro case IB_QPT_UC: 389cb427057SDon Hiatt rlid = ib_get_slid(rhdr); 390f48ad614SDennis Dalessandro rqpn = qp->remote_qpn; 391f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UC; 392f48ad614SDennis Dalessandro break; 393f48ad614SDennis Dalessandro default: 39441904439SBart Van Assche rcu_read_unlock(); 395f48ad614SDennis Dalessandro goto drop; 396f48ad614SDennis Dalessandro } 397f48ad614SDennis Dalessandro 398f48ad614SDennis Dalessandro process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 399f48ad614SDennis Dalessandro rcu_read_unlock(); 400f48ad614SDennis Dalessandro } 401f48ad614SDennis Dalessandro 402f48ad614SDennis Dalessandro packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK; 403f48ad614SDennis Dalessandro break; 404f48ad614SDennis Dalessandro } 405f48ad614SDennis Dalessandro default: 406f48ad614SDennis Dalessandro break; 407f48ad614SDennis Dalessandro } 408f48ad614SDennis Dalessandro 409f48ad614SDennis Dalessandro drop: 410f48ad614SDennis Dalessandro return; 411f48ad614SDennis Dalessandro } 412f48ad614SDennis Dalessandro 413f48ad614SDennis Dalessandro static inline void init_packet(struct hfi1_ctxtdata *rcd, 414f48ad614SDennis Dalessandro struct hfi1_packet *packet) 415f48ad614SDennis Dalessandro { 416f48ad614SDennis Dalessandro packet->rsize = rcd->rcvhdrqentsize; /* words */ 417f48ad614SDennis Dalessandro packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */ 418f48ad614SDennis Dalessandro packet->rcd = rcd; 419f48ad614SDennis Dalessandro packet->updegr = 0; 420f48ad614SDennis Dalessandro packet->etail = -1; 421f48ad614SDennis Dalessandro packet->rhf_addr = get_rhf_addr(rcd); 422f48ad614SDennis Dalessandro packet->rhf = rhf_to_cpu(packet->rhf_addr); 423f48ad614SDennis Dalessandro packet->rhqoff = rcd->head; 424f48ad614SDennis Dalessandro packet->numpkt = 0; 425f48ad614SDennis Dalessandro } 426f48ad614SDennis Dalessandro 427e2fdbc23SBart Van Assche /* We support only two types - 9B and 16B for now */ 428e2fdbc23SBart Van Assche static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = { 429e2fdbc23SBart Van Assche [HFI1_PKT_TYPE_9B] = &return_cnp, 430e2fdbc23SBart Van Assche [HFI1_PKT_TYPE_16B] = &return_cnp_16B 431e2fdbc23SBart Van Assche }; 432e2fdbc23SBart Van Assche 433fe4dd423SMitko Haralanov /** 434fe4dd423SMitko Haralanov * hfi1_process_ecn_slowpath - Process FECN or BECN bits 435fe4dd423SMitko Haralanov * @qp: The packet's destination QP 436fe4dd423SMitko Haralanov * @pkt: The packet itself. 437fe4dd423SMitko Haralanov * @prescan: Is the caller the RXQ prescan 438fe4dd423SMitko Haralanov * 439fe4dd423SMitko Haralanov * Process the packet's FECN or BECN bits. By now, the packet 440fe4dd423SMitko Haralanov * has already been evaluated whether processing of those bit should 441fe4dd423SMitko Haralanov * be done. 442fe4dd423SMitko Haralanov * The significance of the @prescan argument is that if the caller 443fe4dd423SMitko Haralanov * is the RXQ prescan, a CNP will be send out instead of waiting for the 444fe4dd423SMitko Haralanov * normal packet processing to send an ACK with BECN set (or a CNP). 445fe4dd423SMitko Haralanov */ 446fe4dd423SMitko Haralanov bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt, 447fe4dd423SMitko Haralanov bool prescan) 448f48ad614SDennis Dalessandro { 449f48ad614SDennis Dalessandro struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); 450f59fb9e0SMike Marciniszyn struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 451261a4351SMike Marciniszyn struct ib_other_headers *ohdr = pkt->ohdr; 4529039746cSDon Hiatt struct ib_grh *grh = pkt->grh; 453fe4dd423SMitko Haralanov u32 rqpn = 0; 454f59fb9e0SMike Marciniszyn u16 pkey; 455f59fb9e0SMike Marciniszyn u32 rlid, slid, dlid = 0; 456fe4dd423SMitko Haralanov u8 hdr_type, sc, svc_type, opcode; 457fe4dd423SMitko Haralanov bool is_mcast = false, ignore_fecn = false, do_cnp = false, 458fe4dd423SMitko Haralanov fecn, becn; 4595fd2b562SMitko Haralanov 460f59fb9e0SMike Marciniszyn /* can be called from prescan */ 46188733e3bSDon Hiatt if (pkt->etype == RHF_RCV_TYPE_BYPASS) { 46288733e3bSDon Hiatt pkey = hfi1_16B_get_pkey(pkt->hdr); 46388733e3bSDon Hiatt sc = hfi1_16B_get_sc(pkt->hdr); 464f59fb9e0SMike Marciniszyn dlid = hfi1_16B_get_dlid(pkt->hdr); 465f59fb9e0SMike Marciniszyn slid = hfi1_16B_get_slid(pkt->hdr); 466fe4dd423SMitko Haralanov is_mcast = hfi1_is_16B_mcast(dlid); 467fe4dd423SMitko Haralanov opcode = ib_bth_get_opcode(ohdr); 46888733e3bSDon Hiatt hdr_type = HFI1_PKT_TYPE_16B; 469fe4dd423SMitko Haralanov fecn = hfi1_16B_get_fecn(pkt->hdr); 470fe4dd423SMitko Haralanov becn = hfi1_16B_get_becn(pkt->hdr); 47188733e3bSDon Hiatt } else { 47288733e3bSDon Hiatt pkey = ib_bth_get_pkey(ohdr); 47388733e3bSDon Hiatt sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf); 474fe4dd423SMitko Haralanov dlid = qp->ibqp.qp_type != IB_QPT_UD ? ib_get_dlid(pkt->hdr) : 475fe4dd423SMitko Haralanov ppd->lid; 476f59fb9e0SMike Marciniszyn slid = ib_get_slid(pkt->hdr); 477fe4dd423SMitko Haralanov is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) && 478fe4dd423SMitko Haralanov (dlid != be16_to_cpu(IB_LID_PERMISSIVE)); 479fe4dd423SMitko Haralanov opcode = ib_bth_get_opcode(ohdr); 48088733e3bSDon Hiatt hdr_type = HFI1_PKT_TYPE_9B; 481fe4dd423SMitko Haralanov fecn = ib_bth_get_fecn(ohdr); 482fe4dd423SMitko Haralanov becn = ib_bth_get_becn(ohdr); 48388733e3bSDon Hiatt } 48488733e3bSDon Hiatt 485f48ad614SDennis Dalessandro switch (qp->ibqp.qp_type) { 486f59fb9e0SMike Marciniszyn case IB_QPT_UD: 487f59fb9e0SMike Marciniszyn rlid = slid; 488f59fb9e0SMike Marciniszyn rqpn = ib_get_sqpn(pkt->ohdr); 489f59fb9e0SMike Marciniszyn svc_type = IB_CC_SVCTYPE_UD; 490f59fb9e0SMike Marciniszyn break; 491f48ad614SDennis Dalessandro case IB_QPT_SMI: 492f48ad614SDennis Dalessandro case IB_QPT_GSI: 493f59fb9e0SMike Marciniszyn rlid = slid; 49488733e3bSDon Hiatt rqpn = ib_get_sqpn(pkt->ohdr); 495f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UD; 496f48ad614SDennis Dalessandro break; 497f48ad614SDennis Dalessandro case IB_QPT_UC: 498d8966fcdSDasaratharaman Chandramouli rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 499f48ad614SDennis Dalessandro rqpn = qp->remote_qpn; 500f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_UC; 501f48ad614SDennis Dalessandro break; 502f48ad614SDennis Dalessandro case IB_QPT_RC: 503d8966fcdSDasaratharaman Chandramouli rlid = rdma_ah_get_dlid(&qp->remote_ah_attr); 504f48ad614SDennis Dalessandro rqpn = qp->remote_qpn; 505f48ad614SDennis Dalessandro svc_type = IB_CC_SVCTYPE_RC; 506f48ad614SDennis Dalessandro break; 507f48ad614SDennis Dalessandro default: 508fe4dd423SMitko Haralanov return false; 509f48ad614SDennis Dalessandro } 510f48ad614SDennis Dalessandro 511fe4dd423SMitko Haralanov ignore_fecn = is_mcast || (opcode == IB_OPCODE_CNP) || 512fe4dd423SMitko Haralanov (opcode == IB_OPCODE_RC_ACKNOWLEDGE); 513fe4dd423SMitko Haralanov /* 514fe4dd423SMitko Haralanov * ACKNOWLEDGE packets do not get a CNP but this will be 515fe4dd423SMitko Haralanov * guarded by ignore_fecn above. 516fe4dd423SMitko Haralanov */ 517fe4dd423SMitko Haralanov do_cnp = prescan || 518fe4dd423SMitko Haralanov (opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST && 519fe4dd423SMitko Haralanov opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE); 520fe4dd423SMitko Haralanov 52188733e3bSDon Hiatt /* Call appropriate CNP handler */ 522fe4dd423SMitko Haralanov if (!ignore_fecn && do_cnp && fecn) 52388733e3bSDon Hiatt hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey, 52488733e3bSDon Hiatt dlid, rlid, sc, grh); 525f48ad614SDennis Dalessandro 526fe4dd423SMitko Haralanov if (becn) { 527fe4dd423SMitko Haralanov u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK; 5285fd2b562SMitko Haralanov u8 sl = ibp->sc_to_sl[sc]; 529f48ad614SDennis Dalessandro 530f48ad614SDennis Dalessandro process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type); 531f48ad614SDennis Dalessandro } 532fe4dd423SMitko Haralanov return !ignore_fecn && fecn; 533f48ad614SDennis Dalessandro } 534f48ad614SDennis Dalessandro 535f48ad614SDennis Dalessandro struct ps_mdata { 536f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd; 537f48ad614SDennis Dalessandro u32 rsize; 538f48ad614SDennis Dalessandro u32 maxcnt; 539f48ad614SDennis Dalessandro u32 ps_head; 540f48ad614SDennis Dalessandro u32 ps_tail; 541f48ad614SDennis Dalessandro u32 ps_seq; 542f48ad614SDennis Dalessandro }; 543f48ad614SDennis Dalessandro 544f48ad614SDennis Dalessandro static inline void init_ps_mdata(struct ps_mdata *mdata, 545f48ad614SDennis Dalessandro struct hfi1_packet *packet) 546f48ad614SDennis Dalessandro { 547f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 548f48ad614SDennis Dalessandro 549f48ad614SDennis Dalessandro mdata->rcd = rcd; 550f48ad614SDennis Dalessandro mdata->rsize = packet->rsize; 551f48ad614SDennis Dalessandro mdata->maxcnt = packet->maxcnt; 552f48ad614SDennis Dalessandro mdata->ps_head = packet->rhqoff; 553f48ad614SDennis Dalessandro 554f48ad614SDennis Dalessandro if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 555f48ad614SDennis Dalessandro mdata->ps_tail = get_rcvhdrtail(rcd); 556f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT) 557f48ad614SDennis Dalessandro mdata->ps_seq = rcd->seq_cnt; 558f48ad614SDennis Dalessandro else 559f48ad614SDennis Dalessandro mdata->ps_seq = 0; /* not used with DMA_RTAIL */ 560f48ad614SDennis Dalessandro } else { 561f48ad614SDennis Dalessandro mdata->ps_tail = 0; /* used only with DMA_RTAIL*/ 562f48ad614SDennis Dalessandro mdata->ps_seq = rcd->seq_cnt; 563f48ad614SDennis Dalessandro } 564f48ad614SDennis Dalessandro } 565f48ad614SDennis Dalessandro 566f48ad614SDennis Dalessandro static inline int ps_done(struct ps_mdata *mdata, u64 rhf, 567f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 568f48ad614SDennis Dalessandro { 569f48ad614SDennis Dalessandro if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) 570f48ad614SDennis Dalessandro return mdata->ps_head == mdata->ps_tail; 571f48ad614SDennis Dalessandro return mdata->ps_seq != rhf_rcv_seq(rhf); 572f48ad614SDennis Dalessandro } 573f48ad614SDennis Dalessandro 574f48ad614SDennis Dalessandro static inline int ps_skip(struct ps_mdata *mdata, u64 rhf, 575f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 576f48ad614SDennis Dalessandro { 577f48ad614SDennis Dalessandro /* 578f48ad614SDennis Dalessandro * Control context can potentially receive an invalid rhf. 579f48ad614SDennis Dalessandro * Drop such packets. 580f48ad614SDennis Dalessandro */ 581f48ad614SDennis Dalessandro if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail)) 582f48ad614SDennis Dalessandro return mdata->ps_seq != rhf_rcv_seq(rhf); 583f48ad614SDennis Dalessandro 584f48ad614SDennis Dalessandro return 0; 585f48ad614SDennis Dalessandro } 586f48ad614SDennis Dalessandro 587f48ad614SDennis Dalessandro static inline void update_ps_mdata(struct ps_mdata *mdata, 588f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 589f48ad614SDennis Dalessandro { 590f48ad614SDennis Dalessandro mdata->ps_head += mdata->rsize; 591f48ad614SDennis Dalessandro if (mdata->ps_head >= mdata->maxcnt) 592f48ad614SDennis Dalessandro mdata->ps_head = 0; 593f48ad614SDennis Dalessandro 594f48ad614SDennis Dalessandro /* Control context must do seq counting */ 595f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) || 596f48ad614SDennis Dalessandro (rcd->ctxt == HFI1_CTRL_CTXT)) { 597f48ad614SDennis Dalessandro if (++mdata->ps_seq > 13) 598f48ad614SDennis Dalessandro mdata->ps_seq = 1; 599f48ad614SDennis Dalessandro } 600f48ad614SDennis Dalessandro } 601f48ad614SDennis Dalessandro 602f48ad614SDennis Dalessandro /* 603f48ad614SDennis Dalessandro * prescan_rxq - search through the receive queue looking for packets 604f48ad614SDennis Dalessandro * containing Excplicit Congestion Notifications (FECNs, or BECNs). 605f48ad614SDennis Dalessandro * When an ECN is found, process the Congestion Notification, and toggle 606f48ad614SDennis Dalessandro * it off. 607f48ad614SDennis Dalessandro * This is declared as a macro to allow quick checking of the port to avoid 608f48ad614SDennis Dalessandro * the overhead of a function call if not enabled. 609f48ad614SDennis Dalessandro */ 610f48ad614SDennis Dalessandro #define prescan_rxq(rcd, packet) \ 611f48ad614SDennis Dalessandro do { \ 612f48ad614SDennis Dalessandro if (rcd->ppd->cc_prescan) \ 613f48ad614SDennis Dalessandro __prescan_rxq(packet); \ 614f48ad614SDennis Dalessandro } while (0) 615f48ad614SDennis Dalessandro static void __prescan_rxq(struct hfi1_packet *packet) 616f48ad614SDennis Dalessandro { 617f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 618f48ad614SDennis Dalessandro struct ps_mdata mdata; 619f48ad614SDennis Dalessandro 620f48ad614SDennis Dalessandro init_ps_mdata(&mdata, packet); 621f48ad614SDennis Dalessandro 622f48ad614SDennis Dalessandro while (1) { 623f3e862cbSSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(rcd); 624f48ad614SDennis Dalessandro __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 62540442b30SMike Marciniszyn packet->rcd->rhf_offset; 626f48ad614SDennis Dalessandro struct rvt_qp *qp; 627261a4351SMike Marciniszyn struct ib_header *hdr; 62840442b30SMike Marciniszyn struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi; 629f48ad614SDennis Dalessandro u64 rhf = rhf_to_cpu(rhf_addr); 630f48ad614SDennis Dalessandro u32 etype = rhf_rcv_type(rhf), qpn, bth1; 631f48ad614SDennis Dalessandro u8 lnh; 632f48ad614SDennis Dalessandro 633f48ad614SDennis Dalessandro if (ps_done(&mdata, rhf, rcd)) 634f48ad614SDennis Dalessandro break; 635f48ad614SDennis Dalessandro 636f48ad614SDennis Dalessandro if (ps_skip(&mdata, rhf, rcd)) 637f48ad614SDennis Dalessandro goto next; 638f48ad614SDennis Dalessandro 639f48ad614SDennis Dalessandro if (etype != RHF_RCV_TYPE_IB) 640f48ad614SDennis Dalessandro goto next; 641f48ad614SDennis Dalessandro 64240442b30SMike Marciniszyn packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr); 643f2d8a0b3SDasaratharaman Chandramouli hdr = packet->hdr; 644cb427057SDon Hiatt lnh = ib_get_lnh(hdr); 645f48ad614SDennis Dalessandro 646f2d8a0b3SDasaratharaman Chandramouli if (lnh == HFI1_LRH_BTH) { 647f2d8a0b3SDasaratharaman Chandramouli packet->ohdr = &hdr->u.oth; 6489039746cSDon Hiatt packet->grh = NULL; 649f2d8a0b3SDasaratharaman Chandramouli } else if (lnh == HFI1_LRH_GRH) { 650f2d8a0b3SDasaratharaman Chandramouli packet->ohdr = &hdr->u.l.oth; 6519039746cSDon Hiatt packet->grh = &hdr->u.l.grh; 652f2d8a0b3SDasaratharaman Chandramouli } else { 653f48ad614SDennis Dalessandro goto next; /* just in case */ 654f2d8a0b3SDasaratharaman Chandramouli } 6555fd2b562SMitko Haralanov 656fe4dd423SMitko Haralanov if (!hfi1_may_ecn(packet)) 657f48ad614SDennis Dalessandro goto next; 658f48ad614SDennis Dalessandro 659fe4dd423SMitko Haralanov bth1 = be32_to_cpu(packet->ohdr->bth[1]); 660f48ad614SDennis Dalessandro qpn = bth1 & RVT_QPN_MASK; 661f48ad614SDennis Dalessandro rcu_read_lock(); 662f48ad614SDennis Dalessandro qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn); 663f48ad614SDennis Dalessandro 664f48ad614SDennis Dalessandro if (!qp) { 665f48ad614SDennis Dalessandro rcu_read_unlock(); 666f48ad614SDennis Dalessandro goto next; 667f48ad614SDennis Dalessandro } 668f48ad614SDennis Dalessandro 669fe4dd423SMitko Haralanov hfi1_process_ecn_slowpath(qp, packet, true); 670f48ad614SDennis Dalessandro rcu_read_unlock(); 671f48ad614SDennis Dalessandro 672f48ad614SDennis Dalessandro /* turn off BECN, FECN */ 6733d591099SDon Hiatt bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK); 674f2d8a0b3SDasaratharaman Chandramouli packet->ohdr->bth[1] = cpu_to_be32(bth1); 675f48ad614SDennis Dalessandro next: 676f48ad614SDennis Dalessandro update_ps_mdata(&mdata, rcd); 677f48ad614SDennis Dalessandro } 678f48ad614SDennis Dalessandro } 679f48ad614SDennis Dalessandro 680bdaf96f6SSebastian Sanchez static void process_rcv_qp_work(struct hfi1_packet *packet) 681a82a7fcdSMike Marciniszyn { 682a82a7fcdSMike Marciniszyn struct rvt_qp *qp, *nqp; 683bdaf96f6SSebastian Sanchez struct hfi1_ctxtdata *rcd = packet->rcd; 684a82a7fcdSMike Marciniszyn 685a82a7fcdSMike Marciniszyn /* 686a82a7fcdSMike Marciniszyn * Iterate over all QPs waiting to respond. 687a82a7fcdSMike Marciniszyn * The list won't change since the IRQ is only run on one CPU. 688a82a7fcdSMike Marciniszyn */ 689a82a7fcdSMike Marciniszyn list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) { 690a82a7fcdSMike Marciniszyn list_del_init(&qp->rspwait); 691a82a7fcdSMike Marciniszyn if (qp->r_flags & RVT_R_RSP_NAK) { 692a82a7fcdSMike Marciniszyn qp->r_flags &= ~RVT_R_RSP_NAK; 693bdaf96f6SSebastian Sanchez packet->qp = qp; 694bdaf96f6SSebastian Sanchez hfi1_send_rc_ack(packet, 0); 695a82a7fcdSMike Marciniszyn } 696a82a7fcdSMike Marciniszyn if (qp->r_flags & RVT_R_RSP_SEND) { 697a82a7fcdSMike Marciniszyn unsigned long flags; 698a82a7fcdSMike Marciniszyn 699a82a7fcdSMike Marciniszyn qp->r_flags &= ~RVT_R_RSP_SEND; 700a82a7fcdSMike Marciniszyn spin_lock_irqsave(&qp->s_lock, flags); 701a82a7fcdSMike Marciniszyn if (ib_rvt_state_ops[qp->state] & 702a82a7fcdSMike Marciniszyn RVT_PROCESS_OR_FLUSH_SEND) 703a82a7fcdSMike Marciniszyn hfi1_schedule_send(qp); 704a82a7fcdSMike Marciniszyn spin_unlock_irqrestore(&qp->s_lock, flags); 705a82a7fcdSMike Marciniszyn } 706a82a7fcdSMike Marciniszyn rvt_put_qp(qp); 707a82a7fcdSMike Marciniszyn } 708a82a7fcdSMike Marciniszyn } 709a82a7fcdSMike Marciniszyn 710a82a7fcdSMike Marciniszyn static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread) 711a82a7fcdSMike Marciniszyn { 712a82a7fcdSMike Marciniszyn if (thread) { 713a82a7fcdSMike Marciniszyn if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0) 714a82a7fcdSMike Marciniszyn /* allow defered processing */ 715bdaf96f6SSebastian Sanchez process_rcv_qp_work(packet); 716a82a7fcdSMike Marciniszyn cond_resched(); 717a82a7fcdSMike Marciniszyn return RCV_PKT_OK; 718a82a7fcdSMike Marciniszyn } else { 719a82a7fcdSMike Marciniszyn this_cpu_inc(*packet->rcd->dd->rcv_limit); 720a82a7fcdSMike Marciniszyn return RCV_PKT_LIMIT; 721a82a7fcdSMike Marciniszyn } 722a82a7fcdSMike Marciniszyn } 723a82a7fcdSMike Marciniszyn 724a82a7fcdSMike Marciniszyn static inline int check_max_packet(struct hfi1_packet *packet, int thread) 725f48ad614SDennis Dalessandro { 726f48ad614SDennis Dalessandro int ret = RCV_PKT_OK; 727f48ad614SDennis Dalessandro 728a82a7fcdSMike Marciniszyn if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0)) 729a82a7fcdSMike Marciniszyn ret = max_packet_exceeded(packet, thread); 730a82a7fcdSMike Marciniszyn return ret; 731a82a7fcdSMike Marciniszyn } 732a82a7fcdSMike Marciniszyn 733a82a7fcdSMike Marciniszyn static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread) 734a82a7fcdSMike Marciniszyn { 735a82a7fcdSMike Marciniszyn int ret; 736a82a7fcdSMike Marciniszyn 737f48ad614SDennis Dalessandro /* Set up for the next packet */ 738f48ad614SDennis Dalessandro packet->rhqoff += packet->rsize; 739f48ad614SDennis Dalessandro if (packet->rhqoff >= packet->maxcnt) 740f48ad614SDennis Dalessandro packet->rhqoff = 0; 741f48ad614SDennis Dalessandro 742f48ad614SDennis Dalessandro packet->numpkt++; 743a82a7fcdSMike Marciniszyn ret = check_max_packet(packet, thread); 744f48ad614SDennis Dalessandro 745f48ad614SDennis Dalessandro packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 74640442b30SMike Marciniszyn packet->rcd->rhf_offset; 747f48ad614SDennis Dalessandro packet->rhf = rhf_to_cpu(packet->rhf_addr); 748f48ad614SDennis Dalessandro 749f48ad614SDennis Dalessandro return ret; 750f48ad614SDennis Dalessandro } 751f48ad614SDennis Dalessandro 752f48ad614SDennis Dalessandro static inline int process_rcv_packet(struct hfi1_packet *packet, int thread) 753f48ad614SDennis Dalessandro { 754a82a7fcdSMike Marciniszyn int ret; 755f48ad614SDennis Dalessandro 756f48ad614SDennis Dalessandro packet->etype = rhf_rcv_type(packet->rhf); 7579039746cSDon Hiatt 758f48ad614SDennis Dalessandro /* total length */ 759f48ad614SDennis Dalessandro packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */ 760f48ad614SDennis Dalessandro /* retrieve eager buffer details */ 761f48ad614SDennis Dalessandro packet->ebuf = NULL; 762f48ad614SDennis Dalessandro if (rhf_use_egr_bfr(packet->rhf)) { 763f48ad614SDennis Dalessandro packet->etail = rhf_egr_index(packet->rhf); 764f48ad614SDennis Dalessandro packet->ebuf = get_egrbuf(packet->rcd, packet->rhf, 765f48ad614SDennis Dalessandro &packet->updegr); 766f48ad614SDennis Dalessandro /* 767f48ad614SDennis Dalessandro * Prefetch the contents of the eager buffer. It is 768f48ad614SDennis Dalessandro * OK to send a negative length to prefetch_range(). 769f48ad614SDennis Dalessandro * The +2 is the size of the RHF. 770f48ad614SDennis Dalessandro */ 771f48ad614SDennis Dalessandro prefetch_range(packet->ebuf, 772f48ad614SDennis Dalessandro packet->tlen - ((packet->rcd->rcvhdrqentsize - 773f48ad614SDennis Dalessandro (rhf_hdrq_offset(packet->rhf) 774f48ad614SDennis Dalessandro + 2)) * 4)); 775f48ad614SDennis Dalessandro } 776f48ad614SDennis Dalessandro 777f48ad614SDennis Dalessandro /* 778f48ad614SDennis Dalessandro * Call a type specific handler for the packet. We 779f48ad614SDennis Dalessandro * should be able to trust that etype won't be beyond 780f48ad614SDennis Dalessandro * the range of valid indexes. If so something is really 781f48ad614SDennis Dalessandro * wrong and we can probably just let things come 782f48ad614SDennis Dalessandro * crashing down. There is no need to eat another 783f48ad614SDennis Dalessandro * comparison in this performance critical code. 784f48ad614SDennis Dalessandro */ 785b0ba3c18SMike Marciniszyn packet->rcd->rhf_rcv_function_map[packet->etype](packet); 786f48ad614SDennis Dalessandro packet->numpkt++; 787f48ad614SDennis Dalessandro 788f48ad614SDennis Dalessandro /* Set up for the next packet */ 789f48ad614SDennis Dalessandro packet->rhqoff += packet->rsize; 790f48ad614SDennis Dalessandro if (packet->rhqoff >= packet->maxcnt) 791f48ad614SDennis Dalessandro packet->rhqoff = 0; 792f48ad614SDennis Dalessandro 793a82a7fcdSMike Marciniszyn ret = check_max_packet(packet, thread); 794f48ad614SDennis Dalessandro 795f48ad614SDennis Dalessandro packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff + 79640442b30SMike Marciniszyn packet->rcd->rhf_offset; 797f48ad614SDennis Dalessandro packet->rhf = rhf_to_cpu(packet->rhf_addr); 798f48ad614SDennis Dalessandro 799f48ad614SDennis Dalessandro return ret; 800f48ad614SDennis Dalessandro } 801f48ad614SDennis Dalessandro 802f48ad614SDennis Dalessandro static inline void process_rcv_update(int last, struct hfi1_packet *packet) 803f48ad614SDennis Dalessandro { 804f48ad614SDennis Dalessandro /* 805f48ad614SDennis Dalessandro * Update head regs etc., every 16 packets, if not last pkt, 806f48ad614SDennis Dalessandro * to help prevent rcvhdrq overflows, when many packets 807f48ad614SDennis Dalessandro * are processed and queue is nearly full. 808f48ad614SDennis Dalessandro * Don't request an interrupt for intermediate updates. 809f48ad614SDennis Dalessandro */ 810f48ad614SDennis Dalessandro if (!last && !(packet->numpkt & 0xf)) { 811f48ad614SDennis Dalessandro update_usrhead(packet->rcd, packet->rhqoff, packet->updegr, 812f48ad614SDennis Dalessandro packet->etail, 0, 0); 813f48ad614SDennis Dalessandro packet->updegr = 0; 814f48ad614SDennis Dalessandro } 8159039746cSDon Hiatt packet->grh = NULL; 816f48ad614SDennis Dalessandro } 817f48ad614SDennis Dalessandro 818f48ad614SDennis Dalessandro static inline void finish_packet(struct hfi1_packet *packet) 819f48ad614SDennis Dalessandro { 820f48ad614SDennis Dalessandro /* 821f48ad614SDennis Dalessandro * Nothing we need to free for the packet. 822f48ad614SDennis Dalessandro * 823f48ad614SDennis Dalessandro * The only thing we need to do is a final update and call for an 824f48ad614SDennis Dalessandro * interrupt 825f48ad614SDennis Dalessandro */ 826f48ad614SDennis Dalessandro update_usrhead(packet->rcd, packet->rcd->head, packet->updegr, 827f48ad614SDennis Dalessandro packet->etail, rcv_intr_dynamic, packet->numpkt); 828f48ad614SDennis Dalessandro } 829f48ad614SDennis Dalessandro 830f48ad614SDennis Dalessandro /* 831f48ad614SDennis Dalessandro * Handle receive interrupts when using the no dma rtail option. 832f48ad614SDennis Dalessandro */ 833f48ad614SDennis Dalessandro int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread) 834f48ad614SDennis Dalessandro { 835f48ad614SDennis Dalessandro u32 seq; 836f48ad614SDennis Dalessandro int last = RCV_PKT_OK; 837f48ad614SDennis Dalessandro struct hfi1_packet packet; 838f48ad614SDennis Dalessandro 839f48ad614SDennis Dalessandro init_packet(rcd, &packet); 840f48ad614SDennis Dalessandro seq = rhf_rcv_seq(packet.rhf); 841f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) { 842f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 843f48ad614SDennis Dalessandro goto bail; 844f48ad614SDennis Dalessandro } 845f48ad614SDennis Dalessandro 846f48ad614SDennis Dalessandro prescan_rxq(rcd, &packet); 847f48ad614SDennis Dalessandro 848f48ad614SDennis Dalessandro while (last == RCV_PKT_OK) { 849f48ad614SDennis Dalessandro last = process_rcv_packet(&packet, thread); 850f48ad614SDennis Dalessandro seq = rhf_rcv_seq(packet.rhf); 851f48ad614SDennis Dalessandro if (++rcd->seq_cnt > 13) 852f48ad614SDennis Dalessandro rcd->seq_cnt = 1; 853f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) 854f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 855f48ad614SDennis Dalessandro process_rcv_update(last, &packet); 856f48ad614SDennis Dalessandro } 857bdaf96f6SSebastian Sanchez process_rcv_qp_work(&packet); 858a82a7fcdSMike Marciniszyn rcd->head = packet.rhqoff; 859f48ad614SDennis Dalessandro bail: 860f48ad614SDennis Dalessandro finish_packet(&packet); 861f48ad614SDennis Dalessandro return last; 862f48ad614SDennis Dalessandro } 863f48ad614SDennis Dalessandro 864f48ad614SDennis Dalessandro int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread) 865f48ad614SDennis Dalessandro { 866f48ad614SDennis Dalessandro u32 hdrqtail; 867f48ad614SDennis Dalessandro int last = RCV_PKT_OK; 868f48ad614SDennis Dalessandro struct hfi1_packet packet; 869f48ad614SDennis Dalessandro 870f48ad614SDennis Dalessandro init_packet(rcd, &packet); 871f48ad614SDennis Dalessandro hdrqtail = get_rcvhdrtail(rcd); 872f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) { 873f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 874f48ad614SDennis Dalessandro goto bail; 875f48ad614SDennis Dalessandro } 876f48ad614SDennis Dalessandro smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 877f48ad614SDennis Dalessandro 878f48ad614SDennis Dalessandro prescan_rxq(rcd, &packet); 879f48ad614SDennis Dalessandro 880f48ad614SDennis Dalessandro while (last == RCV_PKT_OK) { 881f48ad614SDennis Dalessandro last = process_rcv_packet(&packet, thread); 882f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) 883f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 884f48ad614SDennis Dalessandro process_rcv_update(last, &packet); 885f48ad614SDennis Dalessandro } 886bdaf96f6SSebastian Sanchez process_rcv_qp_work(&packet); 887a82a7fcdSMike Marciniszyn rcd->head = packet.rhqoff; 888f48ad614SDennis Dalessandro bail: 889f48ad614SDennis Dalessandro finish_packet(&packet); 890f48ad614SDennis Dalessandro return last; 891f48ad614SDennis Dalessandro } 892f48ad614SDennis Dalessandro 893e6f7622dSMichael J. Ruhl static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt) 894f48ad614SDennis Dalessandro { 895d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd; 896e6f7622dSMichael J. Ruhl u16 i; 897f48ad614SDennis Dalessandro 8982280740fSVishwanathapura, Niranjana /* 8992280740fSVishwanathapura, Niranjana * For dynamically allocated kernel contexts (like vnic) switch 9002280740fSVishwanathapura, Niranjana * interrupt handler only for that context. Otherwise, switch 9012280740fSVishwanathapura, Niranjana * interrupt handler for all statically allocated kernel contexts. 9022280740fSVishwanathapura, Niranjana */ 9032280740fSVishwanathapura, Niranjana if (ctxt >= dd->first_dyn_alloc_ctxt) { 904d59075adSMichael J. Ruhl rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); 905d295dbebSMichael J. Ruhl if (rcd) { 906d295dbebSMichael J. Ruhl rcd->do_interrupt = 9072280740fSVishwanathapura, Niranjana &handle_receive_interrupt_nodma_rtail; 908d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd); 909d295dbebSMichael J. Ruhl } 9102280740fSVishwanathapura, Niranjana return; 9112280740fSVishwanathapura, Niranjana } 9122280740fSVishwanathapura, Niranjana 913d295dbebSMichael J. Ruhl for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { 914d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i); 915d295dbebSMichael J. Ruhl if (rcd) 916d295dbebSMichael J. Ruhl rcd->do_interrupt = 917f48ad614SDennis Dalessandro &handle_receive_interrupt_nodma_rtail; 918d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd); 919d295dbebSMichael J. Ruhl } 920f48ad614SDennis Dalessandro } 921f48ad614SDennis Dalessandro 922e6f7622dSMichael J. Ruhl static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt) 923f48ad614SDennis Dalessandro { 924d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd; 925e6f7622dSMichael J. Ruhl u16 i; 926f48ad614SDennis Dalessandro 9272280740fSVishwanathapura, Niranjana /* 9282280740fSVishwanathapura, Niranjana * For dynamically allocated kernel contexts (like vnic) switch 9292280740fSVishwanathapura, Niranjana * interrupt handler only for that context. Otherwise, switch 9302280740fSVishwanathapura, Niranjana * interrupt handler for all statically allocated kernel contexts. 9312280740fSVishwanathapura, Niranjana */ 9322280740fSVishwanathapura, Niranjana if (ctxt >= dd->first_dyn_alloc_ctxt) { 933d59075adSMichael J. Ruhl rcd = hfi1_rcd_get_by_index_safe(dd, ctxt); 934d295dbebSMichael J. Ruhl if (rcd) { 935d295dbebSMichael J. Ruhl rcd->do_interrupt = 9362280740fSVishwanathapura, Niranjana &handle_receive_interrupt_dma_rtail; 937d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd); 938d295dbebSMichael J. Ruhl } 9392280740fSVishwanathapura, Niranjana return; 9402280740fSVishwanathapura, Niranjana } 9412280740fSVishwanathapura, Niranjana 942d295dbebSMichael J. Ruhl for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) { 943d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i); 944d295dbebSMichael J. Ruhl if (rcd) 945d295dbebSMichael J. Ruhl rcd->do_interrupt = 946f48ad614SDennis Dalessandro &handle_receive_interrupt_dma_rtail; 947d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd); 948d295dbebSMichael J. Ruhl } 949f48ad614SDennis Dalessandro } 950f48ad614SDennis Dalessandro 951f48ad614SDennis Dalessandro void set_all_slowpath(struct hfi1_devdata *dd) 952f48ad614SDennis Dalessandro { 953d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd; 954e6f7622dSMichael J. Ruhl u16 i; 955f48ad614SDennis Dalessandro 956f48ad614SDennis Dalessandro /* HFI1_CTRL_CTXT must always use the slow path interrupt handler */ 9572280740fSVishwanathapura, Niranjana for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) { 958d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i); 959d295dbebSMichael J. Ruhl if (!rcd) 960d295dbebSMichael J. Ruhl continue; 961cc9a97eaSNiranjana Vishwanathapura if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic) 9622280740fSVishwanathapura, Niranjana rcd->do_interrupt = &handle_receive_interrupt; 963cc9a97eaSNiranjana Vishwanathapura 964d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd); 965d295dbebSMichael J. Ruhl } 966f48ad614SDennis Dalessandro } 967f48ad614SDennis Dalessandro 968f48ad614SDennis Dalessandro static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd, 969c867caafSMike Marciniszyn struct hfi1_packet *packet, 970f48ad614SDennis Dalessandro struct hfi1_devdata *dd) 971f48ad614SDennis Dalessandro { 972f48ad614SDennis Dalessandro struct work_struct *lsaw = &rcd->ppd->linkstate_active_work; 9739039746cSDon Hiatt u8 etype = rhf_rcv_type(packet->rhf); 9749039746cSDon Hiatt u8 sc = SC15_PACKET; 9759039746cSDon Hiatt 9769039746cSDon Hiatt if (etype == RHF_RCV_TYPE_IB) { 97740442b30SMike Marciniszyn struct ib_header *hdr = hfi1_get_msgheader(packet->rcd, 978c867caafSMike Marciniszyn packet->rhf_addr); 9799039746cSDon Hiatt sc = hfi1_9B_get_sc5(hdr, packet->rhf); 98072c07e2bSDon Hiatt } else if (etype == RHF_RCV_TYPE_BYPASS) { 98172c07e2bSDon Hiatt struct hfi1_16b_header *hdr = hfi1_get_16B_header( 98240442b30SMike Marciniszyn packet->rcd, 98372c07e2bSDon Hiatt packet->rhf_addr); 98472c07e2bSDon Hiatt sc = hfi1_16B_get_sc(hdr); 9859039746cSDon Hiatt } 9869039746cSDon Hiatt if (sc != SC15_PACKET) { 987bec7c79cSByczkowski, Jakub int hwstate = driver_lstate(rcd->ppd); 988f48ad614SDennis Dalessandro 989bec7c79cSByczkowski, Jakub if (hwstate != IB_PORT_ACTIVE) { 990bec7c79cSByczkowski, Jakub dd_dev_info(dd, 991bec7c79cSByczkowski, Jakub "Unexpected link state %s\n", 992bec7c79cSByczkowski, Jakub opa_lstate_name(hwstate)); 993f48ad614SDennis Dalessandro return 0; 994f48ad614SDennis Dalessandro } 995f48ad614SDennis Dalessandro 99671d47008SSebastian Sanchez queue_work(rcd->ppd->link_wq, lsaw); 997f48ad614SDennis Dalessandro return 1; 998f48ad614SDennis Dalessandro } 999f48ad614SDennis Dalessandro return 0; 1000f48ad614SDennis Dalessandro } 1001f48ad614SDennis Dalessandro 1002f48ad614SDennis Dalessandro /* 1003f48ad614SDennis Dalessandro * handle_receive_interrupt - receive a packet 1004f48ad614SDennis Dalessandro * @rcd: the context 1005f48ad614SDennis Dalessandro * 1006f48ad614SDennis Dalessandro * Called from interrupt handler for errors or receive interrupt. 1007f48ad614SDennis Dalessandro * This is the slow path interrupt handler. 1008f48ad614SDennis Dalessandro */ 1009f48ad614SDennis Dalessandro int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread) 1010f48ad614SDennis Dalessandro { 1011f48ad614SDennis Dalessandro struct hfi1_devdata *dd = rcd->dd; 1012f48ad614SDennis Dalessandro u32 hdrqtail; 1013f48ad614SDennis Dalessandro int needset, last = RCV_PKT_OK; 1014f48ad614SDennis Dalessandro struct hfi1_packet packet; 1015f48ad614SDennis Dalessandro int skip_pkt = 0; 1016f48ad614SDennis Dalessandro 1017f48ad614SDennis Dalessandro /* Control context will always use the slow path interrupt handler */ 1018f48ad614SDennis Dalessandro needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1; 1019f48ad614SDennis Dalessandro 1020f48ad614SDennis Dalessandro init_packet(rcd, &packet); 1021f48ad614SDennis Dalessandro 1022f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 1023f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 1024f48ad614SDennis Dalessandro 1025f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) { 1026f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 1027f48ad614SDennis Dalessandro goto bail; 1028f48ad614SDennis Dalessandro } 1029f48ad614SDennis Dalessandro hdrqtail = 0; 1030f48ad614SDennis Dalessandro } else { 1031f48ad614SDennis Dalessandro hdrqtail = get_rcvhdrtail(rcd); 1032f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) { 1033f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 1034f48ad614SDennis Dalessandro goto bail; 1035f48ad614SDennis Dalessandro } 1036f48ad614SDennis Dalessandro smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ 1037f48ad614SDennis Dalessandro 1038f48ad614SDennis Dalessandro /* 1039f48ad614SDennis Dalessandro * Control context can potentially receive an invalid 1040f48ad614SDennis Dalessandro * rhf. Drop such packets. 1041f48ad614SDennis Dalessandro */ 1042f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT) { 1043f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 1044f48ad614SDennis Dalessandro 1045f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) 1046f48ad614SDennis Dalessandro skip_pkt = 1; 1047f48ad614SDennis Dalessandro } 1048f48ad614SDennis Dalessandro } 1049f48ad614SDennis Dalessandro 1050f48ad614SDennis Dalessandro prescan_rxq(rcd, &packet); 1051f48ad614SDennis Dalessandro 1052f48ad614SDennis Dalessandro while (last == RCV_PKT_OK) { 1053f48ad614SDennis Dalessandro if (unlikely(dd->do_drop && 1054f48ad614SDennis Dalessandro atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) == 1055f48ad614SDennis Dalessandro DROP_PACKET_ON)) { 1056f48ad614SDennis Dalessandro dd->do_drop = 0; 1057f48ad614SDennis Dalessandro 1058f48ad614SDennis Dalessandro /* On to the next packet */ 1059f48ad614SDennis Dalessandro packet.rhqoff += packet.rsize; 1060f48ad614SDennis Dalessandro packet.rhf_addr = (__le32 *)rcd->rcvhdrq + 1061f48ad614SDennis Dalessandro packet.rhqoff + 106240442b30SMike Marciniszyn rcd->rhf_offset; 1063f48ad614SDennis Dalessandro packet.rhf = rhf_to_cpu(packet.rhf_addr); 1064f48ad614SDennis Dalessandro 1065f48ad614SDennis Dalessandro } else if (skip_pkt) { 1066f48ad614SDennis Dalessandro last = skip_rcv_packet(&packet, thread); 1067f48ad614SDennis Dalessandro skip_pkt = 0; 1068f48ad614SDennis Dalessandro } else { 1069f48ad614SDennis Dalessandro /* Auto activate link on non-SC15 packet receive */ 1070f48ad614SDennis Dalessandro if (unlikely(rcd->ppd->host_link_state == 1071f48ad614SDennis Dalessandro HLS_UP_ARMED) && 1072c867caafSMike Marciniszyn set_armed_to_active(rcd, &packet, dd)) 1073f48ad614SDennis Dalessandro goto bail; 1074f48ad614SDennis Dalessandro last = process_rcv_packet(&packet, thread); 1075f48ad614SDennis Dalessandro } 1076f48ad614SDennis Dalessandro 1077f48ad614SDennis Dalessandro if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) { 1078f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 1079f48ad614SDennis Dalessandro 1080f48ad614SDennis Dalessandro if (++rcd->seq_cnt > 13) 1081f48ad614SDennis Dalessandro rcd->seq_cnt = 1; 1082f48ad614SDennis Dalessandro if (seq != rcd->seq_cnt) 1083f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 1084f48ad614SDennis Dalessandro if (needset) { 1085f48ad614SDennis Dalessandro dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n"); 10862280740fSVishwanathapura, Niranjana set_nodma_rtail(dd, rcd->ctxt); 1087f48ad614SDennis Dalessandro needset = 0; 1088f48ad614SDennis Dalessandro } 1089f48ad614SDennis Dalessandro } else { 1090f48ad614SDennis Dalessandro if (packet.rhqoff == hdrqtail) 1091f48ad614SDennis Dalessandro last = RCV_PKT_DONE; 1092f48ad614SDennis Dalessandro /* 1093f48ad614SDennis Dalessandro * Control context can potentially receive an invalid 1094f48ad614SDennis Dalessandro * rhf. Drop such packets. 1095f48ad614SDennis Dalessandro */ 1096f48ad614SDennis Dalessandro if (rcd->ctxt == HFI1_CTRL_CTXT) { 1097f48ad614SDennis Dalessandro u32 seq = rhf_rcv_seq(packet.rhf); 1098f48ad614SDennis Dalessandro 1099f48ad614SDennis Dalessandro if (++rcd->seq_cnt > 13) 1100f48ad614SDennis Dalessandro rcd->seq_cnt = 1; 1101f48ad614SDennis Dalessandro if (!last && (seq != rcd->seq_cnt)) 1102f48ad614SDennis Dalessandro skip_pkt = 1; 1103f48ad614SDennis Dalessandro } 1104f48ad614SDennis Dalessandro 1105f48ad614SDennis Dalessandro if (needset) { 1106f48ad614SDennis Dalessandro dd_dev_info(dd, 1107f48ad614SDennis Dalessandro "Switching to DMA_RTAIL\n"); 11082280740fSVishwanathapura, Niranjana set_dma_rtail(dd, rcd->ctxt); 1109f48ad614SDennis Dalessandro needset = 0; 1110f48ad614SDennis Dalessandro } 1111f48ad614SDennis Dalessandro } 1112f48ad614SDennis Dalessandro 1113f48ad614SDennis Dalessandro process_rcv_update(last, &packet); 1114f48ad614SDennis Dalessandro } 1115f48ad614SDennis Dalessandro 1116bdaf96f6SSebastian Sanchez process_rcv_qp_work(&packet); 1117a82a7fcdSMike Marciniszyn rcd->head = packet.rhqoff; 1118f48ad614SDennis Dalessandro 1119f48ad614SDennis Dalessandro bail: 1120f48ad614SDennis Dalessandro /* 1121f48ad614SDennis Dalessandro * Always write head at end, and setup rcv interrupt, even 1122f48ad614SDennis Dalessandro * if no packets were processed. 1123f48ad614SDennis Dalessandro */ 1124f48ad614SDennis Dalessandro finish_packet(&packet); 1125f48ad614SDennis Dalessandro return last; 1126f48ad614SDennis Dalessandro } 1127f48ad614SDennis Dalessandro 1128f48ad614SDennis Dalessandro /* 1129f48ad614SDennis Dalessandro * We may discover in the interrupt that the hardware link state has 1130f48ad614SDennis Dalessandro * changed from ARMED to ACTIVE (due to the arrival of a non-SC15 packet), 1131f48ad614SDennis Dalessandro * and we need to update the driver's notion of the link state. We cannot 1132f48ad614SDennis Dalessandro * run set_link_state from interrupt context, so we queue this function on 1133f48ad614SDennis Dalessandro * a workqueue. 1134f48ad614SDennis Dalessandro * 1135f48ad614SDennis Dalessandro * We delay the regular interrupt processing until after the state changes 1136f48ad614SDennis Dalessandro * so that the link will be in the correct state by the time any application 1137f48ad614SDennis Dalessandro * we wake up attempts to send a reply to any message it received. 1138f48ad614SDennis Dalessandro * (Subsequent receive interrupts may possibly force the wakeup before we 1139f48ad614SDennis Dalessandro * update the link state.) 1140f48ad614SDennis Dalessandro * 1141f48ad614SDennis Dalessandro * The rcd is freed in hfi1_free_ctxtdata after hfi1_postinit_cleanup invokes 1142f48ad614SDennis Dalessandro * dd->f_cleanup(dd) to disable the interrupt handler and flush workqueues, 1143f48ad614SDennis Dalessandro * so we're safe from use-after-free of the rcd. 1144f48ad614SDennis Dalessandro */ 1145f48ad614SDennis Dalessandro void receive_interrupt_work(struct work_struct *work) 1146f48ad614SDennis Dalessandro { 1147f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 1148f48ad614SDennis Dalessandro linkstate_active_work); 1149f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1150d295dbebSMichael J. Ruhl struct hfi1_ctxtdata *rcd; 1151e6f7622dSMichael J. Ruhl u16 i; 1152f48ad614SDennis Dalessandro 1153f48ad614SDennis Dalessandro /* Received non-SC15 packet implies neighbor_normal */ 1154f48ad614SDennis Dalessandro ppd->neighbor_normal = 1; 1155f48ad614SDennis Dalessandro set_link_state(ppd, HLS_UP_ACTIVE); 1156f48ad614SDennis Dalessandro 1157f48ad614SDennis Dalessandro /* 11582280740fSVishwanathapura, Niranjana * Interrupt all statically allocated kernel contexts that could 11592280740fSVishwanathapura, Niranjana * have had an interrupt during auto activation. 1160f48ad614SDennis Dalessandro */ 1161d295dbebSMichael J. Ruhl for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) { 1162d295dbebSMichael J. Ruhl rcd = hfi1_rcd_get_by_index(dd, i); 1163d295dbebSMichael J. Ruhl if (rcd) 1164d295dbebSMichael J. Ruhl force_recv_intr(rcd); 1165d295dbebSMichael J. Ruhl hfi1_rcd_put(rcd); 1166d295dbebSMichael J. Ruhl } 1167f48ad614SDennis Dalessandro } 1168f48ad614SDennis Dalessandro 1169f48ad614SDennis Dalessandro /* 1170f48ad614SDennis Dalessandro * Convert a given MTU size to the on-wire MAD packet enumeration. 1171f48ad614SDennis Dalessandro * Return -1 if the size is invalid. 1172f48ad614SDennis Dalessandro */ 1173f48ad614SDennis Dalessandro int mtu_to_enum(u32 mtu, int default_if_bad) 1174f48ad614SDennis Dalessandro { 1175f48ad614SDennis Dalessandro switch (mtu) { 1176f48ad614SDennis Dalessandro case 0: return OPA_MTU_0; 1177f48ad614SDennis Dalessandro case 256: return OPA_MTU_256; 1178f48ad614SDennis Dalessandro case 512: return OPA_MTU_512; 1179f48ad614SDennis Dalessandro case 1024: return OPA_MTU_1024; 1180f48ad614SDennis Dalessandro case 2048: return OPA_MTU_2048; 1181f48ad614SDennis Dalessandro case 4096: return OPA_MTU_4096; 1182f48ad614SDennis Dalessandro case 8192: return OPA_MTU_8192; 1183f48ad614SDennis Dalessandro case 10240: return OPA_MTU_10240; 1184f48ad614SDennis Dalessandro } 1185f48ad614SDennis Dalessandro return default_if_bad; 1186f48ad614SDennis Dalessandro } 1187f48ad614SDennis Dalessandro 1188f48ad614SDennis Dalessandro u16 enum_to_mtu(int mtu) 1189f48ad614SDennis Dalessandro { 1190f48ad614SDennis Dalessandro switch (mtu) { 1191f48ad614SDennis Dalessandro case OPA_MTU_0: return 0; 1192f48ad614SDennis Dalessandro case OPA_MTU_256: return 256; 1193f48ad614SDennis Dalessandro case OPA_MTU_512: return 512; 1194f48ad614SDennis Dalessandro case OPA_MTU_1024: return 1024; 1195f48ad614SDennis Dalessandro case OPA_MTU_2048: return 2048; 1196f48ad614SDennis Dalessandro case OPA_MTU_4096: return 4096; 1197f48ad614SDennis Dalessandro case OPA_MTU_8192: return 8192; 1198f48ad614SDennis Dalessandro case OPA_MTU_10240: return 10240; 1199f48ad614SDennis Dalessandro default: return 0xffff; 1200f48ad614SDennis Dalessandro } 1201f48ad614SDennis Dalessandro } 1202f48ad614SDennis Dalessandro 1203f48ad614SDennis Dalessandro /* 1204f48ad614SDennis Dalessandro * set_mtu - set the MTU 1205f48ad614SDennis Dalessandro * @ppd: the per port data 1206f48ad614SDennis Dalessandro * 1207f48ad614SDennis Dalessandro * We can handle "any" incoming size, the issue here is whether we 1208f48ad614SDennis Dalessandro * need to restrict our outgoing size. We do not deal with what happens 1209f48ad614SDennis Dalessandro * to programs that are already running when the size changes. 1210f48ad614SDennis Dalessandro */ 1211f48ad614SDennis Dalessandro int set_mtu(struct hfi1_pportdata *ppd) 1212f48ad614SDennis Dalessandro { 1213f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1214f48ad614SDennis Dalessandro int i, drain, ret = 0, is_up = 0; 1215f48ad614SDennis Dalessandro 1216f48ad614SDennis Dalessandro ppd->ibmtu = 0; 1217f48ad614SDennis Dalessandro for (i = 0; i < ppd->vls_supported; i++) 1218f48ad614SDennis Dalessandro if (ppd->ibmtu < dd->vld[i].mtu) 1219f48ad614SDennis Dalessandro ppd->ibmtu = dd->vld[i].mtu; 1220f48ad614SDennis Dalessandro ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd); 1221f48ad614SDennis Dalessandro 1222f48ad614SDennis Dalessandro mutex_lock(&ppd->hls_lock); 1223f48ad614SDennis Dalessandro if (ppd->host_link_state == HLS_UP_INIT || 1224f48ad614SDennis Dalessandro ppd->host_link_state == HLS_UP_ARMED || 1225f48ad614SDennis Dalessandro ppd->host_link_state == HLS_UP_ACTIVE) 1226f48ad614SDennis Dalessandro is_up = 1; 1227f48ad614SDennis Dalessandro 1228f48ad614SDennis Dalessandro drain = !is_ax(dd) && is_up; 1229f48ad614SDennis Dalessandro 1230f48ad614SDennis Dalessandro if (drain) 1231f48ad614SDennis Dalessandro /* 1232f48ad614SDennis Dalessandro * MTU is specified per-VL. To ensure that no packet gets 1233f48ad614SDennis Dalessandro * stuck (due, e.g., to the MTU for the packet's VL being 1234f48ad614SDennis Dalessandro * reduced), empty the per-VL FIFOs before adjusting MTU. 1235f48ad614SDennis Dalessandro */ 1236f48ad614SDennis Dalessandro ret = stop_drain_data_vls(dd); 1237f48ad614SDennis Dalessandro 1238f48ad614SDennis Dalessandro if (ret) { 1239f48ad614SDennis Dalessandro dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n", 1240f48ad614SDennis Dalessandro __func__); 1241f48ad614SDennis Dalessandro goto err; 1242f48ad614SDennis Dalessandro } 1243f48ad614SDennis Dalessandro 1244f48ad614SDennis Dalessandro hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0); 1245f48ad614SDennis Dalessandro 1246f48ad614SDennis Dalessandro if (drain) 1247f48ad614SDennis Dalessandro open_fill_data_vls(dd); /* reopen all VLs */ 1248f48ad614SDennis Dalessandro 1249f48ad614SDennis Dalessandro err: 1250f48ad614SDennis Dalessandro mutex_unlock(&ppd->hls_lock); 1251f48ad614SDennis Dalessandro 1252f48ad614SDennis Dalessandro return ret; 1253f48ad614SDennis Dalessandro } 1254f48ad614SDennis Dalessandro 1255f48ad614SDennis Dalessandro int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc) 1256f48ad614SDennis Dalessandro { 1257f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1258f48ad614SDennis Dalessandro 1259f48ad614SDennis Dalessandro ppd->lid = lid; 1260f48ad614SDennis Dalessandro ppd->lmc = lmc; 1261f48ad614SDennis Dalessandro hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0); 1262f48ad614SDennis Dalessandro 1263f48ad614SDennis Dalessandro dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid); 1264f48ad614SDennis Dalessandro 1265f48ad614SDennis Dalessandro return 0; 1266f48ad614SDennis Dalessandro } 1267f48ad614SDennis Dalessandro 1268f48ad614SDennis Dalessandro void shutdown_led_override(struct hfi1_pportdata *ppd) 1269f48ad614SDennis Dalessandro { 1270f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1271f48ad614SDennis Dalessandro 1272f48ad614SDennis Dalessandro /* 1273f48ad614SDennis Dalessandro * This pairs with the memory barrier in hfi1_start_led_override to 1274f48ad614SDennis Dalessandro * ensure that we read the correct state of LED beaconing represented 1275f48ad614SDennis Dalessandro * by led_override_timer_active 1276f48ad614SDennis Dalessandro */ 1277f48ad614SDennis Dalessandro smp_rmb(); 1278f48ad614SDennis Dalessandro if (atomic_read(&ppd->led_override_timer_active)) { 1279f48ad614SDennis Dalessandro del_timer_sync(&ppd->led_override_timer); 1280f48ad614SDennis Dalessandro atomic_set(&ppd->led_override_timer_active, 0); 1281f48ad614SDennis Dalessandro /* Ensure the atomic_set is visible to all CPUs */ 1282f48ad614SDennis Dalessandro smp_wmb(); 1283f48ad614SDennis Dalessandro } 1284f48ad614SDennis Dalessandro 1285f48ad614SDennis Dalessandro /* Hand control of the LED to the DC for normal operation */ 1286f48ad614SDennis Dalessandro write_csr(dd, DCC_CFG_LED_CNTRL, 0); 1287f48ad614SDennis Dalessandro } 1288f48ad614SDennis Dalessandro 12898064135eSKees Cook static void run_led_override(struct timer_list *t) 1290f48ad614SDennis Dalessandro { 12918064135eSKees Cook struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer); 1292f48ad614SDennis Dalessandro struct hfi1_devdata *dd = ppd->dd; 1293f48ad614SDennis Dalessandro unsigned long timeout; 1294f48ad614SDennis Dalessandro int phase_idx; 1295f48ad614SDennis Dalessandro 1296f48ad614SDennis Dalessandro if (!(dd->flags & HFI1_INITTED)) 1297f48ad614SDennis Dalessandro return; 1298f48ad614SDennis Dalessandro 1299f48ad614SDennis Dalessandro phase_idx = ppd->led_override_phase & 1; 1300f48ad614SDennis Dalessandro 1301f48ad614SDennis Dalessandro setextled(dd, phase_idx); 1302f48ad614SDennis Dalessandro 1303f48ad614SDennis Dalessandro timeout = ppd->led_override_vals[phase_idx]; 1304f48ad614SDennis Dalessandro 1305f48ad614SDennis Dalessandro /* Set up for next phase */ 1306f48ad614SDennis Dalessandro ppd->led_override_phase = !ppd->led_override_phase; 1307f48ad614SDennis Dalessandro 1308f48ad614SDennis Dalessandro mod_timer(&ppd->led_override_timer, jiffies + timeout); 1309f48ad614SDennis Dalessandro } 1310f48ad614SDennis Dalessandro 1311f48ad614SDennis Dalessandro /* 1312f48ad614SDennis Dalessandro * To have the LED blink in a particular pattern, provide timeon and timeoff 1313f48ad614SDennis Dalessandro * in milliseconds. 1314f48ad614SDennis Dalessandro * To turn off custom blinking and return to normal operation, use 1315f48ad614SDennis Dalessandro * shutdown_led_override() 1316f48ad614SDennis Dalessandro */ 1317f48ad614SDennis Dalessandro void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon, 1318f48ad614SDennis Dalessandro unsigned int timeoff) 1319f48ad614SDennis Dalessandro { 1320f48ad614SDennis Dalessandro if (!(ppd->dd->flags & HFI1_INITTED)) 1321f48ad614SDennis Dalessandro return; 1322f48ad614SDennis Dalessandro 1323f48ad614SDennis Dalessandro /* Convert to jiffies for direct use in timer */ 1324f48ad614SDennis Dalessandro ppd->led_override_vals[0] = msecs_to_jiffies(timeoff); 1325f48ad614SDennis Dalessandro ppd->led_override_vals[1] = msecs_to_jiffies(timeon); 1326f48ad614SDennis Dalessandro 1327f48ad614SDennis Dalessandro /* Arbitrarily start from LED on phase */ 1328f48ad614SDennis Dalessandro ppd->led_override_phase = 1; 1329f48ad614SDennis Dalessandro 1330f48ad614SDennis Dalessandro /* 1331f48ad614SDennis Dalessandro * If the timer has not already been started, do so. Use a "quick" 1332f48ad614SDennis Dalessandro * timeout so the handler will be called soon to look at our request. 1333f48ad614SDennis Dalessandro */ 1334f48ad614SDennis Dalessandro if (!timer_pending(&ppd->led_override_timer)) { 13358064135eSKees Cook timer_setup(&ppd->led_override_timer, run_led_override, 0); 1336f48ad614SDennis Dalessandro ppd->led_override_timer.expires = jiffies + 1; 1337f48ad614SDennis Dalessandro add_timer(&ppd->led_override_timer); 1338f48ad614SDennis Dalessandro atomic_set(&ppd->led_override_timer_active, 1); 1339f48ad614SDennis Dalessandro /* Ensure the atomic_set is visible to all CPUs */ 1340f48ad614SDennis Dalessandro smp_wmb(); 1341f48ad614SDennis Dalessandro } 1342f48ad614SDennis Dalessandro } 1343f48ad614SDennis Dalessandro 1344f48ad614SDennis Dalessandro /** 1345f48ad614SDennis Dalessandro * hfi1_reset_device - reset the chip if possible 1346f48ad614SDennis Dalessandro * @unit: the device to reset 1347f48ad614SDennis Dalessandro * 1348f48ad614SDennis Dalessandro * Whether or not reset is successful, we attempt to re-initialize the chip 1349f48ad614SDennis Dalessandro * (that is, much like a driver unload/reload). We clear the INITTED flag 1350f48ad614SDennis Dalessandro * so that the various entry points will fail until we reinitialize. For 1351f48ad614SDennis Dalessandro * now, we only allow this if no user contexts are open that use chip resources 1352f48ad614SDennis Dalessandro */ 1353f48ad614SDennis Dalessandro int hfi1_reset_device(int unit) 1354f48ad614SDennis Dalessandro { 1355e6f7622dSMichael J. Ruhl int ret; 1356f48ad614SDennis Dalessandro struct hfi1_devdata *dd = hfi1_lookup(unit); 1357f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd; 1358f48ad614SDennis Dalessandro int pidx; 1359f48ad614SDennis Dalessandro 1360f48ad614SDennis Dalessandro if (!dd) { 1361f48ad614SDennis Dalessandro ret = -ENODEV; 1362f48ad614SDennis Dalessandro goto bail; 1363f48ad614SDennis Dalessandro } 1364f48ad614SDennis Dalessandro 1365f48ad614SDennis Dalessandro dd_dev_info(dd, "Reset on unit %u requested\n", unit); 1366f48ad614SDennis Dalessandro 1367cb51c5d2SMike Marciniszyn if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) { 1368f48ad614SDennis Dalessandro dd_dev_info(dd, 1369f48ad614SDennis Dalessandro "Invalid unit number %u or not initialized or not present\n", 1370f48ad614SDennis Dalessandro unit); 1371f48ad614SDennis Dalessandro ret = -ENXIO; 1372f48ad614SDennis Dalessandro goto bail; 1373f48ad614SDennis Dalessandro } 1374f48ad614SDennis Dalessandro 1375d295dbebSMichael J. Ruhl /* If there are any user/vnic contexts, we cannot reset */ 1376d295dbebSMichael J. Ruhl mutex_lock(&hfi1_mutex); 1377f48ad614SDennis Dalessandro if (dd->rcd) 1378d295dbebSMichael J. Ruhl if (hfi1_stats.sps_ctxts) { 1379d295dbebSMichael J. Ruhl mutex_unlock(&hfi1_mutex); 1380f48ad614SDennis Dalessandro ret = -EBUSY; 1381f48ad614SDennis Dalessandro goto bail; 1382f48ad614SDennis Dalessandro } 1383d295dbebSMichael J. Ruhl mutex_unlock(&hfi1_mutex); 1384f48ad614SDennis Dalessandro 1385f48ad614SDennis Dalessandro for (pidx = 0; pidx < dd->num_pports; ++pidx) { 1386f48ad614SDennis Dalessandro ppd = dd->pport + pidx; 1387f48ad614SDennis Dalessandro 1388f48ad614SDennis Dalessandro shutdown_led_override(ppd); 1389f48ad614SDennis Dalessandro } 1390f48ad614SDennis Dalessandro if (dd->flags & HFI1_HAS_SEND_DMA) 1391f48ad614SDennis Dalessandro sdma_exit(dd); 1392f48ad614SDennis Dalessandro 1393f48ad614SDennis Dalessandro hfi1_reset_cpu_counters(dd); 1394f48ad614SDennis Dalessandro 1395f48ad614SDennis Dalessandro ret = hfi1_init(dd, 1); 1396f48ad614SDennis Dalessandro 1397f48ad614SDennis Dalessandro if (ret) 1398f48ad614SDennis Dalessandro dd_dev_err(dd, 1399f48ad614SDennis Dalessandro "Reinitialize unit %u after reset failed with %d\n", 1400f48ad614SDennis Dalessandro unit, ret); 1401f48ad614SDennis Dalessandro else 1402f48ad614SDennis Dalessandro dd_dev_info(dd, "Reinitialized unit %u after resetting\n", 1403f48ad614SDennis Dalessandro unit); 1404f48ad614SDennis Dalessandro 1405f48ad614SDennis Dalessandro bail: 1406f48ad614SDennis Dalessandro return ret; 1407f48ad614SDennis Dalessandro } 1408f48ad614SDennis Dalessandro 14099039746cSDon Hiatt static inline void hfi1_setup_ib_header(struct hfi1_packet *packet) 14109039746cSDon Hiatt { 14119039746cSDon Hiatt packet->hdr = (struct hfi1_ib_message_header *) 141240442b30SMike Marciniszyn hfi1_get_msgheader(packet->rcd, 14139039746cSDon Hiatt packet->rhf_addr); 14149039746cSDon Hiatt packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr; 14159039746cSDon Hiatt } 14169039746cSDon Hiatt 14175786adf3SDon Hiatt static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet) 14185786adf3SDon Hiatt { 14195786adf3SDon Hiatt struct hfi1_pportdata *ppd = packet->rcd->ppd; 14205786adf3SDon Hiatt 14215786adf3SDon Hiatt /* slid and dlid cannot be 0 */ 14225786adf3SDon Hiatt if ((!packet->slid) || (!packet->dlid)) 14235786adf3SDon Hiatt return -EINVAL; 14245786adf3SDon Hiatt 14255786adf3SDon Hiatt /* Compare port lid with incoming packet dlid */ 14265786adf3SDon Hiatt if ((!(hfi1_is_16B_mcast(packet->dlid))) && 14275786adf3SDon Hiatt (packet->dlid != 14285786adf3SDon Hiatt opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) { 1429ff8b67fcSAshutosh Dixit if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid) 14305786adf3SDon Hiatt return -EINVAL; 14315786adf3SDon Hiatt } 14325786adf3SDon Hiatt 14335786adf3SDon Hiatt /* No multicast packets with SC15 */ 14345786adf3SDon Hiatt if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF)) 14355786adf3SDon Hiatt return -EINVAL; 14365786adf3SDon Hiatt 14375786adf3SDon Hiatt /* Packets with permissive DLID always on SC15 */ 14385786adf3SDon Hiatt if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 14395786adf3SDon Hiatt 16B)) && 14405786adf3SDon Hiatt (packet->sc != 0xF)) 14415786adf3SDon Hiatt return -EINVAL; 14425786adf3SDon Hiatt 14435786adf3SDon Hiatt return 0; 14445786adf3SDon Hiatt } 14455786adf3SDon Hiatt 14469039746cSDon Hiatt static int hfi1_setup_9B_packet(struct hfi1_packet *packet) 14479039746cSDon Hiatt { 14489039746cSDon Hiatt struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); 14499039746cSDon Hiatt struct ib_header *hdr; 14509039746cSDon Hiatt u8 lnh; 14519039746cSDon Hiatt 14529039746cSDon Hiatt hfi1_setup_ib_header(packet); 14539039746cSDon Hiatt hdr = packet->hdr; 14549039746cSDon Hiatt 14559039746cSDon Hiatt lnh = ib_get_lnh(hdr); 14569039746cSDon Hiatt if (lnh == HFI1_LRH_BTH) { 14579039746cSDon Hiatt packet->ohdr = &hdr->u.oth; 14589039746cSDon Hiatt packet->grh = NULL; 14599039746cSDon Hiatt } else if (lnh == HFI1_LRH_GRH) { 14609039746cSDon Hiatt u32 vtf; 14619039746cSDon Hiatt 14629039746cSDon Hiatt packet->ohdr = &hdr->u.l.oth; 14639039746cSDon Hiatt packet->grh = &hdr->u.l.grh; 14649039746cSDon Hiatt if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) 14659039746cSDon Hiatt goto drop; 14669039746cSDon Hiatt vtf = be32_to_cpu(packet->grh->version_tclass_flow); 14679039746cSDon Hiatt if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 14689039746cSDon Hiatt goto drop; 14699039746cSDon Hiatt } else { 14709039746cSDon Hiatt goto drop; 14719039746cSDon Hiatt } 14729039746cSDon Hiatt 14739039746cSDon Hiatt /* Query commonly used fields from packet header */ 147472c07e2bSDon Hiatt packet->payload = packet->ebuf; 14759039746cSDon Hiatt packet->opcode = ib_bth_get_opcode(packet->ohdr); 14769039746cSDon Hiatt packet->slid = ib_get_slid(hdr); 14779039746cSDon Hiatt packet->dlid = ib_get_dlid(hdr); 147872c07e2bSDon Hiatt if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) && 147972c07e2bSDon Hiatt (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE)))) 148072c07e2bSDon Hiatt packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - 148172c07e2bSDon Hiatt be16_to_cpu(IB_MULTICAST_LID_BASE); 14829039746cSDon Hiatt packet->sl = ib_get_sl(hdr); 14839039746cSDon Hiatt packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf); 14849039746cSDon Hiatt packet->pad = ib_bth_get_pad(packet->ohdr); 14859039746cSDon Hiatt packet->extra_byte = 0; 14866d6b8848SSebastian Sanchez packet->pkey = ib_bth_get_pkey(packet->ohdr); 14876d6b8848SSebastian Sanchez packet->migrated = ib_bth_is_migration(packet->ohdr); 14889039746cSDon Hiatt 14899039746cSDon Hiatt return 0; 14909039746cSDon Hiatt drop: 14919039746cSDon Hiatt ibp->rvp.n_pkt_drops++; 14929039746cSDon Hiatt return -EINVAL; 14939039746cSDon Hiatt } 14949039746cSDon Hiatt 149572c07e2bSDon Hiatt static int hfi1_setup_bypass_packet(struct hfi1_packet *packet) 149672c07e2bSDon Hiatt { 149772c07e2bSDon Hiatt /* 149872c07e2bSDon Hiatt * Bypass packets have a different header/payload split 149972c07e2bSDon Hiatt * compared to an IB packet. 150072c07e2bSDon Hiatt * Current split is set such that 16 bytes of the actual 150172c07e2bSDon Hiatt * header is in the header buffer and the remining is in 150272c07e2bSDon Hiatt * the eager buffer. We chose 16 since hfi1 driver only 150372c07e2bSDon Hiatt * supports 16B bypass packets and we will be able to 150472c07e2bSDon Hiatt * receive the entire LRH with such a split. 150572c07e2bSDon Hiatt */ 150672c07e2bSDon Hiatt 150772c07e2bSDon Hiatt struct hfi1_ctxtdata *rcd = packet->rcd; 150872c07e2bSDon Hiatt struct hfi1_pportdata *ppd = rcd->ppd; 150972c07e2bSDon Hiatt struct hfi1_ibport *ibp = &ppd->ibport_data; 151072c07e2bSDon Hiatt u8 l4; 151172c07e2bSDon Hiatt 151272c07e2bSDon Hiatt packet->hdr = (struct hfi1_16b_header *) 151340442b30SMike Marciniszyn hfi1_get_16B_header(packet->rcd, 151472c07e2bSDon Hiatt packet->rhf_addr); 151572c07e2bSDon Hiatt l4 = hfi1_16B_get_l4(packet->hdr); 151672c07e2bSDon Hiatt if (l4 == OPA_16B_L4_IB_LOCAL) { 151772c07e2bSDon Hiatt packet->ohdr = packet->ebuf; 151872c07e2bSDon Hiatt packet->grh = NULL; 151981cd3891SDon Hiatt packet->opcode = ib_bth_get_opcode(packet->ohdr); 152081cd3891SDon Hiatt packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); 152181cd3891SDon Hiatt /* hdr_len_by_opcode already has an IB LRH factored in */ 152281cd3891SDon Hiatt packet->hlen = hdr_len_by_opcode[packet->opcode] + 152381cd3891SDon Hiatt (LRH_16B_BYTES - LRH_9B_BYTES); 152481cd3891SDon Hiatt packet->migrated = opa_bth_is_migration(packet->ohdr); 152572c07e2bSDon Hiatt } else if (l4 == OPA_16B_L4_IB_GLOBAL) { 152672c07e2bSDon Hiatt u32 vtf; 152781cd3891SDon Hiatt u8 grh_len = sizeof(struct ib_grh); 152872c07e2bSDon Hiatt 152972c07e2bSDon Hiatt packet->ohdr = packet->ebuf + grh_len; 153072c07e2bSDon Hiatt packet->grh = packet->ebuf; 153181cd3891SDon Hiatt packet->opcode = ib_bth_get_opcode(packet->ohdr); 153281cd3891SDon Hiatt packet->pad = hfi1_16B_bth_get_pad(packet->ohdr); 153381cd3891SDon Hiatt /* hdr_len_by_opcode already has an IB LRH factored in */ 153481cd3891SDon Hiatt packet->hlen = hdr_len_by_opcode[packet->opcode] + 153581cd3891SDon Hiatt (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len; 153681cd3891SDon Hiatt packet->migrated = opa_bth_is_migration(packet->ohdr); 153781cd3891SDon Hiatt 153872c07e2bSDon Hiatt if (packet->grh->next_hdr != IB_GRH_NEXT_HDR) 153972c07e2bSDon Hiatt goto drop; 154072c07e2bSDon Hiatt vtf = be32_to_cpu(packet->grh->version_tclass_flow); 154172c07e2bSDon Hiatt if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION) 154272c07e2bSDon Hiatt goto drop; 154381cd3891SDon Hiatt } else if (l4 == OPA_16B_L4_FM) { 154481cd3891SDon Hiatt packet->mgmt = packet->ebuf; 154581cd3891SDon Hiatt packet->ohdr = NULL; 154681cd3891SDon Hiatt packet->grh = NULL; 154781cd3891SDon Hiatt packet->opcode = IB_OPCODE_UD_SEND_ONLY; 154881cd3891SDon Hiatt packet->pad = OPA_16B_L4_FM_PAD; 154981cd3891SDon Hiatt packet->hlen = OPA_16B_L4_FM_HLEN; 155081cd3891SDon Hiatt packet->migrated = false; 155172c07e2bSDon Hiatt } else { 155272c07e2bSDon Hiatt goto drop; 155372c07e2bSDon Hiatt } 155472c07e2bSDon Hiatt 155572c07e2bSDon Hiatt /* Query commonly used fields from packet header */ 155678d3633bSMike Marciniszyn packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES; 155772c07e2bSDon Hiatt packet->slid = hfi1_16B_get_slid(packet->hdr); 155872c07e2bSDon Hiatt packet->dlid = hfi1_16B_get_dlid(packet->hdr); 155972c07e2bSDon Hiatt if (unlikely(hfi1_is_16B_mcast(packet->dlid))) 156072c07e2bSDon Hiatt packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) - 156172c07e2bSDon Hiatt opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR), 156272c07e2bSDon Hiatt 16B); 156372c07e2bSDon Hiatt packet->sc = hfi1_16B_get_sc(packet->hdr); 156472c07e2bSDon Hiatt packet->sl = ibp->sc_to_sl[packet->sc]; 156572c07e2bSDon Hiatt packet->extra_byte = SIZE_OF_LT; 15666d6b8848SSebastian Sanchez packet->pkey = hfi1_16B_get_pkey(packet->hdr); 156772c07e2bSDon Hiatt 15685786adf3SDon Hiatt if (hfi1_bypass_ingress_pkt_check(packet)) 15695786adf3SDon Hiatt goto drop; 15705786adf3SDon Hiatt 157172c07e2bSDon Hiatt return 0; 157272c07e2bSDon Hiatt drop: 157372c07e2bSDon Hiatt hfi1_cdbg(PKT, "%s: packet dropped\n", __func__); 157472c07e2bSDon Hiatt ibp->rvp.n_pkt_drops++; 157572c07e2bSDon Hiatt return -EINVAL; 157672c07e2bSDon Hiatt } 157772c07e2bSDon Hiatt 157822d136d7SKaike Wan static void show_eflags_errs(struct hfi1_packet *packet) 1579f48ad614SDennis Dalessandro { 1580f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 1581f48ad614SDennis Dalessandro u32 rte = rhf_rcv_type_err(packet->rhf); 1582f48ad614SDennis Dalessandro 1583f48ad614SDennis Dalessandro dd_dev_err(rcd->dd, 1584f48ad614SDennis Dalessandro "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n", 1585f48ad614SDennis Dalessandro rcd->ctxt, packet->rhf, 1586f48ad614SDennis Dalessandro packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "", 1587f48ad614SDennis Dalessandro packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "", 1588f48ad614SDennis Dalessandro packet->rhf & RHF_DC_ERR ? "dc " : "", 1589f48ad614SDennis Dalessandro packet->rhf & RHF_TID_ERR ? "tid " : "", 1590f48ad614SDennis Dalessandro packet->rhf & RHF_LEN_ERR ? "len " : "", 1591f48ad614SDennis Dalessandro packet->rhf & RHF_ECC_ERR ? "ecc " : "", 1592f48ad614SDennis Dalessandro packet->rhf & RHF_VCRC_ERR ? "vcrc " : "", 1593f48ad614SDennis Dalessandro packet->rhf & RHF_ICRC_ERR ? "icrc " : "", 1594f48ad614SDennis Dalessandro rte); 1595f48ad614SDennis Dalessandro } 1596f48ad614SDennis Dalessandro 159722d136d7SKaike Wan void handle_eflags(struct hfi1_packet *packet) 159822d136d7SKaike Wan { 159922d136d7SKaike Wan struct hfi1_ctxtdata *rcd = packet->rcd; 160022d136d7SKaike Wan 160122d136d7SKaike Wan rcv_hdrerr(rcd, rcd->ppd, packet); 160222d136d7SKaike Wan if (rhf_err_flags(packet->rhf)) 160322d136d7SKaike Wan show_eflags_errs(packet); 160422d136d7SKaike Wan } 160522d136d7SKaike Wan 1606f48ad614SDennis Dalessandro /* 1607f48ad614SDennis Dalessandro * The following functions are called by the interrupt handler. They are type 1608f48ad614SDennis Dalessandro * specific handlers for each packet type. 1609f48ad614SDennis Dalessandro */ 1610b0ba3c18SMike Marciniszyn static int process_receive_ib(struct hfi1_packet *packet) 1611f48ad614SDennis Dalessandro { 1612a74d5307SMitko Haralanov if (hfi1_setup_9B_packet(packet)) 16130181ce31SDon Hiatt return RHF_RCV_CONTINUE; 16140181ce31SDon Hiatt 1615a74d5307SMitko Haralanov if (unlikely(hfi1_dbg_should_fault_rx(packet))) 16169039746cSDon Hiatt return RHF_RCV_CONTINUE; 16179039746cSDon Hiatt 16186197a815SDon Hiatt trace_hfi1_rcvhdr(packet); 1619243d9f43SDon Hiatt 1620f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) { 1621f48ad614SDennis Dalessandro handle_eflags(packet); 1622f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1623f48ad614SDennis Dalessandro } 1624f48ad614SDennis Dalessandro 1625f48ad614SDennis Dalessandro hfi1_ib_rcv(packet); 1626f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1627f48ad614SDennis Dalessandro } 1628f48ad614SDennis Dalessandro 1629d4829ea6SVishwanathapura, Niranjana static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet) 1630d4829ea6SVishwanathapura, Niranjana { 1631d4829ea6SVishwanathapura, Niranjana /* Packet received in VNIC context via RSM */ 1632d4829ea6SVishwanathapura, Niranjana if (packet->rcd->is_vnic) 1633d4829ea6SVishwanathapura, Niranjana return true; 1634d4829ea6SVishwanathapura, Niranjana 163572c07e2bSDon Hiatt if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) && 163672c07e2bSDon Hiatt (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR)) 1637d4829ea6SVishwanathapura, Niranjana return true; 1638d4829ea6SVishwanathapura, Niranjana 1639d4829ea6SVishwanathapura, Niranjana return false; 1640d4829ea6SVishwanathapura, Niranjana } 1641d4829ea6SVishwanathapura, Niranjana 1642b0ba3c18SMike Marciniszyn static int process_receive_bypass(struct hfi1_packet *packet) 1643f48ad614SDennis Dalessandro { 1644505efe3eSJakub Pawlak struct hfi1_devdata *dd = packet->rcd->dd; 1645505efe3eSJakub Pawlak 164672c07e2bSDon Hiatt if (hfi1_is_vnic_packet(packet)) { 1647d4829ea6SVishwanathapura, Niranjana hfi1_vnic_bypass_rcv(packet); 1648d4829ea6SVishwanathapura, Niranjana return RHF_RCV_CONTINUE; 1649d4829ea6SVishwanathapura, Niranjana } 1650f48ad614SDennis Dalessandro 165172c07e2bSDon Hiatt if (hfi1_setup_bypass_packet(packet)) 165272c07e2bSDon Hiatt return RHF_RCV_CONTINUE; 165372c07e2bSDon Hiatt 16546197a815SDon Hiatt trace_hfi1_rcvhdr(packet); 16556197a815SDon Hiatt 165672c07e2bSDon Hiatt if (unlikely(rhf_err_flags(packet->rhf))) { 165772c07e2bSDon Hiatt handle_eflags(packet); 165872c07e2bSDon Hiatt return RHF_RCV_CONTINUE; 165972c07e2bSDon Hiatt } 166072c07e2bSDon Hiatt 166172c07e2bSDon Hiatt if (hfi1_16B_get_l2(packet->hdr) == 0x2) { 166272c07e2bSDon Hiatt hfi1_16B_rcv(packet); 166372c07e2bSDon Hiatt } else { 166472c07e2bSDon Hiatt dd_dev_err(dd, 166572c07e2bSDon Hiatt "Bypass packets other than 16B are not supported in normal operation. Dropping\n"); 1666505efe3eSJakub Pawlak incr_cntr64(&dd->sw_rcv_bypass_packet_errors); 166772c07e2bSDon Hiatt if (!(dd->err_info_rcvport.status_and_code & 166872c07e2bSDon Hiatt OPA_EI_STATUS_SMASK)) { 1669505efe3eSJakub Pawlak u64 *flits = packet->ebuf; 1670505efe3eSJakub Pawlak 1671505efe3eSJakub Pawlak if (flits && !(packet->rhf & RHF_LEN_ERR)) { 1672505efe3eSJakub Pawlak dd->err_info_rcvport.packet_flit1 = flits[0]; 1673505efe3eSJakub Pawlak dd->err_info_rcvport.packet_flit2 = 167472c07e2bSDon Hiatt packet->tlen > sizeof(flits[0]) ? 167572c07e2bSDon Hiatt flits[1] : 0; 1676505efe3eSJakub Pawlak } 1677505efe3eSJakub Pawlak dd->err_info_rcvport.status_and_code |= 1678505efe3eSJakub Pawlak (OPA_EI_STATUS_SMASK | BAD_L2_ERR); 1679505efe3eSJakub Pawlak } 168072c07e2bSDon Hiatt } 1681f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1682f48ad614SDennis Dalessandro } 1683f48ad614SDennis Dalessandro 1684b0ba3c18SMike Marciniszyn static int process_receive_error(struct hfi1_packet *packet) 1685f48ad614SDennis Dalessandro { 1686243d9f43SDon Hiatt /* KHdrHCRCErr -- KDETH packet with a bad HCRC */ 1687243d9f43SDon Hiatt if (unlikely( 1688243d9f43SDon Hiatt hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) && 1689a74d5307SMitko Haralanov (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR || 1690a74d5307SMitko Haralanov packet->rhf & RHF_DC_ERR))) 1691243d9f43SDon Hiatt return RHF_RCV_CONTINUE; 1692243d9f43SDon Hiatt 16939039746cSDon Hiatt hfi1_setup_ib_header(packet); 1694f48ad614SDennis Dalessandro handle_eflags(packet); 1695f48ad614SDennis Dalessandro 1696f48ad614SDennis Dalessandro if (unlikely(rhf_err_flags(packet->rhf))) 1697f48ad614SDennis Dalessandro dd_dev_err(packet->rcd->dd, 1698f48ad614SDennis Dalessandro "Unhandled error packet received. Dropping.\n"); 1699f48ad614SDennis Dalessandro 1700f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1701f48ad614SDennis Dalessandro } 1702f48ad614SDennis Dalessandro 1703b0ba3c18SMike Marciniszyn static int kdeth_process_expected(struct hfi1_packet *packet) 1704f48ad614SDennis Dalessandro { 1705a74d5307SMitko Haralanov hfi1_setup_9B_packet(packet); 1706a74d5307SMitko Haralanov if (unlikely(hfi1_dbg_should_fault_rx(packet))) 17070181ce31SDon Hiatt return RHF_RCV_CONTINUE; 17089039746cSDon Hiatt 170922d136d7SKaike Wan if (unlikely(rhf_err_flags(packet->rhf))) { 171022d136d7SKaike Wan struct hfi1_ctxtdata *rcd = packet->rcd; 1711f48ad614SDennis Dalessandro 171222d136d7SKaike Wan if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) 171322d136d7SKaike Wan return RHF_RCV_CONTINUE; 171422d136d7SKaike Wan } 171522d136d7SKaike Wan 171622d136d7SKaike Wan hfi1_kdeth_expected_rcv(packet); 1717f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1718f48ad614SDennis Dalessandro } 1719f48ad614SDennis Dalessandro 1720b0ba3c18SMike Marciniszyn static int kdeth_process_eager(struct hfi1_packet *packet) 1721f48ad614SDennis Dalessandro { 1722a74d5307SMitko Haralanov hfi1_setup_9B_packet(packet); 1723a74d5307SMitko Haralanov if (unlikely(hfi1_dbg_should_fault_rx(packet))) 1724a74d5307SMitko Haralanov return RHF_RCV_CONTINUE; 1725f48ad614SDennis Dalessandro 172622d136d7SKaike Wan trace_hfi1_rcvhdr(packet); 172722d136d7SKaike Wan if (unlikely(rhf_err_flags(packet->rhf))) { 172822d136d7SKaike Wan struct hfi1_ctxtdata *rcd = packet->rcd; 172922d136d7SKaike Wan 173022d136d7SKaike Wan show_eflags_errs(packet); 173122d136d7SKaike Wan if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet)) 173222d136d7SKaike Wan return RHF_RCV_CONTINUE; 173322d136d7SKaike Wan } 173422d136d7SKaike Wan 173522d136d7SKaike Wan hfi1_kdeth_eager_rcv(packet); 1736f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1737f48ad614SDennis Dalessandro } 1738f48ad614SDennis Dalessandro 1739b0ba3c18SMike Marciniszyn static int process_receive_invalid(struct hfi1_packet *packet) 1740f48ad614SDennis Dalessandro { 1741f48ad614SDennis Dalessandro dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n", 1742f48ad614SDennis Dalessandro rhf_rcv_type(packet->rhf)); 1743f48ad614SDennis Dalessandro return RHF_RCV_CONTINUE; 1744f48ad614SDennis Dalessandro } 1745bf808b50SKaike Wan 1746bf808b50SKaike Wan void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd) 1747bf808b50SKaike Wan { 1748bf808b50SKaike Wan struct hfi1_packet packet; 1749bf808b50SKaike Wan struct ps_mdata mdata; 1750bf808b50SKaike Wan 1751bf808b50SKaike Wan seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n", 1752bf808b50SKaike Wan rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize, 1753bf808b50SKaike Wan HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ? 1754bf808b50SKaike Wan "dma_rtail" : "nodma_rtail", 1755bf808b50SKaike Wan read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) & 1756bf808b50SKaike Wan RCV_HDR_HEAD_HEAD_MASK, 1757bf808b50SKaike Wan read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL)); 1758bf808b50SKaike Wan 1759bf808b50SKaike Wan init_packet(rcd, &packet); 1760bf808b50SKaike Wan init_ps_mdata(&mdata, &packet); 1761bf808b50SKaike Wan 1762bf808b50SKaike Wan while (1) { 1763bf808b50SKaike Wan __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head + 176440442b30SMike Marciniszyn rcd->rhf_offset; 1765bf808b50SKaike Wan struct ib_header *hdr; 1766bf808b50SKaike Wan u64 rhf = rhf_to_cpu(rhf_addr); 1767bf808b50SKaike Wan u32 etype = rhf_rcv_type(rhf), qpn; 1768bf808b50SKaike Wan u8 opcode; 1769bf808b50SKaike Wan u32 psn; 1770bf808b50SKaike Wan u8 lnh; 1771bf808b50SKaike Wan 1772bf808b50SKaike Wan if (ps_done(&mdata, rhf, rcd)) 1773bf808b50SKaike Wan break; 1774bf808b50SKaike Wan 1775bf808b50SKaike Wan if (ps_skip(&mdata, rhf, rcd)) 1776bf808b50SKaike Wan goto next; 1777bf808b50SKaike Wan 1778bf808b50SKaike Wan if (etype > RHF_RCV_TYPE_IB) 1779bf808b50SKaike Wan goto next; 1780bf808b50SKaike Wan 178140442b30SMike Marciniszyn packet.hdr = hfi1_get_msgheader(rcd, rhf_addr); 1782bf808b50SKaike Wan hdr = packet.hdr; 1783bf808b50SKaike Wan 1784bf808b50SKaike Wan lnh = be16_to_cpu(hdr->lrh[0]) & 3; 1785bf808b50SKaike Wan 1786bf808b50SKaike Wan if (lnh == HFI1_LRH_BTH) 1787bf808b50SKaike Wan packet.ohdr = &hdr->u.oth; 1788bf808b50SKaike Wan else if (lnh == HFI1_LRH_GRH) 1789bf808b50SKaike Wan packet.ohdr = &hdr->u.l.oth; 1790bf808b50SKaike Wan else 1791bf808b50SKaike Wan goto next; /* just in case */ 1792bf808b50SKaike Wan 1793bf808b50SKaike Wan opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24); 1794bf808b50SKaike Wan qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK; 1795bf808b50SKaike Wan psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2])); 1796bf808b50SKaike Wan 1797bf808b50SKaike Wan seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n", 1798bf808b50SKaike Wan mdata.ps_head, opcode, qpn, psn); 1799bf808b50SKaike Wan next: 1800bf808b50SKaike Wan update_ps_mdata(&mdata, rcd); 1801bf808b50SKaike Wan } 1802bf808b50SKaike Wan } 1803b0ba3c18SMike Marciniszyn 1804b0ba3c18SMike Marciniszyn const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = { 1805b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_EXPECTED] = kdeth_process_expected, 1806b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_EAGER] = kdeth_process_eager, 1807b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_IB] = process_receive_ib, 1808b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_ERROR] = process_receive_error, 1809b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_BYPASS] = process_receive_bypass, 1810b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_INVALID5] = process_receive_invalid, 1811b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_INVALID6] = process_receive_invalid, 1812b0ba3c18SMike Marciniszyn [RHF_RCV_TYPE_INVALID7] = process_receive_invalid, 1813b0ba3c18SMike Marciniszyn }; 1814