1f48ad614SDennis Dalessandro /* 22e2ba09eSMike Marciniszyn * Copyright(c) 2015 - 2018 Intel Corporation. 3f48ad614SDennis Dalessandro * 4f48ad614SDennis Dalessandro * This file is provided under a dual BSD/GPLv2 license. When using or 5f48ad614SDennis Dalessandro * redistributing this file, you may do so under either license. 6f48ad614SDennis Dalessandro * 7f48ad614SDennis Dalessandro * GPL LICENSE SUMMARY 8f48ad614SDennis Dalessandro * 9f48ad614SDennis Dalessandro * This program is free software; you can redistribute it and/or modify 10f48ad614SDennis Dalessandro * it under the terms of version 2 of the GNU General Public License as 11f48ad614SDennis Dalessandro * published by the Free Software Foundation. 12f48ad614SDennis Dalessandro * 13f48ad614SDennis Dalessandro * This program is distributed in the hope that it will be useful, but 14f48ad614SDennis Dalessandro * WITHOUT ANY WARRANTY; without even the implied warranty of 15f48ad614SDennis Dalessandro * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16f48ad614SDennis Dalessandro * General Public License for more details. 17f48ad614SDennis Dalessandro * 18f48ad614SDennis Dalessandro * BSD LICENSE 19f48ad614SDennis Dalessandro * 20f48ad614SDennis Dalessandro * Redistribution and use in source and binary forms, with or without 21f48ad614SDennis Dalessandro * modification, are permitted provided that the following conditions 22f48ad614SDennis Dalessandro * are met: 23f48ad614SDennis Dalessandro * 24f48ad614SDennis Dalessandro * - Redistributions of source code must retain the above copyright 25f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer. 26f48ad614SDennis Dalessandro * - Redistributions in binary form must reproduce the above copyright 27f48ad614SDennis Dalessandro * notice, this list of conditions and the following disclaimer in 28f48ad614SDennis Dalessandro * the documentation and/or other materials provided with the 29f48ad614SDennis Dalessandro * distribution. 30f48ad614SDennis Dalessandro * - Neither the name of Intel Corporation nor the names of its 31f48ad614SDennis Dalessandro * contributors may be used to endorse or promote products derived 32f48ad614SDennis Dalessandro * from this software without specific prior written permission. 33f48ad614SDennis Dalessandro * 34f48ad614SDennis Dalessandro * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 35f48ad614SDennis Dalessandro * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 36f48ad614SDennis Dalessandro * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 37f48ad614SDennis Dalessandro * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 38f48ad614SDennis Dalessandro * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 39f48ad614SDennis Dalessandro * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 40f48ad614SDennis Dalessandro * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 41f48ad614SDennis Dalessandro * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 42f48ad614SDennis Dalessandro * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 43f48ad614SDennis Dalessandro * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 44f48ad614SDennis Dalessandro * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 45f48ad614SDennis Dalessandro * 46f48ad614SDennis Dalessandro */ 47f48ad614SDennis Dalessandro 48f48ad614SDennis Dalessandro #include <linux/io.h> 49f48ad614SDennis Dalessandro #include <rdma/rdma_vt.h> 50f48ad614SDennis Dalessandro #include <rdma/rdmavt_qp.h> 51f48ad614SDennis Dalessandro 52f48ad614SDennis Dalessandro #include "hfi.h" 53f48ad614SDennis Dalessandro #include "qp.h" 54385156c5SKaike Wan #include "rc.h" 55f48ad614SDennis Dalessandro #include "verbs_txreq.h" 56f48ad614SDennis Dalessandro #include "trace.h" 57f48ad614SDennis Dalessandro 58385156c5SKaike Wan struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev, 59385156c5SKaike Wan u8 *prev_ack, bool *scheduled) 60385156c5SKaike Wan __must_hold(&qp->s_lock) 61f48ad614SDennis Dalessandro { 62385156c5SKaike Wan struct rvt_ack_entry *e = NULL; 63385156c5SKaike Wan u8 i, p; 64385156c5SKaike Wan bool s = true; 65f48ad614SDennis Dalessandro 66385156c5SKaike Wan for (i = qp->r_head_ack_queue; ; i = p) { 67385156c5SKaike Wan if (i == qp->s_tail_ack_queue) 68385156c5SKaike Wan s = false; 69385156c5SKaike Wan if (i) 70385156c5SKaike Wan p = i - 1; 71385156c5SKaike Wan else 72385156c5SKaike Wan p = rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); 73385156c5SKaike Wan if (p == qp->r_head_ack_queue) { 74385156c5SKaike Wan e = NULL; 75385156c5SKaike Wan break; 76385156c5SKaike Wan } 77385156c5SKaike Wan e = &qp->s_ack_queue[p]; 78385156c5SKaike Wan if (!e->opcode) { 79385156c5SKaike Wan e = NULL; 80385156c5SKaike Wan break; 81385156c5SKaike Wan } 82385156c5SKaike Wan if (cmp_psn(psn, e->psn) >= 0) { 83385156c5SKaike Wan if (p == qp->s_tail_ack_queue && 84385156c5SKaike Wan cmp_psn(psn, e->lpsn) <= 0) 85385156c5SKaike Wan s = false; 86385156c5SKaike Wan break; 87385156c5SKaike Wan } 88385156c5SKaike Wan } 89385156c5SKaike Wan if (prev) 90385156c5SKaike Wan *prev = p; 91385156c5SKaike Wan if (prev_ack) 92385156c5SKaike Wan *prev_ack = i; 93385156c5SKaike Wan if (scheduled) 94385156c5SKaike Wan *scheduled = s; 95385156c5SKaike Wan return e; 96f48ad614SDennis Dalessandro } 97f48ad614SDennis Dalessandro 98f48ad614SDennis Dalessandro /** 99f48ad614SDennis Dalessandro * make_rc_ack - construct a response packet (ACK, NAK, or RDMA read) 100f48ad614SDennis Dalessandro * @dev: the device for this QP 101f48ad614SDennis Dalessandro * @qp: a pointer to the QP 102f48ad614SDennis Dalessandro * @ohdr: a pointer to the IB header being constructed 103f48ad614SDennis Dalessandro * @ps: the xmit packet state 104f48ad614SDennis Dalessandro * 105f48ad614SDennis Dalessandro * Return 1 if constructed; otherwise, return 0. 106f48ad614SDennis Dalessandro * Note that we are in the responder's side of the QP context. 107f48ad614SDennis Dalessandro * Note the QP s_lock must be held. 108f48ad614SDennis Dalessandro */ 109f48ad614SDennis Dalessandro static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, 110261a4351SMike Marciniszyn struct ib_other_headers *ohdr, 111f48ad614SDennis Dalessandro struct hfi1_pkt_state *ps) 112f48ad614SDennis Dalessandro { 113f48ad614SDennis Dalessandro struct rvt_ack_entry *e; 1143c6cb20aSKaike Wan u32 hwords, hdrlen; 11524b11923SKaike Wan u32 len = 0; 11624b11923SKaike Wan u32 bth0 = 0, bth2 = 0; 11744e43d91SMitko Haralanov u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); 118f48ad614SDennis Dalessandro int middle = 0; 119f48ad614SDennis Dalessandro u32 pmtu = qp->pmtu; 1203c6cb20aSKaike Wan struct hfi1_qp_priv *qpriv = qp->priv; 12124b11923SKaike Wan bool last_pkt; 12224b11923SKaike Wan u32 delta; 1234f9264d1SKaike Wan u8 next = qp->s_tail_ack_queue; 1243c6cb20aSKaike Wan struct tid_rdma_request *req; 125f48ad614SDennis Dalessandro 1263ce5daa2SKaike Wan trace_hfi1_rsp_make_rc_ack(qp, 0); 12768e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 128f48ad614SDennis Dalessandro /* Don't send an ACK if we aren't supposed to. */ 129f48ad614SDennis Dalessandro if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) 130f48ad614SDennis Dalessandro goto bail; 131f48ad614SDennis Dalessandro 1323c6cb20aSKaike Wan if (qpriv->hdr_type == HFI1_PKT_TYPE_9B) 133f48ad614SDennis Dalessandro /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 134f48ad614SDennis Dalessandro hwords = 5; 1355b6cabb0SDon Hiatt else 1365b6cabb0SDon Hiatt /* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */ 1375b6cabb0SDon Hiatt hwords = 7; 138f48ad614SDennis Dalessandro 139f48ad614SDennis Dalessandro switch (qp->s_ack_state) { 140f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_LAST): 141f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_ONLY): 142f48ad614SDennis Dalessandro e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 143f6f3f532SKaike Wan release_rdma_sge_mr(e); 144f48ad614SDennis Dalessandro /* FALLTHROUGH */ 145f48ad614SDennis Dalessandro case OP(ATOMIC_ACKNOWLEDGE): 146f48ad614SDennis Dalessandro /* 147f48ad614SDennis Dalessandro * We can increment the tail pointer now that the last 148f48ad614SDennis Dalessandro * response has been sent instead of only being 149f48ad614SDennis Dalessandro * constructed. 150f48ad614SDennis Dalessandro */ 1514f9264d1SKaike Wan if (++next > rvt_size_atomic(&dev->rdi)) 1524f9264d1SKaike Wan next = 0; 1534f9264d1SKaike Wan /* 1544f9264d1SKaike Wan * Only advance the s_acked_ack_queue pointer if there 1554f9264d1SKaike Wan * have been no TID RDMA requests. 1564f9264d1SKaike Wan */ 1574f9264d1SKaike Wan e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 1584f9264d1SKaike Wan if (e->opcode != TID_OP(WRITE_REQ) && 1594f9264d1SKaike Wan qp->s_acked_ack_queue == qp->s_tail_ack_queue) 1604f9264d1SKaike Wan qp->s_acked_ack_queue = next; 1614f9264d1SKaike Wan qp->s_tail_ack_queue = next; 162a05c9bdcSKaike Wan trace_hfi1_rsp_make_rc_ack(qp, e->psn); 163f48ad614SDennis Dalessandro /* FALLTHROUGH */ 164f48ad614SDennis Dalessandro case OP(SEND_ONLY): 165f48ad614SDennis Dalessandro case OP(ACKNOWLEDGE): 166f48ad614SDennis Dalessandro /* Check for no next entry in the queue. */ 167f48ad614SDennis Dalessandro if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { 168f48ad614SDennis Dalessandro if (qp->s_flags & RVT_S_ACK_PENDING) 169f48ad614SDennis Dalessandro goto normal; 170f48ad614SDennis Dalessandro goto bail; 171f48ad614SDennis Dalessandro } 172f48ad614SDennis Dalessandro 173f48ad614SDennis Dalessandro e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 174c6c23117SKaike Wan /* Check for tid write fence */ 175c6c23117SKaike Wan if ((qpriv->s_flags & HFI1_R_TID_WAIT_INTERLCK) || 176c6c23117SKaike Wan hfi1_tid_rdma_ack_interlock(qp, e)) { 177c6c23117SKaike Wan iowait_set_flag(&qpriv->s_iowait, IOWAIT_PENDING_IB); 178c6c23117SKaike Wan goto bail; 179c6c23117SKaike Wan } 180f48ad614SDennis Dalessandro if (e->opcode == OP(RDMA_READ_REQUEST)) { 181f48ad614SDennis Dalessandro /* 182f48ad614SDennis Dalessandro * If a RDMA read response is being resent and 183f48ad614SDennis Dalessandro * we haven't seen the duplicate request yet, 184f48ad614SDennis Dalessandro * then stop sending the remaining responses the 185f48ad614SDennis Dalessandro * responder has seen until the requester re-sends it. 186f48ad614SDennis Dalessandro */ 187f48ad614SDennis Dalessandro len = e->rdma_sge.sge_length; 188f48ad614SDennis Dalessandro if (len && !e->rdma_sge.mr) { 1894f9264d1SKaike Wan if (qp->s_acked_ack_queue == 1904f9264d1SKaike Wan qp->s_tail_ack_queue) 1914f9264d1SKaike Wan qp->s_acked_ack_queue = 1924f9264d1SKaike Wan qp->r_head_ack_queue; 193f48ad614SDennis Dalessandro qp->s_tail_ack_queue = qp->r_head_ack_queue; 194f48ad614SDennis Dalessandro goto bail; 195f48ad614SDennis Dalessandro } 196f48ad614SDennis Dalessandro /* Copy SGE state in case we need to resend */ 197f48ad614SDennis Dalessandro ps->s_txreq->mr = e->rdma_sge.mr; 198f48ad614SDennis Dalessandro if (ps->s_txreq->mr) 199f48ad614SDennis Dalessandro rvt_get_mr(ps->s_txreq->mr); 200f48ad614SDennis Dalessandro qp->s_ack_rdma_sge.sge = e->rdma_sge; 201f48ad614SDennis Dalessandro qp->s_ack_rdma_sge.num_sge = 1; 202b777f154SMitko Haralanov ps->s_txreq->ss = &qp->s_ack_rdma_sge; 203f48ad614SDennis Dalessandro if (len > pmtu) { 204f48ad614SDennis Dalessandro len = pmtu; 205f48ad614SDennis Dalessandro qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); 206f48ad614SDennis Dalessandro } else { 207f48ad614SDennis Dalessandro qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); 208f48ad614SDennis Dalessandro e->sent = 1; 209f48ad614SDennis Dalessandro } 210696513e8SBrian Welty ohdr->u.aeth = rvt_compute_aeth(qp); 211f48ad614SDennis Dalessandro hwords++; 212f48ad614SDennis Dalessandro qp->s_ack_rdma_psn = e->psn; 213f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_ack_rdma_psn++); 2143c6cb20aSKaike Wan } else if (e->opcode == TID_OP(WRITE_REQ)) { 2153c6cb20aSKaike Wan /* 2163c6cb20aSKaike Wan * If a TID RDMA WRITE RESP is being resent, we have to 2173c6cb20aSKaike Wan * wait for the actual request. All requests that are to 2183c6cb20aSKaike Wan * be resent will have their state set to 2193c6cb20aSKaike Wan * TID_REQUEST_RESEND. When the new request arrives, the 2203c6cb20aSKaike Wan * state will be changed to TID_REQUEST_RESEND_ACTIVE. 2213c6cb20aSKaike Wan */ 2223c6cb20aSKaike Wan req = ack_to_tid_req(e); 2233c6cb20aSKaike Wan if (req->state == TID_REQUEST_RESEND || 2243c6cb20aSKaike Wan req->state == TID_REQUEST_INIT_RESEND) 2253c6cb20aSKaike Wan goto bail; 2263c6cb20aSKaike Wan qp->s_ack_state = TID_OP(WRITE_RESP); 2273c6cb20aSKaike Wan qp->s_ack_rdma_psn = mask_psn(e->psn + req->cur_seg); 2283c6cb20aSKaike Wan goto write_resp; 22924b11923SKaike Wan } else if (e->opcode == TID_OP(READ_REQ)) { 23024b11923SKaike Wan /* 23124b11923SKaike Wan * If a TID RDMA read response is being resent and 23224b11923SKaike Wan * we haven't seen the duplicate request yet, 23324b11923SKaike Wan * then stop sending the remaining responses the 23424b11923SKaike Wan * responder has seen until the requester re-sends it. 23524b11923SKaike Wan */ 23624b11923SKaike Wan len = e->rdma_sge.sge_length; 23724b11923SKaike Wan if (len && !e->rdma_sge.mr) { 2384f9264d1SKaike Wan if (qp->s_acked_ack_queue == 2394f9264d1SKaike Wan qp->s_tail_ack_queue) 2404f9264d1SKaike Wan qp->s_acked_ack_queue = 2414f9264d1SKaike Wan qp->r_head_ack_queue; 24224b11923SKaike Wan qp->s_tail_ack_queue = qp->r_head_ack_queue; 24324b11923SKaike Wan goto bail; 24424b11923SKaike Wan } 24524b11923SKaike Wan /* Copy SGE state in case we need to resend */ 24624b11923SKaike Wan ps->s_txreq->mr = e->rdma_sge.mr; 24724b11923SKaike Wan if (ps->s_txreq->mr) 24824b11923SKaike Wan rvt_get_mr(ps->s_txreq->mr); 24924b11923SKaike Wan qp->s_ack_rdma_sge.sge = e->rdma_sge; 25024b11923SKaike Wan qp->s_ack_rdma_sge.num_sge = 1; 25124b11923SKaike Wan qp->s_ack_state = TID_OP(READ_RESP); 25224b11923SKaike Wan goto read_resp; 253f48ad614SDennis Dalessandro } else { 254f48ad614SDennis Dalessandro /* COMPARE_SWAP or FETCH_ADD */ 255b777f154SMitko Haralanov ps->s_txreq->ss = NULL; 256f48ad614SDennis Dalessandro len = 0; 257f48ad614SDennis Dalessandro qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 258696513e8SBrian Welty ohdr->u.at.aeth = rvt_compute_aeth(qp); 259261a4351SMike Marciniszyn ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth); 260f48ad614SDennis Dalessandro hwords += sizeof(ohdr->u.at) / sizeof(u32); 261f48ad614SDennis Dalessandro bth2 = mask_psn(e->psn); 262f48ad614SDennis Dalessandro e->sent = 1; 263f48ad614SDennis Dalessandro } 264a05c9bdcSKaike Wan trace_hfi1_tid_write_rsp_make_rc_ack(qp); 265f48ad614SDennis Dalessandro bth0 = qp->s_ack_state << 24; 266f48ad614SDennis Dalessandro break; 267f48ad614SDennis Dalessandro 268f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_FIRST): 269f48ad614SDennis Dalessandro qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); 270f48ad614SDennis Dalessandro /* FALLTHROUGH */ 271f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_MIDDLE): 272b777f154SMitko Haralanov ps->s_txreq->ss = &qp->s_ack_rdma_sge; 273f48ad614SDennis Dalessandro ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr; 274f48ad614SDennis Dalessandro if (ps->s_txreq->mr) 275f48ad614SDennis Dalessandro rvt_get_mr(ps->s_txreq->mr); 276f48ad614SDennis Dalessandro len = qp->s_ack_rdma_sge.sge.sge_length; 277f48ad614SDennis Dalessandro if (len > pmtu) { 278f48ad614SDennis Dalessandro len = pmtu; 279f48ad614SDennis Dalessandro middle = HFI1_CAP_IS_KSET(SDMA_AHG); 280f48ad614SDennis Dalessandro } else { 281696513e8SBrian Welty ohdr->u.aeth = rvt_compute_aeth(qp); 282f48ad614SDennis Dalessandro hwords++; 283f48ad614SDennis Dalessandro qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 284f48ad614SDennis Dalessandro e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 285f48ad614SDennis Dalessandro e->sent = 1; 286f48ad614SDennis Dalessandro } 287f48ad614SDennis Dalessandro bth0 = qp->s_ack_state << 24; 288f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_ack_rdma_psn++); 289f48ad614SDennis Dalessandro break; 290f48ad614SDennis Dalessandro 2913c6cb20aSKaike Wan case TID_OP(WRITE_RESP): 2923c6cb20aSKaike Wan write_resp: 2933c6cb20aSKaike Wan /* 2943c6cb20aSKaike Wan * 1. Check if RVT_S_ACK_PENDING is set. If yes, 2953c6cb20aSKaike Wan * goto normal. 2963c6cb20aSKaike Wan * 2. Attempt to allocate TID resources. 2973c6cb20aSKaike Wan * 3. Remove RVT_S_RESP_PENDING flags from s_flags 2983c6cb20aSKaike Wan * 4. If resources not available: 2993c6cb20aSKaike Wan * 4.1 Set RVT_S_WAIT_TID_SPACE 3003c6cb20aSKaike Wan * 4.2 Queue QP on RCD TID queue 3013c6cb20aSKaike Wan * 4.3 Put QP on iowait list. 3023c6cb20aSKaike Wan * 4.4 Build IB RNR NAK with appropriate timeout value 3033c6cb20aSKaike Wan * 4.5 Return indication progress made. 3043c6cb20aSKaike Wan * 5. If resources are available: 3053c6cb20aSKaike Wan * 5.1 Program HW flow CSRs 3063c6cb20aSKaike Wan * 5.2 Build TID RDMA WRITE RESP packet 3073c6cb20aSKaike Wan * 5.3 If more resources needed, do 2.1 - 2.3. 3083c6cb20aSKaike Wan * 5.4 Wake up next QP on RCD TID queue. 3093c6cb20aSKaike Wan * 5.5 Return indication progress made. 3103c6cb20aSKaike Wan */ 3113c6cb20aSKaike Wan 3123c6cb20aSKaike Wan e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 3133c6cb20aSKaike Wan req = ack_to_tid_req(e); 3143c6cb20aSKaike Wan 3153c6cb20aSKaike Wan /* 3163c6cb20aSKaike Wan * Send scheduled RNR NAK's. RNR NAK's need to be sent at 3173c6cb20aSKaike Wan * segment boundaries, not at request boundaries. Don't change 3183c6cb20aSKaike Wan * s_ack_state because we are still in the middle of a request 3193c6cb20aSKaike Wan */ 3203c6cb20aSKaike Wan if (qpriv->rnr_nak_state == TID_RNR_NAK_SEND && 3213c6cb20aSKaike Wan qp->s_tail_ack_queue == qpriv->r_tid_alloc && 3223c6cb20aSKaike Wan req->cur_seg == req->alloc_seg) { 3233c6cb20aSKaike Wan qpriv->rnr_nak_state = TID_RNR_NAK_SENT; 3243c6cb20aSKaike Wan goto normal_no_state; 3253c6cb20aSKaike Wan } 3263c6cb20aSKaike Wan 3273c6cb20aSKaike Wan bth2 = mask_psn(qp->s_ack_rdma_psn); 3283c6cb20aSKaike Wan hdrlen = hfi1_build_tid_rdma_write_resp(qp, e, ohdr, &bth1, 3293c6cb20aSKaike Wan bth2, &len, 3303c6cb20aSKaike Wan &ps->s_txreq->ss); 3313c6cb20aSKaike Wan if (!hdrlen) 3323c6cb20aSKaike Wan return 0; 3333c6cb20aSKaike Wan 3343c6cb20aSKaike Wan hwords += hdrlen; 3353c6cb20aSKaike Wan bth0 = qp->s_ack_state << 24; 3363c6cb20aSKaike Wan qp->s_ack_rdma_psn++; 337a05c9bdcSKaike Wan trace_hfi1_tid_req_make_rc_ack_write(qp, 0, e->opcode, e->psn, 338a05c9bdcSKaike Wan e->lpsn, req); 3393c6cb20aSKaike Wan if (req->cur_seg != req->total_segs) 3403c6cb20aSKaike Wan break; 3413c6cb20aSKaike Wan 3423c6cb20aSKaike Wan e->sent = 1; 343f6f3f532SKaike Wan /* Do not free e->rdma_sge until all data are received */ 344f6f3f532SKaike Wan qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); 3453c6cb20aSKaike Wan break; 3463c6cb20aSKaike Wan 34724b11923SKaike Wan case TID_OP(READ_RESP): 34824b11923SKaike Wan read_resp: 34924b11923SKaike Wan e = &qp->s_ack_queue[qp->s_tail_ack_queue]; 35024b11923SKaike Wan ps->s_txreq->ss = &qp->s_ack_rdma_sge; 35124b11923SKaike Wan delta = hfi1_build_tid_rdma_read_resp(qp, e, ohdr, &bth0, 35224b11923SKaike Wan &bth1, &bth2, &len, 35324b11923SKaike Wan &last_pkt); 35424b11923SKaike Wan if (delta == 0) 35524b11923SKaike Wan goto error_qp; 35624b11923SKaike Wan hwords += delta; 35724b11923SKaike Wan if (last_pkt) { 35824b11923SKaike Wan e->sent = 1; 35924b11923SKaike Wan /* 36024b11923SKaike Wan * Increment qp->s_tail_ack_queue through s_ack_state 36124b11923SKaike Wan * transition. 36224b11923SKaike Wan */ 36324b11923SKaike Wan qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); 36424b11923SKaike Wan } 36524b11923SKaike Wan break; 36624b11923SKaike Wan case TID_OP(READ_REQ): 36724b11923SKaike Wan goto bail; 36824b11923SKaike Wan 369f48ad614SDennis Dalessandro default: 370f48ad614SDennis Dalessandro normal: 371f48ad614SDennis Dalessandro /* 372f48ad614SDennis Dalessandro * Send a regular ACK. 373f48ad614SDennis Dalessandro * Set the s_ack_state so we wait until after sending 374f48ad614SDennis Dalessandro * the ACK before setting s_ack_state to ACKNOWLEDGE 375f48ad614SDennis Dalessandro * (see above). 376f48ad614SDennis Dalessandro */ 377f48ad614SDennis Dalessandro qp->s_ack_state = OP(SEND_ONLY); 3783c6cb20aSKaike Wan normal_no_state: 379f48ad614SDennis Dalessandro if (qp->s_nak_state) 380f48ad614SDennis Dalessandro ohdr->u.aeth = 381832666c1SDon Hiatt cpu_to_be32((qp->r_msn & IB_MSN_MASK) | 382f48ad614SDennis Dalessandro (qp->s_nak_state << 383832666c1SDon Hiatt IB_AETH_CREDIT_SHIFT)); 384f48ad614SDennis Dalessandro else 385696513e8SBrian Welty ohdr->u.aeth = rvt_compute_aeth(qp); 386f48ad614SDennis Dalessandro hwords++; 387f48ad614SDennis Dalessandro len = 0; 388f48ad614SDennis Dalessandro bth0 = OP(ACKNOWLEDGE) << 24; 389f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_ack_psn); 3903c6cb20aSKaike Wan qp->s_flags &= ~RVT_S_ACK_PENDING; 39134025fb0SKaike Wan ps->s_txreq->txreq.flags |= SDMA_TXREQ_F_VIP; 3923c6cb20aSKaike Wan ps->s_txreq->ss = NULL; 393f48ad614SDennis Dalessandro } 394f48ad614SDennis Dalessandro qp->s_rdma_ack_cnt++; 3953c6cb20aSKaike Wan ps->s_txreq->sde = qpriv->s_sde; 396e922ae06SDon Hiatt ps->s_txreq->s_cur_size = len; 3979636258fSMitko Haralanov ps->s_txreq->hdr_dwords = hwords; 39844e43d91SMitko Haralanov hfi1_make_ruc_header(qp, ohdr, bth0, bth1, bth2, middle, ps); 399f48ad614SDennis Dalessandro return 1; 40024b11923SKaike Wan error_qp: 40124b11923SKaike Wan spin_unlock_irqrestore(&qp->s_lock, ps->flags); 40224b11923SKaike Wan spin_lock_irqsave(&qp->r_lock, ps->flags); 40324b11923SKaike Wan spin_lock(&qp->s_lock); 40424b11923SKaike Wan rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 40524b11923SKaike Wan spin_unlock(&qp->s_lock); 40624b11923SKaike Wan spin_unlock_irqrestore(&qp->r_lock, ps->flags); 40724b11923SKaike Wan spin_lock_irqsave(&qp->s_lock, ps->flags); 408f48ad614SDennis Dalessandro bail: 409f48ad614SDennis Dalessandro qp->s_ack_state = OP(ACKNOWLEDGE); 410f48ad614SDennis Dalessandro /* 411f48ad614SDennis Dalessandro * Ensure s_rdma_ack_cnt changes are committed prior to resetting 412f48ad614SDennis Dalessandro * RVT_S_RESP_PENDING 413f48ad614SDennis Dalessandro */ 414f48ad614SDennis Dalessandro smp_wmb(); 415f48ad614SDennis Dalessandro qp->s_flags &= ~(RVT_S_RESP_PENDING 416f48ad614SDennis Dalessandro | RVT_S_ACK_PENDING 4172e2ba09eSMike Marciniszyn | HFI1_S_AHG_VALID); 418f48ad614SDennis Dalessandro return 0; 419f48ad614SDennis Dalessandro } 420f48ad614SDennis Dalessandro 421f48ad614SDennis Dalessandro /** 422f48ad614SDennis Dalessandro * hfi1_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC) 423f48ad614SDennis Dalessandro * @qp: a pointer to the QP 424f48ad614SDennis Dalessandro * 425f48ad614SDennis Dalessandro * Assumes s_lock is held. 426f48ad614SDennis Dalessandro * 427f48ad614SDennis Dalessandro * Return 1 if constructed; otherwise, return 0. 428f48ad614SDennis Dalessandro */ 429f48ad614SDennis Dalessandro int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) 430f48ad614SDennis Dalessandro { 431f48ad614SDennis Dalessandro struct hfi1_qp_priv *priv = qp->priv; 432f48ad614SDennis Dalessandro struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); 433261a4351SMike Marciniszyn struct ib_other_headers *ohdr; 43424b11923SKaike Wan struct rvt_sge_state *ss = NULL; 435f48ad614SDennis Dalessandro struct rvt_swqe *wqe; 43624b11923SKaike Wan struct hfi1_swqe_priv *wpriv; 43724b11923SKaike Wan struct tid_rdma_request *req = NULL; 43824b11923SKaike Wan /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 43924b11923SKaike Wan u32 hwords = 5; 44024b11923SKaike Wan u32 len = 0; 44124b11923SKaike Wan u32 bth0 = 0, bth2 = 0; 44244e43d91SMitko Haralanov u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); 443f48ad614SDennis Dalessandro u32 pmtu = qp->pmtu; 444f48ad614SDennis Dalessandro char newreq; 445f48ad614SDennis Dalessandro int middle = 0; 446f48ad614SDennis Dalessandro int delta; 44724b11923SKaike Wan struct tid_rdma_flow *flow = NULL; 4483c6cb20aSKaike Wan struct tid_rdma_params *remote; 449f48ad614SDennis Dalessandro 4503ce5daa2SKaike Wan trace_hfi1_sender_make_rc_req(qp); 45168e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 452f48ad614SDennis Dalessandro ps->s_txreq = get_txreq(ps->dev, qp); 453b697d7d8SMichael J. Ruhl if (!ps->s_txreq) 454f48ad614SDennis Dalessandro goto bail_no_tx; 455f48ad614SDennis Dalessandro 4565b6cabb0SDon Hiatt if (priv->hdr_type == HFI1_PKT_TYPE_9B) { 4575b6cabb0SDon Hiatt /* header size in 32-bit words LRH+BTH = (8+12)/4. */ 4585b6cabb0SDon Hiatt hwords = 5; 459d8966fcdSDasaratharaman Chandramouli if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) 46030e07416SDon Hiatt ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth; 4615b6cabb0SDon Hiatt else 4625b6cabb0SDon Hiatt ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth; 4635b6cabb0SDon Hiatt } else { 4645b6cabb0SDon Hiatt /* header size in 32-bit words 16B LRH+BTH = (16+12)/4. */ 4655b6cabb0SDon Hiatt hwords = 7; 4665b6cabb0SDon Hiatt if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && 4675b6cabb0SDon Hiatt (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr)))) 4685b6cabb0SDon Hiatt ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth; 4695b6cabb0SDon Hiatt else 4705b6cabb0SDon Hiatt ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth; 4715b6cabb0SDon Hiatt } 472f48ad614SDennis Dalessandro 473f48ad614SDennis Dalessandro /* Sending responses has higher priority over sending requests. */ 474f48ad614SDennis Dalessandro if ((qp->s_flags & RVT_S_RESP_PENDING) && 475f48ad614SDennis Dalessandro make_rc_ack(dev, qp, ohdr, ps)) 476f48ad614SDennis Dalessandro return 1; 477f48ad614SDennis Dalessandro 478f48ad614SDennis Dalessandro if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { 479f48ad614SDennis Dalessandro if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) 480f48ad614SDennis Dalessandro goto bail; 481f48ad614SDennis Dalessandro /* We are in the error state, flush the work request. */ 482eb04ff09SMike Marciniszyn if (qp->s_last == READ_ONCE(qp->s_head)) 483f48ad614SDennis Dalessandro goto bail; 484f48ad614SDennis Dalessandro /* If DMAs are in progress, we can't flush immediately. */ 485f48ad614SDennis Dalessandro if (iowait_sdma_pending(&priv->s_iowait)) { 486f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_DMA; 487f48ad614SDennis Dalessandro goto bail; 488f48ad614SDennis Dalessandro } 489f48ad614SDennis Dalessandro clear_ahg(qp); 490f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_last); 49124b11923SKaike Wan hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? 492f48ad614SDennis Dalessandro IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR); 493f48ad614SDennis Dalessandro /* will get called again */ 494f48ad614SDennis Dalessandro goto done_free_tx; 495f48ad614SDennis Dalessandro } 496f48ad614SDennis Dalessandro 4973c6cb20aSKaike Wan if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK | HFI1_S_WAIT_HALT)) 498f48ad614SDennis Dalessandro goto bail; 499f48ad614SDennis Dalessandro 500f48ad614SDennis Dalessandro if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) { 501f48ad614SDennis Dalessandro if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { 502f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_PSN; 503f48ad614SDennis Dalessandro goto bail; 504f48ad614SDennis Dalessandro } 505f48ad614SDennis Dalessandro qp->s_sending_psn = qp->s_psn; 506f48ad614SDennis Dalessandro qp->s_sending_hpsn = qp->s_psn - 1; 507f48ad614SDennis Dalessandro } 508f48ad614SDennis Dalessandro 509f48ad614SDennis Dalessandro /* Send a request. */ 510f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_cur); 51124b11923SKaike Wan check_s_state: 512f48ad614SDennis Dalessandro switch (qp->s_state) { 513f48ad614SDennis Dalessandro default: 514f48ad614SDennis Dalessandro if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) 515f48ad614SDennis Dalessandro goto bail; 516f48ad614SDennis Dalessandro /* 517f48ad614SDennis Dalessandro * Resend an old request or start a new one. 518f48ad614SDennis Dalessandro * 519f48ad614SDennis Dalessandro * We keep track of the current SWQE so that 520f48ad614SDennis Dalessandro * we don't reset the "furthest progress" state 521f48ad614SDennis Dalessandro * if we need to back up. 522f48ad614SDennis Dalessandro */ 523f48ad614SDennis Dalessandro newreq = 0; 524f48ad614SDennis Dalessandro if (qp->s_cur == qp->s_tail) { 525f48ad614SDennis Dalessandro /* Check if send work queue is empty. */ 526eb04ff09SMike Marciniszyn if (qp->s_tail == READ_ONCE(qp->s_head)) { 527f48ad614SDennis Dalessandro clear_ahg(qp); 528f48ad614SDennis Dalessandro goto bail; 529f48ad614SDennis Dalessandro } 530f48ad614SDennis Dalessandro /* 531f48ad614SDennis Dalessandro * If a fence is requested, wait for previous 532f48ad614SDennis Dalessandro * RDMA read and atomic operations to finish. 53324b11923SKaike Wan * However, there is no need to guard against 53424b11923SKaike Wan * TID RDMA READ after TID RDMA READ. 535f48ad614SDennis Dalessandro */ 536f48ad614SDennis Dalessandro if ((wqe->wr.send_flags & IB_SEND_FENCE) && 53724b11923SKaike Wan qp->s_num_rd_atomic && 53824b11923SKaike Wan (wqe->wr.opcode != IB_WR_TID_RDMA_READ || 53924b11923SKaike Wan priv->pending_tid_r_segs < qp->s_num_rd_atomic)) { 540f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_FENCE; 541f48ad614SDennis Dalessandro goto bail; 542f48ad614SDennis Dalessandro } 5430db3dfa0SJianxin Xiong /* 5440db3dfa0SJianxin Xiong * Local operations are processed immediately 5450db3dfa0SJianxin Xiong * after all prior requests have completed 5460db3dfa0SJianxin Xiong */ 5470db3dfa0SJianxin Xiong if (wqe->wr.opcode == IB_WR_REG_MR || 5480db3dfa0SJianxin Xiong wqe->wr.opcode == IB_WR_LOCAL_INV) { 549d9b13c20SJianxin Xiong int local_ops = 0; 550d9b13c20SJianxin Xiong int err = 0; 551d9b13c20SJianxin Xiong 5520db3dfa0SJianxin Xiong if (qp->s_last != qp->s_cur) 5530db3dfa0SJianxin Xiong goto bail; 5540db3dfa0SJianxin Xiong if (++qp->s_cur == qp->s_size) 5550db3dfa0SJianxin Xiong qp->s_cur = 0; 5560db3dfa0SJianxin Xiong if (++qp->s_tail == qp->s_size) 5570db3dfa0SJianxin Xiong qp->s_tail = 0; 558d9b13c20SJianxin Xiong if (!(wqe->wr.send_flags & 559d9b13c20SJianxin Xiong RVT_SEND_COMPLETION_ONLY)) { 5600db3dfa0SJianxin Xiong err = rvt_invalidate_rkey( 5610db3dfa0SJianxin Xiong qp, 5620db3dfa0SJianxin Xiong wqe->wr.ex.invalidate_rkey); 563d9b13c20SJianxin Xiong local_ops = 1; 564d9b13c20SJianxin Xiong } 565116aa033SVenkata Sandeep Dhanalakota rvt_send_complete(qp, wqe, 5660db3dfa0SJianxin Xiong err ? IB_WC_LOC_PROT_ERR 5670db3dfa0SJianxin Xiong : IB_WC_SUCCESS); 568d9b13c20SJianxin Xiong if (local_ops) 5690db3dfa0SJianxin Xiong atomic_dec(&qp->local_ops_pending); 5700db3dfa0SJianxin Xiong goto done_free_tx; 5710db3dfa0SJianxin Xiong } 5720db3dfa0SJianxin Xiong 573f48ad614SDennis Dalessandro newreq = 1; 574f48ad614SDennis Dalessandro qp->s_psn = wqe->psn; 575f48ad614SDennis Dalessandro } 576f48ad614SDennis Dalessandro /* 577f48ad614SDennis Dalessandro * Note that we have to be careful not to modify the 578f48ad614SDennis Dalessandro * original work request since we may need to resend 579f48ad614SDennis Dalessandro * it. 580f48ad614SDennis Dalessandro */ 581f48ad614SDennis Dalessandro len = wqe->length; 582f48ad614SDennis Dalessandro ss = &qp->s_sge; 583f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_psn); 584a0b34f75SKaike Wan 585a0b34f75SKaike Wan /* 586a0b34f75SKaike Wan * Interlock between various IB requests and TID RDMA 587a0b34f75SKaike Wan * if necessary. 588a0b34f75SKaike Wan */ 589a0b34f75SKaike Wan if ((priv->s_flags & HFI1_S_TID_WAIT_INTERLCK) || 590a0b34f75SKaike Wan hfi1_tid_rdma_wqe_interlock(qp, wqe)) 591a0b34f75SKaike Wan goto bail; 592a0b34f75SKaike Wan 593f48ad614SDennis Dalessandro switch (wqe->wr.opcode) { 594f48ad614SDennis Dalessandro case IB_WR_SEND: 595f48ad614SDennis Dalessandro case IB_WR_SEND_WITH_IMM: 5960db3dfa0SJianxin Xiong case IB_WR_SEND_WITH_INV: 597f48ad614SDennis Dalessandro /* If no credit, return. */ 59871994354SKaike Wan if (!rvt_rc_credit_avail(qp, wqe)) 599f48ad614SDennis Dalessandro goto bail; 600f48ad614SDennis Dalessandro if (len > pmtu) { 601f48ad614SDennis Dalessandro qp->s_state = OP(SEND_FIRST); 602f48ad614SDennis Dalessandro len = pmtu; 603f48ad614SDennis Dalessandro break; 604f48ad614SDennis Dalessandro } 605f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_SEND) { 606f48ad614SDennis Dalessandro qp->s_state = OP(SEND_ONLY); 6070db3dfa0SJianxin Xiong } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { 608f48ad614SDennis Dalessandro qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); 609f48ad614SDennis Dalessandro /* Immediate data comes after the BTH */ 610f48ad614SDennis Dalessandro ohdr->u.imm_data = wqe->wr.ex.imm_data; 611f48ad614SDennis Dalessandro hwords += 1; 6120db3dfa0SJianxin Xiong } else { 6130db3dfa0SJianxin Xiong qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE); 6140db3dfa0SJianxin Xiong /* Invalidate rkey comes after the BTH */ 6150db3dfa0SJianxin Xiong ohdr->u.ieth = cpu_to_be32( 6160db3dfa0SJianxin Xiong wqe->wr.ex.invalidate_rkey); 6170db3dfa0SJianxin Xiong hwords += 1; 618f48ad614SDennis Dalessandro } 619f48ad614SDennis Dalessandro if (wqe->wr.send_flags & IB_SEND_SOLICITED) 620f48ad614SDennis Dalessandro bth0 |= IB_BTH_SOLICITED; 621f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 622f48ad614SDennis Dalessandro if (++qp->s_cur == qp->s_size) 623f48ad614SDennis Dalessandro qp->s_cur = 0; 624f48ad614SDennis Dalessandro break; 625f48ad614SDennis Dalessandro 626f48ad614SDennis Dalessandro case IB_WR_RDMA_WRITE: 627f48ad614SDennis Dalessandro if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 628f48ad614SDennis Dalessandro qp->s_lsn++; 6295b0ef650SMike Marciniszyn goto no_flow_control; 630f48ad614SDennis Dalessandro case IB_WR_RDMA_WRITE_WITH_IMM: 631f48ad614SDennis Dalessandro /* If no credit, return. */ 63271994354SKaike Wan if (!rvt_rc_credit_avail(qp, wqe)) 633f48ad614SDennis Dalessandro goto bail; 6345b0ef650SMike Marciniszyn no_flow_control: 635261a4351SMike Marciniszyn put_ib_reth_vaddr( 636261a4351SMike Marciniszyn wqe->rdma_wr.remote_addr, 637261a4351SMike Marciniszyn &ohdr->u.rc.reth); 638f48ad614SDennis Dalessandro ohdr->u.rc.reth.rkey = 639f48ad614SDennis Dalessandro cpu_to_be32(wqe->rdma_wr.rkey); 640f48ad614SDennis Dalessandro ohdr->u.rc.reth.length = cpu_to_be32(len); 641f48ad614SDennis Dalessandro hwords += sizeof(struct ib_reth) / sizeof(u32); 642f48ad614SDennis Dalessandro if (len > pmtu) { 643f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_WRITE_FIRST); 644f48ad614SDennis Dalessandro len = pmtu; 645f48ad614SDennis Dalessandro break; 646f48ad614SDennis Dalessandro } 647f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { 648f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_WRITE_ONLY); 649f48ad614SDennis Dalessandro } else { 650f48ad614SDennis Dalessandro qp->s_state = 651f48ad614SDennis Dalessandro OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); 652f48ad614SDennis Dalessandro /* Immediate data comes after RETH */ 653f48ad614SDennis Dalessandro ohdr->u.rc.imm_data = wqe->wr.ex.imm_data; 654f48ad614SDennis Dalessandro hwords += 1; 655f48ad614SDennis Dalessandro if (wqe->wr.send_flags & IB_SEND_SOLICITED) 656f48ad614SDennis Dalessandro bth0 |= IB_BTH_SOLICITED; 657f48ad614SDennis Dalessandro } 658f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 659f48ad614SDennis Dalessandro if (++qp->s_cur == qp->s_size) 660f48ad614SDennis Dalessandro qp->s_cur = 0; 661f48ad614SDennis Dalessandro break; 662f48ad614SDennis Dalessandro 6633c6cb20aSKaike Wan case IB_WR_TID_RDMA_WRITE: 6643c6cb20aSKaike Wan if (newreq) { 6653c6cb20aSKaike Wan /* 6663c6cb20aSKaike Wan * Limit the number of TID RDMA WRITE requests. 6673c6cb20aSKaike Wan */ 6683c6cb20aSKaike Wan if (atomic_read(&priv->n_tid_requests) >= 6693c6cb20aSKaike Wan HFI1_TID_RDMA_WRITE_CNT) 6703c6cb20aSKaike Wan goto bail; 6713c6cb20aSKaike Wan 6723c6cb20aSKaike Wan if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 6733c6cb20aSKaike Wan qp->s_lsn++; 6743c6cb20aSKaike Wan } 6753c6cb20aSKaike Wan 6763c6cb20aSKaike Wan hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, 6773c6cb20aSKaike Wan &bth1, &bth2, 6783c6cb20aSKaike Wan &len); 6793c6cb20aSKaike Wan ss = NULL; 6803c6cb20aSKaike Wan if (priv->s_tid_cur == HFI1_QP_WQE_INVALID) { 6813c6cb20aSKaike Wan priv->s_tid_cur = qp->s_cur; 6823c6cb20aSKaike Wan if (priv->s_tid_tail == HFI1_QP_WQE_INVALID) { 6833c6cb20aSKaike Wan priv->s_tid_tail = qp->s_cur; 6843c6cb20aSKaike Wan priv->s_state = TID_OP(WRITE_RESP); 6853c6cb20aSKaike Wan } 6863c6cb20aSKaike Wan } else if (priv->s_tid_cur == priv->s_tid_head) { 6873c6cb20aSKaike Wan struct rvt_swqe *__w; 6883c6cb20aSKaike Wan struct tid_rdma_request *__r; 6893c6cb20aSKaike Wan 6903c6cb20aSKaike Wan __w = rvt_get_swqe_ptr(qp, priv->s_tid_cur); 6913c6cb20aSKaike Wan __r = wqe_to_tid_req(__w); 6923c6cb20aSKaike Wan 6933c6cb20aSKaike Wan /* 6943c6cb20aSKaike Wan * The s_tid_cur pointer is advanced to s_cur if 6953c6cb20aSKaike Wan * any of the following conditions about the WQE 6963c6cb20aSKaike Wan * to which s_ti_cur currently points to are 6973c6cb20aSKaike Wan * satisfied: 6983c6cb20aSKaike Wan * 1. The request is not a TID RDMA WRITE 6993c6cb20aSKaike Wan * request, 7003c6cb20aSKaike Wan * 2. The request is in the INACTIVE or 7013c6cb20aSKaike Wan * COMPLETE states (TID RDMA READ requests 7023c6cb20aSKaike Wan * stay at INACTIVE and TID RDMA WRITE 7033c6cb20aSKaike Wan * transition to COMPLETE when done), 7043c6cb20aSKaike Wan * 3. The request is in the ACTIVE or SYNC 7053c6cb20aSKaike Wan * state and the number of completed 7063c6cb20aSKaike Wan * segments is equal to the total segment 7073c6cb20aSKaike Wan * count. 7083c6cb20aSKaike Wan * (If ACTIVE, the request is waiting for 7093c6cb20aSKaike Wan * ACKs. If SYNC, the request has not 7103c6cb20aSKaike Wan * received any responses because it's 7113c6cb20aSKaike Wan * waiting on a sync point.) 7123c6cb20aSKaike Wan */ 7133c6cb20aSKaike Wan if (__w->wr.opcode != IB_WR_TID_RDMA_WRITE || 7143c6cb20aSKaike Wan __r->state == TID_REQUEST_INACTIVE || 7153c6cb20aSKaike Wan __r->state == TID_REQUEST_COMPLETE || 7163c6cb20aSKaike Wan ((__r->state == TID_REQUEST_ACTIVE || 7173c6cb20aSKaike Wan __r->state == TID_REQUEST_SYNC) && 7183c6cb20aSKaike Wan __r->comp_seg == __r->total_segs)) { 7193c6cb20aSKaike Wan if (priv->s_tid_tail == 7203c6cb20aSKaike Wan priv->s_tid_cur && 7213c6cb20aSKaike Wan priv->s_state == 7223c6cb20aSKaike Wan TID_OP(WRITE_DATA_LAST)) { 7233c6cb20aSKaike Wan priv->s_tid_tail = qp->s_cur; 7243c6cb20aSKaike Wan priv->s_state = 7253c6cb20aSKaike Wan TID_OP(WRITE_RESP); 7263c6cb20aSKaike Wan } 7273c6cb20aSKaike Wan priv->s_tid_cur = qp->s_cur; 7283c6cb20aSKaike Wan } 7293c6cb20aSKaike Wan /* 7303c6cb20aSKaike Wan * A corner case: when the last TID RDMA WRITE 7313c6cb20aSKaike Wan * request was completed, s_tid_head, 7323c6cb20aSKaike Wan * s_tid_cur, and s_tid_tail all point to the 7333c6cb20aSKaike Wan * same location. Other requests are posted and 7343c6cb20aSKaike Wan * s_cur wraps around to the same location, 7353c6cb20aSKaike Wan * where a new TID RDMA WRITE is posted. In 7363c6cb20aSKaike Wan * this case, none of the indices need to be 7373c6cb20aSKaike Wan * updated. However, the priv->s_state should. 7383c6cb20aSKaike Wan */ 7393c6cb20aSKaike Wan if (priv->s_tid_tail == qp->s_cur && 7403c6cb20aSKaike Wan priv->s_state == TID_OP(WRITE_DATA_LAST)) 7413c6cb20aSKaike Wan priv->s_state = TID_OP(WRITE_RESP); 7423c6cb20aSKaike Wan } 7433c6cb20aSKaike Wan req = wqe_to_tid_req(wqe); 7443c6cb20aSKaike Wan if (newreq) { 7453c6cb20aSKaike Wan priv->s_tid_head = qp->s_cur; 7463c6cb20aSKaike Wan priv->pending_tid_w_resp += req->total_segs; 7473c6cb20aSKaike Wan atomic_inc(&priv->n_tid_requests); 7483c6cb20aSKaike Wan atomic_dec(&priv->n_requests); 7493c6cb20aSKaike Wan } else { 7503c6cb20aSKaike Wan req->state = TID_REQUEST_RESEND; 7513c6cb20aSKaike Wan req->comp_seg = delta_psn(bth2, wqe->psn); 7523c6cb20aSKaike Wan /* 7533c6cb20aSKaike Wan * Pull back any segments since we are going 7543c6cb20aSKaike Wan * to re-receive them. 7553c6cb20aSKaike Wan */ 7563c6cb20aSKaike Wan req->setup_head = req->clear_tail; 7573c6cb20aSKaike Wan priv->pending_tid_w_resp += 7583c6cb20aSKaike Wan delta_psn(wqe->lpsn, bth2) + 1; 7593c6cb20aSKaike Wan } 7603c6cb20aSKaike Wan 761a05c9bdcSKaike Wan trace_hfi1_tid_write_sender_make_req(qp, newreq); 762a05c9bdcSKaike Wan trace_hfi1_tid_req_make_req_write(qp, newreq, 763a05c9bdcSKaike Wan wqe->wr.opcode, 764a05c9bdcSKaike Wan wqe->psn, wqe->lpsn, 765a05c9bdcSKaike Wan req); 7663c6cb20aSKaike Wan if (++qp->s_cur == qp->s_size) 7673c6cb20aSKaike Wan qp->s_cur = 0; 7683c6cb20aSKaike Wan break; 7693c6cb20aSKaike Wan 770f48ad614SDennis Dalessandro case IB_WR_RDMA_READ: 771f48ad614SDennis Dalessandro /* 772f48ad614SDennis Dalessandro * Don't allow more operations to be started 773f48ad614SDennis Dalessandro * than the QP limits allow. 774f48ad614SDennis Dalessandro */ 775f48ad614SDennis Dalessandro if (qp->s_num_rd_atomic >= 776f48ad614SDennis Dalessandro qp->s_max_rd_atomic) { 777f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_RDMAR; 778f48ad614SDennis Dalessandro goto bail; 779f48ad614SDennis Dalessandro } 780f48ad614SDennis Dalessandro qp->s_num_rd_atomic++; 781b126078eSKaike Wan if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 782f48ad614SDennis Dalessandro qp->s_lsn++; 783261a4351SMike Marciniszyn put_ib_reth_vaddr( 784261a4351SMike Marciniszyn wqe->rdma_wr.remote_addr, 785261a4351SMike Marciniszyn &ohdr->u.rc.reth); 786f48ad614SDennis Dalessandro ohdr->u.rc.reth.rkey = 787f48ad614SDennis Dalessandro cpu_to_be32(wqe->rdma_wr.rkey); 788f48ad614SDennis Dalessandro ohdr->u.rc.reth.length = cpu_to_be32(len); 789f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_READ_REQUEST); 790f48ad614SDennis Dalessandro hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 791f48ad614SDennis Dalessandro ss = NULL; 792f48ad614SDennis Dalessandro len = 0; 793f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 794f48ad614SDennis Dalessandro if (++qp->s_cur == qp->s_size) 795f48ad614SDennis Dalessandro qp->s_cur = 0; 796f48ad614SDennis Dalessandro break; 797f48ad614SDennis Dalessandro 79824b11923SKaike Wan case IB_WR_TID_RDMA_READ: 7993ce5daa2SKaike Wan trace_hfi1_tid_read_sender_make_req(qp, newreq); 80024b11923SKaike Wan wpriv = wqe->priv; 80124b11923SKaike Wan req = wqe_to_tid_req(wqe); 8023ce5daa2SKaike Wan trace_hfi1_tid_req_make_req_read(qp, newreq, 8033ce5daa2SKaike Wan wqe->wr.opcode, 8043ce5daa2SKaike Wan wqe->psn, wqe->lpsn, 8053ce5daa2SKaike Wan req); 80624b11923SKaike Wan delta = cmp_psn(qp->s_psn, wqe->psn); 80724b11923SKaike Wan 80824b11923SKaike Wan /* 80924b11923SKaike Wan * Don't allow more operations to be started 81024b11923SKaike Wan * than the QP limits allow. We could get here under 81124b11923SKaike Wan * three conditions; (1) It's a new request; (2) We are 81224b11923SKaike Wan * sending the second or later segment of a request, 81324b11923SKaike Wan * but the qp->s_state is set to OP(RDMA_READ_REQUEST) 81424b11923SKaike Wan * when the last segment of a previous request is 81524b11923SKaike Wan * received just before this; (3) We are re-sending a 81624b11923SKaike Wan * request. 81724b11923SKaike Wan */ 81824b11923SKaike Wan if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { 81924b11923SKaike Wan qp->s_flags |= RVT_S_WAIT_RDMAR; 82024b11923SKaike Wan goto bail; 82124b11923SKaike Wan } 82224b11923SKaike Wan if (newreq) { 82324b11923SKaike Wan struct tid_rdma_flow *flow = 82424b11923SKaike Wan &req->flows[req->setup_head]; 82524b11923SKaike Wan 82624b11923SKaike Wan /* 82724b11923SKaike Wan * Set up s_sge as it is needed for TID 82824b11923SKaike Wan * allocation. However, if the pages have been 82924b11923SKaike Wan * walked and mapped, skip it. An earlier try 83024b11923SKaike Wan * has failed to allocate the TID entries. 83124b11923SKaike Wan */ 83224b11923SKaike Wan if (!flow->npagesets) { 83324b11923SKaike Wan qp->s_sge.sge = wqe->sg_list[0]; 83424b11923SKaike Wan qp->s_sge.sg_list = wqe->sg_list + 1; 83524b11923SKaike Wan qp->s_sge.num_sge = wqe->wr.num_sge; 83624b11923SKaike Wan qp->s_sge.total_len = wqe->length; 83724b11923SKaike Wan qp->s_len = wqe->length; 83824b11923SKaike Wan req->isge = 0; 83924b11923SKaike Wan req->clear_tail = req->setup_head; 84024b11923SKaike Wan req->flow_idx = req->setup_head; 84124b11923SKaike Wan req->state = TID_REQUEST_ACTIVE; 84224b11923SKaike Wan } 84324b11923SKaike Wan } else if (delta == 0) { 84424b11923SKaike Wan /* Re-send a request */ 84524b11923SKaike Wan req->cur_seg = 0; 84624b11923SKaike Wan req->comp_seg = 0; 84724b11923SKaike Wan req->ack_pending = 0; 84824b11923SKaike Wan req->flow_idx = req->clear_tail; 84924b11923SKaike Wan req->state = TID_REQUEST_RESEND; 85024b11923SKaike Wan } 85124b11923SKaike Wan req->s_next_psn = qp->s_psn; 85224b11923SKaike Wan /* Read one segment at a time */ 85324b11923SKaike Wan len = min_t(u32, req->seg_len, 85424b11923SKaike Wan wqe->length - req->seg_len * req->cur_seg); 85524b11923SKaike Wan delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, 85624b11923SKaike Wan &bth1, &bth2, 85724b11923SKaike Wan &len); 85824b11923SKaike Wan if (delta <= 0) { 85924b11923SKaike Wan /* Wait for TID space */ 86024b11923SKaike Wan goto bail; 86124b11923SKaike Wan } 86224b11923SKaike Wan if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 86324b11923SKaike Wan qp->s_lsn++; 86424b11923SKaike Wan hwords += delta; 86524b11923SKaike Wan ss = &wpriv->ss; 86624b11923SKaike Wan /* Check if this is the last segment */ 86724b11923SKaike Wan if (req->cur_seg >= req->total_segs && 86824b11923SKaike Wan ++qp->s_cur == qp->s_size) 86924b11923SKaike Wan qp->s_cur = 0; 87024b11923SKaike Wan break; 87124b11923SKaike Wan 872f48ad614SDennis Dalessandro case IB_WR_ATOMIC_CMP_AND_SWP: 873f48ad614SDennis Dalessandro case IB_WR_ATOMIC_FETCH_AND_ADD: 874f48ad614SDennis Dalessandro /* 875f48ad614SDennis Dalessandro * Don't allow more operations to be started 876f48ad614SDennis Dalessandro * than the QP limits allow. 877f48ad614SDennis Dalessandro */ 878f48ad614SDennis Dalessandro if (qp->s_num_rd_atomic >= 879f48ad614SDennis Dalessandro qp->s_max_rd_atomic) { 880f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_RDMAR; 881f48ad614SDennis Dalessandro goto bail; 882f48ad614SDennis Dalessandro } 883f48ad614SDennis Dalessandro qp->s_num_rd_atomic++; 88448a615dcSKaike Wan 88548a615dcSKaike Wan /* FALLTHROUGH */ 88648a615dcSKaike Wan case IB_WR_OPFN: 88748a615dcSKaike Wan if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) 88848a615dcSKaike Wan qp->s_lsn++; 88948a615dcSKaike Wan if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 89048a615dcSKaike Wan wqe->wr.opcode == IB_WR_OPFN) { 891f48ad614SDennis Dalessandro qp->s_state = OP(COMPARE_SWAP); 892261a4351SMike Marciniszyn put_ib_ateth_swap(wqe->atomic_wr.swap, 893261a4351SMike Marciniszyn &ohdr->u.atomic_eth); 894261a4351SMike Marciniszyn put_ib_ateth_compare(wqe->atomic_wr.compare_add, 895261a4351SMike Marciniszyn &ohdr->u.atomic_eth); 896f48ad614SDennis Dalessandro } else { 897f48ad614SDennis Dalessandro qp->s_state = OP(FETCH_ADD); 898261a4351SMike Marciniszyn put_ib_ateth_swap(wqe->atomic_wr.compare_add, 899261a4351SMike Marciniszyn &ohdr->u.atomic_eth); 900261a4351SMike Marciniszyn put_ib_ateth_compare(0, &ohdr->u.atomic_eth); 901f48ad614SDennis Dalessandro } 902261a4351SMike Marciniszyn put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, 903261a4351SMike Marciniszyn &ohdr->u.atomic_eth); 904f48ad614SDennis Dalessandro ohdr->u.atomic_eth.rkey = cpu_to_be32( 905f48ad614SDennis Dalessandro wqe->atomic_wr.rkey); 906f48ad614SDennis Dalessandro hwords += sizeof(struct ib_atomic_eth) / sizeof(u32); 907f48ad614SDennis Dalessandro ss = NULL; 908f48ad614SDennis Dalessandro len = 0; 909f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 910f48ad614SDennis Dalessandro if (++qp->s_cur == qp->s_size) 911f48ad614SDennis Dalessandro qp->s_cur = 0; 912f48ad614SDennis Dalessandro break; 913f48ad614SDennis Dalessandro 914f48ad614SDennis Dalessandro default: 915f48ad614SDennis Dalessandro goto bail; 916f48ad614SDennis Dalessandro } 91724b11923SKaike Wan if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) { 918f48ad614SDennis Dalessandro qp->s_sge.sge = wqe->sg_list[0]; 919f48ad614SDennis Dalessandro qp->s_sge.sg_list = wqe->sg_list + 1; 920f48ad614SDennis Dalessandro qp->s_sge.num_sge = wqe->wr.num_sge; 921f48ad614SDennis Dalessandro qp->s_sge.total_len = wqe->length; 922f48ad614SDennis Dalessandro qp->s_len = wqe->length; 92324b11923SKaike Wan } 924f48ad614SDennis Dalessandro if (newreq) { 925f48ad614SDennis Dalessandro qp->s_tail++; 926f48ad614SDennis Dalessandro if (qp->s_tail >= qp->s_size) 927f48ad614SDennis Dalessandro qp->s_tail = 0; 928f48ad614SDennis Dalessandro } 9293c6cb20aSKaike Wan if (wqe->wr.opcode == IB_WR_RDMA_READ || 9303c6cb20aSKaike Wan wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) 931f48ad614SDennis Dalessandro qp->s_psn = wqe->lpsn + 1; 93224b11923SKaike Wan else if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) 93324b11923SKaike Wan qp->s_psn = req->s_next_psn; 934f48ad614SDennis Dalessandro else 935f48ad614SDennis Dalessandro qp->s_psn++; 936f48ad614SDennis Dalessandro break; 937f48ad614SDennis Dalessandro 938f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_FIRST): 939f48ad614SDennis Dalessandro /* 940f48ad614SDennis Dalessandro * qp->s_state is normally set to the opcode of the 941f48ad614SDennis Dalessandro * last packet constructed for new requests and therefore 942f48ad614SDennis Dalessandro * is never set to RDMA read response. 943f48ad614SDennis Dalessandro * RDMA_READ_RESPONSE_FIRST is used by the ACK processing 944f48ad614SDennis Dalessandro * thread to indicate a SEND needs to be restarted from an 945f48ad614SDennis Dalessandro * earlier PSN without interfering with the sending thread. 946f48ad614SDennis Dalessandro * See restart_rc(). 947f48ad614SDennis Dalessandro */ 948f48ad614SDennis Dalessandro qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 949f48ad614SDennis Dalessandro /* FALLTHROUGH */ 950f48ad614SDennis Dalessandro case OP(SEND_FIRST): 951f48ad614SDennis Dalessandro qp->s_state = OP(SEND_MIDDLE); 952f48ad614SDennis Dalessandro /* FALLTHROUGH */ 953f48ad614SDennis Dalessandro case OP(SEND_MIDDLE): 954f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_psn++); 955f48ad614SDennis Dalessandro ss = &qp->s_sge; 956f48ad614SDennis Dalessandro len = qp->s_len; 957f48ad614SDennis Dalessandro if (len > pmtu) { 958f48ad614SDennis Dalessandro len = pmtu; 959f48ad614SDennis Dalessandro middle = HFI1_CAP_IS_KSET(SDMA_AHG); 960f48ad614SDennis Dalessandro break; 961f48ad614SDennis Dalessandro } 962f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_SEND) { 963f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST); 9640db3dfa0SJianxin Xiong } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) { 965f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); 966f48ad614SDennis Dalessandro /* Immediate data comes after the BTH */ 967f48ad614SDennis Dalessandro ohdr->u.imm_data = wqe->wr.ex.imm_data; 968f48ad614SDennis Dalessandro hwords += 1; 9690db3dfa0SJianxin Xiong } else { 9700db3dfa0SJianxin Xiong qp->s_state = OP(SEND_LAST_WITH_INVALIDATE); 9710db3dfa0SJianxin Xiong /* invalidate data comes after the BTH */ 9720db3dfa0SJianxin Xiong ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey); 9730db3dfa0SJianxin Xiong hwords += 1; 974f48ad614SDennis Dalessandro } 975f48ad614SDennis Dalessandro if (wqe->wr.send_flags & IB_SEND_SOLICITED) 976f48ad614SDennis Dalessandro bth0 |= IB_BTH_SOLICITED; 977f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 978f48ad614SDennis Dalessandro qp->s_cur++; 979f48ad614SDennis Dalessandro if (qp->s_cur >= qp->s_size) 980f48ad614SDennis Dalessandro qp->s_cur = 0; 981f48ad614SDennis Dalessandro break; 982f48ad614SDennis Dalessandro 983f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_LAST): 984f48ad614SDennis Dalessandro /* 985f48ad614SDennis Dalessandro * qp->s_state is normally set to the opcode of the 986f48ad614SDennis Dalessandro * last packet constructed for new requests and therefore 987f48ad614SDennis Dalessandro * is never set to RDMA read response. 988f48ad614SDennis Dalessandro * RDMA_READ_RESPONSE_LAST is used by the ACK processing 989f48ad614SDennis Dalessandro * thread to indicate a RDMA write needs to be restarted from 990f48ad614SDennis Dalessandro * an earlier PSN without interfering with the sending thread. 991f48ad614SDennis Dalessandro * See restart_rc(). 992f48ad614SDennis Dalessandro */ 993f48ad614SDennis Dalessandro qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); 994f48ad614SDennis Dalessandro /* FALLTHROUGH */ 995f48ad614SDennis Dalessandro case OP(RDMA_WRITE_FIRST): 996f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_WRITE_MIDDLE); 997f48ad614SDennis Dalessandro /* FALLTHROUGH */ 998f48ad614SDennis Dalessandro case OP(RDMA_WRITE_MIDDLE): 999f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_psn++); 1000f48ad614SDennis Dalessandro ss = &qp->s_sge; 1001f48ad614SDennis Dalessandro len = qp->s_len; 1002f48ad614SDennis Dalessandro if (len > pmtu) { 1003f48ad614SDennis Dalessandro len = pmtu; 1004f48ad614SDennis Dalessandro middle = HFI1_CAP_IS_KSET(SDMA_AHG); 1005f48ad614SDennis Dalessandro break; 1006f48ad614SDennis Dalessandro } 1007f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_RDMA_WRITE) { 1008f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_WRITE_LAST); 1009f48ad614SDennis Dalessandro } else { 1010f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); 1011f48ad614SDennis Dalessandro /* Immediate data comes after the BTH */ 1012f48ad614SDennis Dalessandro ohdr->u.imm_data = wqe->wr.ex.imm_data; 1013f48ad614SDennis Dalessandro hwords += 1; 1014f48ad614SDennis Dalessandro if (wqe->wr.send_flags & IB_SEND_SOLICITED) 1015f48ad614SDennis Dalessandro bth0 |= IB_BTH_SOLICITED; 1016f48ad614SDennis Dalessandro } 1017f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 1018f48ad614SDennis Dalessandro qp->s_cur++; 1019f48ad614SDennis Dalessandro if (qp->s_cur >= qp->s_size) 1020f48ad614SDennis Dalessandro qp->s_cur = 0; 1021f48ad614SDennis Dalessandro break; 1022f48ad614SDennis Dalessandro 1023f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_MIDDLE): 1024f48ad614SDennis Dalessandro /* 1025f48ad614SDennis Dalessandro * qp->s_state is normally set to the opcode of the 1026f48ad614SDennis Dalessandro * last packet constructed for new requests and therefore 1027f48ad614SDennis Dalessandro * is never set to RDMA read response. 1028f48ad614SDennis Dalessandro * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing 1029f48ad614SDennis Dalessandro * thread to indicate a RDMA read needs to be restarted from 1030f48ad614SDennis Dalessandro * an earlier PSN without interfering with the sending thread. 1031f48ad614SDennis Dalessandro * See restart_rc(). 1032f48ad614SDennis Dalessandro */ 1033f48ad614SDennis Dalessandro len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; 1034261a4351SMike Marciniszyn put_ib_reth_vaddr( 1035261a4351SMike Marciniszyn wqe->rdma_wr.remote_addr + len, 1036261a4351SMike Marciniszyn &ohdr->u.rc.reth); 1037f48ad614SDennis Dalessandro ohdr->u.rc.reth.rkey = 1038f48ad614SDennis Dalessandro cpu_to_be32(wqe->rdma_wr.rkey); 1039f48ad614SDennis Dalessandro ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); 1040f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_READ_REQUEST); 1041f48ad614SDennis Dalessandro hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); 1042f48ad614SDennis Dalessandro bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK; 1043f48ad614SDennis Dalessandro qp->s_psn = wqe->lpsn + 1; 1044f48ad614SDennis Dalessandro ss = NULL; 1045f48ad614SDennis Dalessandro len = 0; 1046f48ad614SDennis Dalessandro qp->s_cur++; 1047f48ad614SDennis Dalessandro if (qp->s_cur == qp->s_size) 1048f48ad614SDennis Dalessandro qp->s_cur = 0; 1049f48ad614SDennis Dalessandro break; 10503c6cb20aSKaike Wan 10513c6cb20aSKaike Wan case TID_OP(WRITE_RESP): 10523c6cb20aSKaike Wan /* 10533c6cb20aSKaike Wan * This value for s_state is used for restarting a TID RDMA 10543c6cb20aSKaike Wan * WRITE request. See comment in OP(RDMA_READ_RESPONSE_MIDDLE 10553c6cb20aSKaike Wan * for more). 10563c6cb20aSKaike Wan */ 10573c6cb20aSKaike Wan req = wqe_to_tid_req(wqe); 10583c6cb20aSKaike Wan req->state = TID_REQUEST_RESEND; 10593c6cb20aSKaike Wan rcu_read_lock(); 10603c6cb20aSKaike Wan remote = rcu_dereference(priv->tid_rdma.remote); 10613c6cb20aSKaike Wan req->comp_seg = delta_psn(qp->s_psn, wqe->psn); 10623c6cb20aSKaike Wan len = wqe->length - (req->comp_seg * remote->max_len); 10633c6cb20aSKaike Wan rcu_read_unlock(); 10643c6cb20aSKaike Wan 10653c6cb20aSKaike Wan bth2 = mask_psn(qp->s_psn); 10663c6cb20aSKaike Wan hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, 10673c6cb20aSKaike Wan &bth2, &len); 10683c6cb20aSKaike Wan qp->s_psn = wqe->lpsn + 1; 10693c6cb20aSKaike Wan ss = NULL; 10703c6cb20aSKaike Wan qp->s_state = TID_OP(WRITE_REQ); 10713c6cb20aSKaike Wan priv->pending_tid_w_resp += delta_psn(wqe->lpsn, bth2) + 1; 10723c6cb20aSKaike Wan priv->s_tid_cur = qp->s_cur; 10733c6cb20aSKaike Wan if (++qp->s_cur == qp->s_size) 10743c6cb20aSKaike Wan qp->s_cur = 0; 1075a05c9bdcSKaike Wan trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, 1076a05c9bdcSKaike Wan wqe->psn, wqe->lpsn, req); 10773c6cb20aSKaike Wan break; 10783c6cb20aSKaike Wan 107924b11923SKaike Wan case TID_OP(READ_RESP): 108024b11923SKaike Wan if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) 108124b11923SKaike Wan goto bail; 108224b11923SKaike Wan /* This is used to restart a TID read request */ 108324b11923SKaike Wan req = wqe_to_tid_req(wqe); 108424b11923SKaike Wan wpriv = wqe->priv; 108524b11923SKaike Wan /* 108624b11923SKaike Wan * Back down. The field qp->s_psn has been set to the psn with 108724b11923SKaike Wan * which the request should be restart. It's OK to use division 108824b11923SKaike Wan * as this is on the retry path. 108924b11923SKaike Wan */ 109024b11923SKaike Wan req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; 109124b11923SKaike Wan 109224b11923SKaike Wan /* 109324b11923SKaike Wan * The following function need to be redefined to return the 109424b11923SKaike Wan * status to make sure that we find the flow. At the same 109524b11923SKaike Wan * time, we can use the req->state change to check if the 109624b11923SKaike Wan * call succeeds or not. 109724b11923SKaike Wan */ 109824b11923SKaike Wan req->state = TID_REQUEST_RESEND; 109924b11923SKaike Wan hfi1_tid_rdma_restart_req(qp, wqe, &bth2); 110024b11923SKaike Wan if (req->state != TID_REQUEST_ACTIVE) { 110124b11923SKaike Wan /* 110224b11923SKaike Wan * Failed to find the flow. Release all allocated tid 110324b11923SKaike Wan * resources. 110424b11923SKaike Wan */ 110524b11923SKaike Wan hfi1_kern_exp_rcv_clear_all(req); 110624b11923SKaike Wan hfi1_kern_clear_hw_flow(priv->rcd, qp); 110724b11923SKaike Wan 110824b11923SKaike Wan hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); 110924b11923SKaike Wan goto bail; 111024b11923SKaike Wan } 111124b11923SKaike Wan req->state = TID_REQUEST_RESEND; 111224b11923SKaike Wan len = min_t(u32, req->seg_len, 111324b11923SKaike Wan wqe->length - req->seg_len * req->cur_seg); 111424b11923SKaike Wan flow = &req->flows[req->flow_idx]; 111524b11923SKaike Wan len -= flow->sent; 111624b11923SKaike Wan req->s_next_psn = flow->flow_state.ib_lpsn + 1; 111724b11923SKaike Wan delta = hfi1_build_tid_rdma_read_packet(wqe, ohdr, &bth1, 111824b11923SKaike Wan &bth2, &len); 111924b11923SKaike Wan if (delta <= 0) { 112024b11923SKaike Wan /* Wait for TID space */ 112124b11923SKaike Wan goto bail; 112224b11923SKaike Wan } 112324b11923SKaike Wan hwords += delta; 112424b11923SKaike Wan ss = &wpriv->ss; 112524b11923SKaike Wan /* Check if this is the last segment */ 112624b11923SKaike Wan if (req->cur_seg >= req->total_segs && 112724b11923SKaike Wan ++qp->s_cur == qp->s_size) 112824b11923SKaike Wan qp->s_cur = 0; 112924b11923SKaike Wan qp->s_psn = req->s_next_psn; 11303ce5daa2SKaike Wan trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, 11313ce5daa2SKaike Wan wqe->psn, wqe->lpsn, req); 113224b11923SKaike Wan break; 113324b11923SKaike Wan case TID_OP(READ_REQ): 113424b11923SKaike Wan req = wqe_to_tid_req(wqe); 113524b11923SKaike Wan delta = cmp_psn(qp->s_psn, wqe->psn); 113624b11923SKaike Wan /* 113724b11923SKaike Wan * If the current WR is not TID RDMA READ, or this is the start 113824b11923SKaike Wan * of a new request, we need to change the qp->s_state so that 113924b11923SKaike Wan * the request can be set up properly. 114024b11923SKaike Wan */ 114124b11923SKaike Wan if (wqe->wr.opcode != IB_WR_TID_RDMA_READ || delta == 0 || 114224b11923SKaike Wan qp->s_cur == qp->s_tail) { 114324b11923SKaike Wan qp->s_state = OP(RDMA_READ_REQUEST); 114424b11923SKaike Wan if (delta == 0 || qp->s_cur == qp->s_tail) 114524b11923SKaike Wan goto check_s_state; 114624b11923SKaike Wan else 114724b11923SKaike Wan goto bail; 114824b11923SKaike Wan } 114924b11923SKaike Wan 115024b11923SKaike Wan /* Rate limiting */ 115124b11923SKaike Wan if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { 115224b11923SKaike Wan qp->s_flags |= RVT_S_WAIT_RDMAR; 115324b11923SKaike Wan goto bail; 115424b11923SKaike Wan } 115524b11923SKaike Wan 115624b11923SKaike Wan wpriv = wqe->priv; 115724b11923SKaike Wan /* Read one segment at a time */ 115824b11923SKaike Wan len = min_t(u32, req->seg_len, 115924b11923SKaike Wan wqe->length - req->seg_len * req->cur_seg); 116024b11923SKaike Wan delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, 116124b11923SKaike Wan &bth2, &len); 116224b11923SKaike Wan if (delta <= 0) { 116324b11923SKaike Wan /* Wait for TID space */ 116424b11923SKaike Wan goto bail; 116524b11923SKaike Wan } 116624b11923SKaike Wan hwords += delta; 116724b11923SKaike Wan ss = &wpriv->ss; 116824b11923SKaike Wan /* Check if this is the last segment */ 116924b11923SKaike Wan if (req->cur_seg >= req->total_segs && 117024b11923SKaike Wan ++qp->s_cur == qp->s_size) 117124b11923SKaike Wan qp->s_cur = 0; 117224b11923SKaike Wan qp->s_psn = req->s_next_psn; 11733ce5daa2SKaike Wan trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, 11743ce5daa2SKaike Wan wqe->psn, wqe->lpsn, req); 117524b11923SKaike Wan break; 1176f48ad614SDennis Dalessandro } 1177f48ad614SDennis Dalessandro qp->s_sending_hpsn = bth2; 1178f48ad614SDennis Dalessandro delta = delta_psn(bth2, wqe->psn); 11793c6cb20aSKaike Wan if (delta && delta % HFI1_PSN_CREDIT == 0 && 11803c6cb20aSKaike Wan wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) 1181f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 1182f48ad614SDennis Dalessandro if (qp->s_flags & RVT_S_SEND_ONE) { 1183f48ad614SDennis Dalessandro qp->s_flags &= ~RVT_S_SEND_ONE; 1184f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_ACK; 1185f48ad614SDennis Dalessandro bth2 |= IB_BTH_REQ_ACK; 1186f48ad614SDennis Dalessandro } 1187f48ad614SDennis Dalessandro qp->s_len -= len; 11889636258fSMitko Haralanov ps->s_txreq->hdr_dwords = hwords; 1189f48ad614SDennis Dalessandro ps->s_txreq->sde = priv->s_sde; 1190b777f154SMitko Haralanov ps->s_txreq->ss = ss; 1191e922ae06SDon Hiatt ps->s_txreq->s_cur_size = len; 1192f48ad614SDennis Dalessandro hfi1_make_ruc_header( 1193f48ad614SDennis Dalessandro qp, 1194f48ad614SDennis Dalessandro ohdr, 1195f48ad614SDennis Dalessandro bth0 | (qp->s_state << 24), 119644e43d91SMitko Haralanov bth1, 1197f48ad614SDennis Dalessandro bth2, 1198f48ad614SDennis Dalessandro middle, 1199f48ad614SDennis Dalessandro ps); 1200f48ad614SDennis Dalessandro return 1; 1201f48ad614SDennis Dalessandro 1202f48ad614SDennis Dalessandro done_free_tx: 1203f48ad614SDennis Dalessandro hfi1_put_txreq(ps->s_txreq); 1204f48ad614SDennis Dalessandro ps->s_txreq = NULL; 1205f48ad614SDennis Dalessandro return 1; 1206f48ad614SDennis Dalessandro 1207f48ad614SDennis Dalessandro bail: 1208f48ad614SDennis Dalessandro hfi1_put_txreq(ps->s_txreq); 1209f48ad614SDennis Dalessandro 1210f48ad614SDennis Dalessandro bail_no_tx: 1211f48ad614SDennis Dalessandro ps->s_txreq = NULL; 1212f48ad614SDennis Dalessandro qp->s_flags &= ~RVT_S_BUSY; 12133c6cb20aSKaike Wan /* 12143c6cb20aSKaike Wan * If we didn't get a txreq, the QP will be woken up later to try 12153c6cb20aSKaike Wan * again. Set the flags to indicate which work item to wake 12163c6cb20aSKaike Wan * up. 12173c6cb20aSKaike Wan */ 12183c6cb20aSKaike Wan iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_IB); 1219f48ad614SDennis Dalessandro return 0; 1220f48ad614SDennis Dalessandro } 1221f48ad614SDennis Dalessandro 12225b6cabb0SDon Hiatt static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, 12235b6cabb0SDon Hiatt struct ib_other_headers *ohdr, 12245b6cabb0SDon Hiatt u32 bth0, u32 bth1) 1225f48ad614SDennis Dalessandro { 1226f48ad614SDennis Dalessandro if (qp->r_nak_state) 1227832666c1SDon Hiatt ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | 1228f48ad614SDennis Dalessandro (qp->r_nak_state << 1229832666c1SDon Hiatt IB_AETH_CREDIT_SHIFT)); 1230f48ad614SDennis Dalessandro else 1231696513e8SBrian Welty ohdr->u.aeth = rvt_compute_aeth(qp); 12325b6cabb0SDon Hiatt 1233f48ad614SDennis Dalessandro ohdr->bth[0] = cpu_to_be32(bth0); 12345b6cabb0SDon Hiatt ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn); 1235f48ad614SDennis Dalessandro ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn)); 1236f48ad614SDennis Dalessandro } 1237f48ad614SDennis Dalessandro 1238bdaf96f6SSebastian Sanchez static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn) 12395b6cabb0SDon Hiatt { 1240bdaf96f6SSebastian Sanchez struct rvt_qp *qp = packet->qp; 1241bdaf96f6SSebastian Sanchez struct hfi1_ibport *ibp; 12425b6cabb0SDon Hiatt unsigned long flags; 1243f48ad614SDennis Dalessandro 1244f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->s_lock, flags); 124572f53af2SMike Marciniszyn if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) 124672f53af2SMike Marciniszyn goto unlock; 1247bdaf96f6SSebastian Sanchez ibp = rcd_to_iport(packet->rcd); 124872f53af2SMike Marciniszyn this_cpu_inc(*ibp->rvp.rc_qacks); 1249f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; 1250f48ad614SDennis Dalessandro qp->s_nak_state = qp->r_nak_state; 1251f48ad614SDennis Dalessandro qp->s_ack_psn = qp->r_ack_psn; 1252f48ad614SDennis Dalessandro if (is_fecn) 1253f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_ECN; 1254f48ad614SDennis Dalessandro 12555b6cabb0SDon Hiatt /* Schedule the send tasklet. */ 1256f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 125772f53af2SMike Marciniszyn unlock: 1258f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 1259f48ad614SDennis Dalessandro } 1260f48ad614SDennis Dalessandro 1261bdaf96f6SSebastian Sanchez static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet, 12625b6cabb0SDon Hiatt struct hfi1_opa_header *opa_hdr, 12635b6cabb0SDon Hiatt u8 sc5, bool is_fecn, 12645b6cabb0SDon Hiatt u64 *pbc_flags, u32 *hwords, 12655b6cabb0SDon Hiatt u32 *nwords) 12665b6cabb0SDon Hiatt { 1267bdaf96f6SSebastian Sanchez struct rvt_qp *qp = packet->qp; 1268bdaf96f6SSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); 12695b6cabb0SDon Hiatt struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 12705b6cabb0SDon Hiatt struct ib_header *hdr = &opa_hdr->ibh; 12715b6cabb0SDon Hiatt struct ib_other_headers *ohdr; 12725b6cabb0SDon Hiatt u16 lrh0 = HFI1_LRH_BTH; 12735b6cabb0SDon Hiatt u16 pkey; 12745b6cabb0SDon Hiatt u32 bth0, bth1; 12755b6cabb0SDon Hiatt 12765b6cabb0SDon Hiatt opa_hdr->hdr_type = HFI1_PKT_TYPE_9B; 12775b6cabb0SDon Hiatt ohdr = &hdr->u.oth; 12785b6cabb0SDon Hiatt /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4 */ 12795b6cabb0SDon Hiatt *hwords = 6; 12805b6cabb0SDon Hiatt 12815b6cabb0SDon Hiatt if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { 12825b6cabb0SDon Hiatt *hwords += hfi1_make_grh(ibp, &hdr->u.l.grh, 12835b6cabb0SDon Hiatt rdma_ah_read_grh(&qp->remote_ah_attr), 12845b6cabb0SDon Hiatt *hwords - 2, SIZE_OF_CRC); 12855b6cabb0SDon Hiatt ohdr = &hdr->u.l.oth; 12865b6cabb0SDon Hiatt lrh0 = HFI1_LRH_GRH; 12875b6cabb0SDon Hiatt } 12885b6cabb0SDon Hiatt /* set PBC_DC_INFO bit (aka SC[4]) in pbc_flags */ 12895b6cabb0SDon Hiatt *pbc_flags |= ((!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT); 12905b6cabb0SDon Hiatt 12915b6cabb0SDon Hiatt /* read pkey_index w/o lock (its atomic) */ 12925b6cabb0SDon Hiatt pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); 12935b6cabb0SDon Hiatt 12945b6cabb0SDon Hiatt lrh0 |= (sc5 & IB_SC_MASK) << IB_SC_SHIFT | 12955b6cabb0SDon Hiatt (rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) << 12965b6cabb0SDon Hiatt IB_SL_SHIFT; 12975b6cabb0SDon Hiatt 12985b6cabb0SDon Hiatt hfi1_make_ib_hdr(hdr, lrh0, *hwords + SIZE_OF_CRC, 12995b6cabb0SDon Hiatt opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), 13005b6cabb0SDon Hiatt ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr)); 13015b6cabb0SDon Hiatt 13025b6cabb0SDon Hiatt bth0 = pkey | (OP(ACKNOWLEDGE) << 24); 13035b6cabb0SDon Hiatt if (qp->s_mig_state == IB_MIG_MIGRATED) 13045b6cabb0SDon Hiatt bth0 |= IB_BTH_MIG_REQ; 13055b6cabb0SDon Hiatt bth1 = (!!is_fecn) << IB_BECN_SHIFT; 130644e43d91SMitko Haralanov /* 130744e43d91SMitko Haralanov * Inline ACKs go out without the use of the Verbs send engine, so 130844e43d91SMitko Haralanov * we need to set the STL Verbs Extended bit here 130944e43d91SMitko Haralanov */ 131044e43d91SMitko Haralanov bth1 |= HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT; 13115b6cabb0SDon Hiatt hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); 13125b6cabb0SDon Hiatt } 13135b6cabb0SDon Hiatt 1314bdaf96f6SSebastian Sanchez static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet, 13155b6cabb0SDon Hiatt struct hfi1_opa_header *opa_hdr, 13165b6cabb0SDon Hiatt u8 sc5, bool is_fecn, 13175b6cabb0SDon Hiatt u64 *pbc_flags, u32 *hwords, 13185b6cabb0SDon Hiatt u32 *nwords) 13195b6cabb0SDon Hiatt { 1320bdaf96f6SSebastian Sanchez struct rvt_qp *qp = packet->qp; 1321bdaf96f6SSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd); 13225b6cabb0SDon Hiatt struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 13235b6cabb0SDon Hiatt struct hfi1_16b_header *hdr = &opa_hdr->opah; 13245b6cabb0SDon Hiatt struct ib_other_headers *ohdr; 13258935780bSDennis Dalessandro u32 bth0, bth1 = 0; 13265b6cabb0SDon Hiatt u16 len, pkey; 1327ca85bb1cSSebastian Sanchez bool becn = is_fecn; 13285b6cabb0SDon Hiatt u8 l4 = OPA_16B_L4_IB_LOCAL; 13295b6cabb0SDon Hiatt u8 extra_bytes; 13305b6cabb0SDon Hiatt 13315b6cabb0SDon Hiatt opa_hdr->hdr_type = HFI1_PKT_TYPE_16B; 13325b6cabb0SDon Hiatt ohdr = &hdr->u.oth; 13335b6cabb0SDon Hiatt /* header size in 32-bit words 16B LRH+BTH+AETH = (16+12+4)/4 */ 13345b6cabb0SDon Hiatt *hwords = 8; 13355b6cabb0SDon Hiatt extra_bytes = hfi1_get_16b_padding(*hwords << 2, 0); 13365b6cabb0SDon Hiatt *nwords = SIZE_OF_CRC + ((extra_bytes + SIZE_OF_LT) >> 2); 13375b6cabb0SDon Hiatt 13385b6cabb0SDon Hiatt if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && 13395b6cabb0SDon Hiatt hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { 13405b6cabb0SDon Hiatt *hwords += hfi1_make_grh(ibp, &hdr->u.l.grh, 13415b6cabb0SDon Hiatt rdma_ah_read_grh(&qp->remote_ah_attr), 13425b6cabb0SDon Hiatt *hwords - 4, *nwords); 13435b6cabb0SDon Hiatt ohdr = &hdr->u.l.oth; 13445b6cabb0SDon Hiatt l4 = OPA_16B_L4_IB_GLOBAL; 13455b6cabb0SDon Hiatt } 13465b6cabb0SDon Hiatt *pbc_flags |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC; 13475b6cabb0SDon Hiatt 13485b6cabb0SDon Hiatt /* read pkey_index w/o lock (its atomic) */ 13495b6cabb0SDon Hiatt pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); 13505b6cabb0SDon Hiatt 13515b6cabb0SDon Hiatt /* Convert dwords to flits */ 13525b6cabb0SDon Hiatt len = (*hwords + *nwords) >> 1; 13535b6cabb0SDon Hiatt 13543cafad43SDon Hiatt hfi1_make_16b_hdr(hdr, ppd->lid | 13553cafad43SDon Hiatt (rdma_ah_get_path_bits(&qp->remote_ah_attr) & 13563cafad43SDon Hiatt ((1 << ppd->lmc) - 1)), 13575b6cabb0SDon Hiatt opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 13583cafad43SDon Hiatt 16B), len, pkey, becn, 0, l4, sc5); 13595b6cabb0SDon Hiatt 13605b6cabb0SDon Hiatt bth0 = pkey | (OP(ACKNOWLEDGE) << 24); 13615b6cabb0SDon Hiatt bth0 |= extra_bytes << 20; 13625b6cabb0SDon Hiatt if (qp->s_mig_state == IB_MIG_MIGRATED) 13635b6cabb0SDon Hiatt bth1 = OPA_BTH_MIG_REQ; 13645b6cabb0SDon Hiatt hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); 13655b6cabb0SDon Hiatt } 13665b6cabb0SDon Hiatt 1367bdaf96f6SSebastian Sanchez typedef void (*hfi1_make_rc_ack)(struct hfi1_packet *packet, 13685b6cabb0SDon Hiatt struct hfi1_opa_header *opa_hdr, 13695b6cabb0SDon Hiatt u8 sc5, bool is_fecn, 13705b6cabb0SDon Hiatt u64 *pbc_flags, u32 *hwords, 13715b6cabb0SDon Hiatt u32 *nwords); 13725b6cabb0SDon Hiatt 13735b6cabb0SDon Hiatt /* We support only two types - 9B and 16B for now */ 13745b6cabb0SDon Hiatt static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = { 13755b6cabb0SDon Hiatt [HFI1_PKT_TYPE_9B] = &hfi1_make_rc_ack_9B, 13765b6cabb0SDon Hiatt [HFI1_PKT_TYPE_16B] = &hfi1_make_rc_ack_16B 13775b6cabb0SDon Hiatt }; 13785b6cabb0SDon Hiatt 13795b6cabb0SDon Hiatt /** 13805b6cabb0SDon Hiatt * hfi1_send_rc_ack - Construct an ACK packet and send it 13815b6cabb0SDon Hiatt * @qp: a pointer to the QP 13825b6cabb0SDon Hiatt * 13835b6cabb0SDon Hiatt * This is called from hfi1_rc_rcv() and handle_receive_interrupt(). 13845b6cabb0SDon Hiatt * Note that RDMA reads and atomics are handled in the 13855b6cabb0SDon Hiatt * send side QP state and send engine. 13865b6cabb0SDon Hiatt */ 1387bdaf96f6SSebastian Sanchez void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn) 13885b6cabb0SDon Hiatt { 1389bdaf96f6SSebastian Sanchez struct hfi1_ctxtdata *rcd = packet->rcd; 1390bdaf96f6SSebastian Sanchez struct rvt_qp *qp = packet->qp; 13915b6cabb0SDon Hiatt struct hfi1_ibport *ibp = rcd_to_iport(rcd); 13925b6cabb0SDon Hiatt struct hfi1_qp_priv *priv = qp->priv; 13935b6cabb0SDon Hiatt struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 13945b6cabb0SDon Hiatt u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; 13955b6cabb0SDon Hiatt u64 pbc, pbc_flags = 0; 13965b6cabb0SDon Hiatt u32 hwords = 0; 13975b6cabb0SDon Hiatt u32 nwords = 0; 13985b6cabb0SDon Hiatt u32 plen; 13995b6cabb0SDon Hiatt struct pio_buf *pbuf; 14005b6cabb0SDon Hiatt struct hfi1_opa_header opa_hdr; 14015b6cabb0SDon Hiatt 14025b6cabb0SDon Hiatt /* clear the defer count */ 14035b6cabb0SDon Hiatt qp->r_adefered = 0; 14045b6cabb0SDon Hiatt 14055b6cabb0SDon Hiatt /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ 14065b6cabb0SDon Hiatt if (qp->s_flags & RVT_S_RESP_PENDING) { 1407bdaf96f6SSebastian Sanchez hfi1_queue_rc_ack(packet, is_fecn); 14085b6cabb0SDon Hiatt return; 14095b6cabb0SDon Hiatt } 14105b6cabb0SDon Hiatt 14115b6cabb0SDon Hiatt /* Ensure s_rdma_ack_cnt changes are committed */ 14125b6cabb0SDon Hiatt if (qp->s_rdma_ack_cnt) { 1413bdaf96f6SSebastian Sanchez hfi1_queue_rc_ack(packet, is_fecn); 14145b6cabb0SDon Hiatt return; 14155b6cabb0SDon Hiatt } 14165b6cabb0SDon Hiatt 14175b6cabb0SDon Hiatt /* Don't try to send ACKs if the link isn't ACTIVE */ 14185b6cabb0SDon Hiatt if (driver_lstate(ppd) != IB_PORT_ACTIVE) 14195b6cabb0SDon Hiatt return; 14205b6cabb0SDon Hiatt 14215b6cabb0SDon Hiatt /* Make the appropriate header */ 1422bdaf96f6SSebastian Sanchez hfi1_make_rc_ack_tbl[priv->hdr_type](packet, &opa_hdr, sc5, is_fecn, 14235b6cabb0SDon Hiatt &pbc_flags, &hwords, &nwords); 14245b6cabb0SDon Hiatt 14255b6cabb0SDon Hiatt plen = 2 /* PBC */ + hwords + nwords; 14265b6cabb0SDon Hiatt pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, 14275b6cabb0SDon Hiatt sc_to_vlt(ppd->dd, sc5), plen); 14285b6cabb0SDon Hiatt pbuf = sc_buffer_alloc(rcd->sc, plen, NULL, NULL); 1429942a8993SMike Marciniszyn if (IS_ERR_OR_NULL(pbuf)) { 14305b6cabb0SDon Hiatt /* 14315b6cabb0SDon Hiatt * We have no room to send at the moment. Pass 14325b6cabb0SDon Hiatt * responsibility for sending the ACK to the send engine 14335b6cabb0SDon Hiatt * so that when enough buffer space becomes available, 14345b6cabb0SDon Hiatt * the ACK is sent ahead of other outgoing packets. 14355b6cabb0SDon Hiatt */ 1436bdaf96f6SSebastian Sanchez hfi1_queue_rc_ack(packet, is_fecn); 14375b6cabb0SDon Hiatt return; 14385b6cabb0SDon Hiatt } 14395b6cabb0SDon Hiatt trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), 14405b6cabb0SDon Hiatt &opa_hdr, ib_is_sc5(sc5)); 14415b6cabb0SDon Hiatt 14425b6cabb0SDon Hiatt /* write the pbc and data */ 14435b6cabb0SDon Hiatt ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc, 14445b6cabb0SDon Hiatt (priv->hdr_type == HFI1_PKT_TYPE_9B ? 14455b6cabb0SDon Hiatt (void *)&opa_hdr.ibh : 14465b6cabb0SDon Hiatt (void *)&opa_hdr.opah), hwords); 14475b6cabb0SDon Hiatt return; 14485b6cabb0SDon Hiatt } 14495b6cabb0SDon Hiatt 1450f48ad614SDennis Dalessandro /** 1451b126078eSKaike Wan * update_num_rd_atomic - update the qp->s_num_rd_atomic 1452b126078eSKaike Wan * @qp: the QP 1453b126078eSKaike Wan * @psn: the packet sequence number to restart at 1454b126078eSKaike Wan * @wqe: the wqe 1455b126078eSKaike Wan * 1456b126078eSKaike Wan * This is called from reset_psn() to update qp->s_num_rd_atomic 1457b126078eSKaike Wan * for the current wqe. 1458b126078eSKaike Wan * Called at interrupt level with the QP s_lock held. 1459b126078eSKaike Wan */ 1460b126078eSKaike Wan static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn, 1461b126078eSKaike Wan struct rvt_swqe *wqe) 1462b126078eSKaike Wan { 1463b126078eSKaike Wan u32 opcode = wqe->wr.opcode; 1464b126078eSKaike Wan 1465b126078eSKaike Wan if (opcode == IB_WR_RDMA_READ || 1466b126078eSKaike Wan opcode == IB_WR_ATOMIC_CMP_AND_SWP || 1467b126078eSKaike Wan opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 1468b126078eSKaike Wan qp->s_num_rd_atomic++; 1469b126078eSKaike Wan } else if (opcode == IB_WR_TID_RDMA_READ) { 1470b126078eSKaike Wan struct tid_rdma_request *req = wqe_to_tid_req(wqe); 1471b126078eSKaike Wan struct hfi1_qp_priv *priv = qp->priv; 1472b126078eSKaike Wan 1473b126078eSKaike Wan if (cmp_psn(psn, wqe->lpsn) <= 0) { 1474b126078eSKaike Wan u32 cur_seg; 1475b126078eSKaike Wan 1476b126078eSKaike Wan cur_seg = (psn - wqe->psn) / priv->pkts_ps; 1477b126078eSKaike Wan req->ack_pending = cur_seg - req->comp_seg; 1478b126078eSKaike Wan priv->pending_tid_r_segs += req->ack_pending; 1479b126078eSKaike Wan qp->s_num_rd_atomic += req->ack_pending; 1480c05fc156SKaike Wan trace_hfi1_tid_req_update_num_rd_atomic(qp, 0, 1481c05fc156SKaike Wan wqe->wr.opcode, 1482c05fc156SKaike Wan wqe->psn, 1483c05fc156SKaike Wan wqe->lpsn, 1484c05fc156SKaike Wan req); 1485b126078eSKaike Wan } else { 1486b126078eSKaike Wan priv->pending_tid_r_segs += req->total_segs; 1487b126078eSKaike Wan qp->s_num_rd_atomic += req->total_segs; 1488b126078eSKaike Wan } 1489b126078eSKaike Wan } 1490b126078eSKaike Wan } 1491b126078eSKaike Wan 1492b126078eSKaike Wan /** 1493f48ad614SDennis Dalessandro * reset_psn - reset the QP state to send starting from PSN 1494f48ad614SDennis Dalessandro * @qp: the QP 1495f48ad614SDennis Dalessandro * @psn: the packet sequence number to restart at 1496f48ad614SDennis Dalessandro * 1497f48ad614SDennis Dalessandro * This is called from hfi1_rc_rcv() to process an incoming RC ACK 1498f48ad614SDennis Dalessandro * for the given QP. 1499f48ad614SDennis Dalessandro * Called at interrupt level with the QP s_lock held. 1500f48ad614SDennis Dalessandro */ 1501f48ad614SDennis Dalessandro static void reset_psn(struct rvt_qp *qp, u32 psn) 1502f48ad614SDennis Dalessandro { 1503f48ad614SDennis Dalessandro u32 n = qp->s_acked; 1504f48ad614SDennis Dalessandro struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); 1505f48ad614SDennis Dalessandro u32 opcode; 1506b126078eSKaike Wan struct hfi1_qp_priv *priv = qp->priv; 1507f48ad614SDennis Dalessandro 150868e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 1509f48ad614SDennis Dalessandro qp->s_cur = n; 1510b126078eSKaike Wan priv->pending_tid_r_segs = 0; 15113c6cb20aSKaike Wan priv->pending_tid_w_resp = 0; 1512b126078eSKaike Wan qp->s_num_rd_atomic = 0; 1513f48ad614SDennis Dalessandro 1514f48ad614SDennis Dalessandro /* 1515f48ad614SDennis Dalessandro * If we are starting the request from the beginning, 1516f48ad614SDennis Dalessandro * let the normal send code handle initialization. 1517f48ad614SDennis Dalessandro */ 1518f48ad614SDennis Dalessandro if (cmp_psn(psn, wqe->psn) <= 0) { 1519f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST); 1520f48ad614SDennis Dalessandro goto done; 1521f48ad614SDennis Dalessandro } 1522b126078eSKaike Wan update_num_rd_atomic(qp, psn, wqe); 1523f48ad614SDennis Dalessandro 1524f48ad614SDennis Dalessandro /* Find the work request opcode corresponding to the given PSN. */ 1525f48ad614SDennis Dalessandro for (;;) { 1526f48ad614SDennis Dalessandro int diff; 1527f48ad614SDennis Dalessandro 1528f48ad614SDennis Dalessandro if (++n == qp->s_size) 1529f48ad614SDennis Dalessandro n = 0; 1530f48ad614SDennis Dalessandro if (n == qp->s_tail) 1531f48ad614SDennis Dalessandro break; 1532f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, n); 1533f48ad614SDennis Dalessandro diff = cmp_psn(psn, wqe->psn); 1534b126078eSKaike Wan if (diff < 0) { 1535b126078eSKaike Wan /* Point wqe back to the previous one*/ 1536b126078eSKaike Wan wqe = rvt_get_swqe_ptr(qp, qp->s_cur); 1537f48ad614SDennis Dalessandro break; 1538b126078eSKaike Wan } 1539f48ad614SDennis Dalessandro qp->s_cur = n; 1540f48ad614SDennis Dalessandro /* 1541f48ad614SDennis Dalessandro * If we are starting the request from the beginning, 1542f48ad614SDennis Dalessandro * let the normal send code handle initialization. 1543f48ad614SDennis Dalessandro */ 1544f48ad614SDennis Dalessandro if (diff == 0) { 1545f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST); 1546f48ad614SDennis Dalessandro goto done; 1547f48ad614SDennis Dalessandro } 1548b126078eSKaike Wan 1549b126078eSKaike Wan update_num_rd_atomic(qp, psn, wqe); 1550f48ad614SDennis Dalessandro } 1551b126078eSKaike Wan opcode = wqe->wr.opcode; 1552f48ad614SDennis Dalessandro 1553f48ad614SDennis Dalessandro /* 1554f48ad614SDennis Dalessandro * Set the state to restart in the middle of a request. 1555f48ad614SDennis Dalessandro * Don't change the s_sge, s_cur_sge, or s_cur_size. 1556f48ad614SDennis Dalessandro * See hfi1_make_rc_req(). 1557f48ad614SDennis Dalessandro */ 1558f48ad614SDennis Dalessandro switch (opcode) { 1559f48ad614SDennis Dalessandro case IB_WR_SEND: 1560f48ad614SDennis Dalessandro case IB_WR_SEND_WITH_IMM: 1561f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); 1562f48ad614SDennis Dalessandro break; 1563f48ad614SDennis Dalessandro 1564f48ad614SDennis Dalessandro case IB_WR_RDMA_WRITE: 1565f48ad614SDennis Dalessandro case IB_WR_RDMA_WRITE_WITH_IMM: 1566f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_READ_RESPONSE_LAST); 1567f48ad614SDennis Dalessandro break; 1568f48ad614SDennis Dalessandro 15693c6cb20aSKaike Wan case IB_WR_TID_RDMA_WRITE: 15703c6cb20aSKaike Wan qp->s_state = TID_OP(WRITE_RESP); 15713c6cb20aSKaike Wan break; 15723c6cb20aSKaike Wan 1573f48ad614SDennis Dalessandro case IB_WR_RDMA_READ: 1574f48ad614SDennis Dalessandro qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); 1575f48ad614SDennis Dalessandro break; 1576f48ad614SDennis Dalessandro 1577b126078eSKaike Wan case IB_WR_TID_RDMA_READ: 1578b126078eSKaike Wan qp->s_state = TID_OP(READ_RESP); 1579b126078eSKaike Wan break; 1580b126078eSKaike Wan 1581f48ad614SDennis Dalessandro default: 1582f48ad614SDennis Dalessandro /* 1583f48ad614SDennis Dalessandro * This case shouldn't happen since its only 1584f48ad614SDennis Dalessandro * one PSN per req. 1585f48ad614SDennis Dalessandro */ 1586f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST); 1587f48ad614SDennis Dalessandro } 1588f48ad614SDennis Dalessandro done: 1589a0b34f75SKaike Wan priv->s_flags &= ~HFI1_S_TID_WAIT_INTERLCK; 1590f48ad614SDennis Dalessandro qp->s_psn = psn; 1591f48ad614SDennis Dalessandro /* 1592f48ad614SDennis Dalessandro * Set RVT_S_WAIT_PSN as rc_complete() may start the timer 1593ca00c62bSDennis Dalessandro * asynchronously before the send engine can get scheduled. 1594f48ad614SDennis Dalessandro * Doing it in hfi1_make_rc_req() is too late. 1595f48ad614SDennis Dalessandro */ 1596f48ad614SDennis Dalessandro if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) && 1597f48ad614SDennis Dalessandro (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) 1598f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_WAIT_PSN; 15992e2ba09eSMike Marciniszyn qp->s_flags &= ~HFI1_S_AHG_VALID; 16003ce5daa2SKaike Wan trace_hfi1_sender_reset_psn(qp); 1601f48ad614SDennis Dalessandro } 1602f48ad614SDennis Dalessandro 1603f48ad614SDennis Dalessandro /* 1604f48ad614SDennis Dalessandro * Back up requester to resend the last un-ACKed request. 1605f48ad614SDennis Dalessandro * The QP r_lock and s_lock should be held and interrupts disabled. 1606f48ad614SDennis Dalessandro */ 160756acbbfbSVenkata Sandeep Dhanalakota void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait) 1608f48ad614SDennis Dalessandro { 160948a615dcSKaike Wan struct hfi1_qp_priv *priv = qp->priv; 1610f48ad614SDennis Dalessandro struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1611f48ad614SDennis Dalessandro struct hfi1_ibport *ibp; 1612f48ad614SDennis Dalessandro 161368e78b3dSMike Marciniszyn lockdep_assert_held(&qp->r_lock); 161468e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 16153ce5daa2SKaike Wan trace_hfi1_sender_restart_rc(qp); 1616f48ad614SDennis Dalessandro if (qp->s_retry == 0) { 1617f48ad614SDennis Dalessandro if (qp->s_mig_state == IB_MIG_ARMED) { 1618f48ad614SDennis Dalessandro hfi1_migrate_qp(qp); 1619f48ad614SDennis Dalessandro qp->s_retry = qp->s_retry_cnt; 1620f48ad614SDennis Dalessandro } else if (qp->s_last == qp->s_acked) { 162148a615dcSKaike Wan /* 162248a615dcSKaike Wan * We need special handling for the OPFN request WQEs as 162348a615dcSKaike Wan * they are not allowed to generate real user errors 162448a615dcSKaike Wan */ 162548a615dcSKaike Wan if (wqe->wr.opcode == IB_WR_OPFN) { 162648a615dcSKaike Wan struct hfi1_ibport *ibp = 162748a615dcSKaike Wan to_iport(qp->ibqp.device, qp->port_num); 162848a615dcSKaike Wan /* 162948a615dcSKaike Wan * Call opfn_conn_reply() with capcode and 163048a615dcSKaike Wan * remaining data as 0 to close out the 163148a615dcSKaike Wan * current request 163248a615dcSKaike Wan */ 163348a615dcSKaike Wan opfn_conn_reply(qp, priv->opfn.curr); 163448a615dcSKaike Wan wqe = do_rc_completion(qp, wqe, ibp); 163548a615dcSKaike Wan qp->s_flags &= ~RVT_S_WAIT_ACK; 163648a615dcSKaike Wan } else { 1637a05c9bdcSKaike Wan trace_hfi1_tid_write_sender_restart_rc(qp, 0); 1638b126078eSKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { 1639b126078eSKaike Wan struct tid_rdma_request *req; 1640b126078eSKaike Wan 1641b126078eSKaike Wan req = wqe_to_tid_req(wqe); 1642b126078eSKaike Wan hfi1_kern_exp_rcv_clear_all(req); 1643b126078eSKaike Wan hfi1_kern_clear_hw_flow(priv->rcd, qp); 1644b126078eSKaike Wan } 1645b126078eSKaike Wan 164624b11923SKaike Wan hfi1_trdma_send_complete(qp, wqe, 164748a615dcSKaike Wan IB_WC_RETRY_EXC_ERR); 1648f48ad614SDennis Dalessandro rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 164948a615dcSKaike Wan } 1650f48ad614SDennis Dalessandro return; 1651f48ad614SDennis Dalessandro } else { /* need to handle delayed completion */ 1652f48ad614SDennis Dalessandro return; 1653f48ad614SDennis Dalessandro } 1654f48ad614SDennis Dalessandro } else { 1655f48ad614SDennis Dalessandro qp->s_retry--; 1656f48ad614SDennis Dalessandro } 1657f48ad614SDennis Dalessandro 1658f48ad614SDennis Dalessandro ibp = to_iport(qp->ibqp.device, qp->port_num); 1659b126078eSKaike Wan if (wqe->wr.opcode == IB_WR_RDMA_READ || 1660b126078eSKaike Wan wqe->wr.opcode == IB_WR_TID_RDMA_READ) 1661f48ad614SDennis Dalessandro ibp->rvp.n_rc_resends++; 1662f48ad614SDennis Dalessandro else 1663f48ad614SDennis Dalessandro ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); 1664f48ad614SDennis Dalessandro 1665f48ad614SDennis Dalessandro qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | 1666f48ad614SDennis Dalessandro RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_PSN | 16673c6cb20aSKaike Wan RVT_S_WAIT_ACK | HFI1_S_WAIT_TID_RESP); 1668f48ad614SDennis Dalessandro if (wait) 1669f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_SEND_ONE; 1670f48ad614SDennis Dalessandro reset_psn(qp, psn); 1671f48ad614SDennis Dalessandro } 1672f48ad614SDennis Dalessandro 1673f48ad614SDennis Dalessandro /* 1674f48ad614SDennis Dalessandro * Set qp->s_sending_psn to the next PSN after the given one. 16753c6cb20aSKaike Wan * This would be psn+1 except when RDMA reads or TID RDMA ops 16763c6cb20aSKaike Wan * are present. 1677f48ad614SDennis Dalessandro */ 1678f48ad614SDennis Dalessandro static void reset_sending_psn(struct rvt_qp *qp, u32 psn) 1679f48ad614SDennis Dalessandro { 1680f48ad614SDennis Dalessandro struct rvt_swqe *wqe; 1681f48ad614SDennis Dalessandro u32 n = qp->s_last; 1682f48ad614SDennis Dalessandro 168368e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 1684f48ad614SDennis Dalessandro /* Find the work request corresponding to the given PSN. */ 1685f48ad614SDennis Dalessandro for (;;) { 1686f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, n); 1687f48ad614SDennis Dalessandro if (cmp_psn(psn, wqe->lpsn) <= 0) { 168824b11923SKaike Wan if (wqe->wr.opcode == IB_WR_RDMA_READ || 16893c6cb20aSKaike Wan wqe->wr.opcode == IB_WR_TID_RDMA_READ || 16903c6cb20aSKaike Wan wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) 1691f48ad614SDennis Dalessandro qp->s_sending_psn = wqe->lpsn + 1; 1692f48ad614SDennis Dalessandro else 1693f48ad614SDennis Dalessandro qp->s_sending_psn = psn + 1; 1694f48ad614SDennis Dalessandro break; 1695f48ad614SDennis Dalessandro } 1696f48ad614SDennis Dalessandro if (++n == qp->s_size) 1697f48ad614SDennis Dalessandro n = 0; 1698f48ad614SDennis Dalessandro if (n == qp->s_tail) 1699f48ad614SDennis Dalessandro break; 1700f48ad614SDennis Dalessandro } 1701f48ad614SDennis Dalessandro } 1702f48ad614SDennis Dalessandro 17034bb02e95SMike Marciniszyn /** 17044bb02e95SMike Marciniszyn * hfi1_rc_verbs_aborted - handle abort status 17054bb02e95SMike Marciniszyn * @qp: the QP 17064bb02e95SMike Marciniszyn * @opah: the opa header 17074bb02e95SMike Marciniszyn * 17084bb02e95SMike Marciniszyn * This code modifies both ACK bit in BTH[2] 17094bb02e95SMike Marciniszyn * and the s_flags to go into send one mode. 17104bb02e95SMike Marciniszyn * 17114bb02e95SMike Marciniszyn * This serves to throttle the send engine to only 17124bb02e95SMike Marciniszyn * send a single packet in the likely case the 17134bb02e95SMike Marciniszyn * a link has gone down. 17144bb02e95SMike Marciniszyn */ 17154bb02e95SMike Marciniszyn void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah) 17164bb02e95SMike Marciniszyn { 17174bb02e95SMike Marciniszyn struct ib_other_headers *ohdr = hfi1_get_rc_ohdr(opah); 17184bb02e95SMike Marciniszyn u8 opcode = ib_bth_get_opcode(ohdr); 17194bb02e95SMike Marciniszyn u32 psn; 17204bb02e95SMike Marciniszyn 17214bb02e95SMike Marciniszyn /* ignore responses */ 17224bb02e95SMike Marciniszyn if ((opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 17234bb02e95SMike Marciniszyn opcode <= OP(ATOMIC_ACKNOWLEDGE)) || 17244bb02e95SMike Marciniszyn opcode == TID_OP(READ_RESP) || 17254bb02e95SMike Marciniszyn opcode == TID_OP(WRITE_RESP)) 17264bb02e95SMike Marciniszyn return; 17274bb02e95SMike Marciniszyn 17284bb02e95SMike Marciniszyn psn = ib_bth_get_psn(ohdr) | IB_BTH_REQ_ACK; 17294bb02e95SMike Marciniszyn ohdr->bth[2] = cpu_to_be32(psn); 17304bb02e95SMike Marciniszyn qp->s_flags |= RVT_S_SEND_ONE; 17314bb02e95SMike Marciniszyn } 17324bb02e95SMike Marciniszyn 1733f48ad614SDennis Dalessandro /* 1734f48ad614SDennis Dalessandro * This should be called with the QP s_lock held and interrupts disabled. 1735f48ad614SDennis Dalessandro */ 173630e07416SDon Hiatt void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) 1737f48ad614SDennis Dalessandro { 1738261a4351SMike Marciniszyn struct ib_other_headers *ohdr; 17395b6cabb0SDon Hiatt struct hfi1_qp_priv *priv = qp->priv; 1740f48ad614SDennis Dalessandro struct rvt_swqe *wqe; 17413c6cb20aSKaike Wan u32 opcode, head, tail; 1742f48ad614SDennis Dalessandro u32 psn; 17433c6cb20aSKaike Wan struct tid_rdma_request *req; 1744f48ad614SDennis Dalessandro 174568e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 1746f9215b5eSMike Marciniszyn if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) 1747f48ad614SDennis Dalessandro return; 1748f48ad614SDennis Dalessandro 17499755f724SMike Marciniszyn ohdr = hfi1_get_rc_ohdr(opah); 1750cb427057SDon Hiatt opcode = ib_bth_get_opcode(ohdr); 175124b11923SKaike Wan if ((opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 175224b11923SKaike Wan opcode <= OP(ATOMIC_ACKNOWLEDGE)) || 17533c6cb20aSKaike Wan opcode == TID_OP(READ_RESP) || 17543c6cb20aSKaike Wan opcode == TID_OP(WRITE_RESP)) { 1755f48ad614SDennis Dalessandro WARN_ON(!qp->s_rdma_ack_cnt); 1756f48ad614SDennis Dalessandro qp->s_rdma_ack_cnt--; 1757f48ad614SDennis Dalessandro return; 1758f48ad614SDennis Dalessandro } 1759f48ad614SDennis Dalessandro 17607dafbab3SDon Hiatt psn = ib_bth_get_psn(ohdr); 17613c6cb20aSKaike Wan /* 17623c6cb20aSKaike Wan * Don't attempt to reset the sending PSN for packets in the 17633c6cb20aSKaike Wan * KDETH PSN space since the PSN does not match anything. 17643c6cb20aSKaike Wan */ 17653c6cb20aSKaike Wan if (opcode != TID_OP(WRITE_DATA) && 17663c6cb20aSKaike Wan opcode != TID_OP(WRITE_DATA_LAST) && 17673c6cb20aSKaike Wan opcode != TID_OP(ACK) && opcode != TID_OP(RESYNC)) 1768f48ad614SDennis Dalessandro reset_sending_psn(qp, psn); 1769f48ad614SDennis Dalessandro 17703c6cb20aSKaike Wan /* Handle TID RDMA WRITE packets differently */ 17713c6cb20aSKaike Wan if (opcode >= TID_OP(WRITE_REQ) && 17723c6cb20aSKaike Wan opcode <= TID_OP(WRITE_DATA_LAST)) { 17733c6cb20aSKaike Wan head = priv->s_tid_head; 17743c6cb20aSKaike Wan tail = priv->s_tid_cur; 17753c6cb20aSKaike Wan /* 17763c6cb20aSKaike Wan * s_tid_cur is set to s_tid_head in the case, where 17773c6cb20aSKaike Wan * a new TID RDMA request is being started and all 17783c6cb20aSKaike Wan * previous ones have been completed. 17793c6cb20aSKaike Wan * Therefore, we need to do a secondary check in order 17803c6cb20aSKaike Wan * to properly determine whether we should start the 17813c6cb20aSKaike Wan * RC timer. 17823c6cb20aSKaike Wan */ 17833c6cb20aSKaike Wan wqe = rvt_get_swqe_ptr(qp, tail); 17843c6cb20aSKaike Wan req = wqe_to_tid_req(wqe); 17853c6cb20aSKaike Wan if (head == tail && req->comp_seg < req->total_segs) { 17863c6cb20aSKaike Wan if (tail == 0) 17873c6cb20aSKaike Wan tail = qp->s_size - 1; 17883c6cb20aSKaike Wan else 17893c6cb20aSKaike Wan tail -= 1; 17903c6cb20aSKaike Wan } 17913c6cb20aSKaike Wan } else { 17923c6cb20aSKaike Wan head = qp->s_tail; 17933c6cb20aSKaike Wan tail = qp->s_acked; 17943c6cb20aSKaike Wan } 17953c6cb20aSKaike Wan 1796f48ad614SDennis Dalessandro /* 1797f48ad614SDennis Dalessandro * Start timer after a packet requesting an ACK has been sent and 1798f48ad614SDennis Dalessandro * there are still requests that haven't been acked. 1799f48ad614SDennis Dalessandro */ 18003c6cb20aSKaike Wan if ((psn & IB_BTH_REQ_ACK) && tail != head && 18013c6cb20aSKaike Wan opcode != TID_OP(WRITE_DATA) && opcode != TID_OP(WRITE_DATA_LAST) && 18023c6cb20aSKaike Wan opcode != TID_OP(RESYNC) && 1803f48ad614SDennis Dalessandro !(qp->s_flags & 1804f48ad614SDennis Dalessandro (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) && 180524b11923SKaike Wan (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { 180624b11923SKaike Wan if (opcode == TID_OP(READ_REQ)) 180724b11923SKaike Wan rvt_add_retry_timer_ext(qp, priv->timeout_shift); 180824b11923SKaike Wan else 180956acbbfbSVenkata Sandeep Dhanalakota rvt_add_retry_timer(qp); 181024b11923SKaike Wan } 1811f48ad614SDennis Dalessandro 18123c6cb20aSKaike Wan /* Start TID RDMA ACK timer */ 18133c6cb20aSKaike Wan if ((opcode == TID_OP(WRITE_DATA) || 18143c6cb20aSKaike Wan opcode == TID_OP(WRITE_DATA_LAST) || 18153c6cb20aSKaike Wan opcode == TID_OP(RESYNC)) && 18163c6cb20aSKaike Wan (psn & IB_BTH_REQ_ACK) && 18173c6cb20aSKaike Wan !(priv->s_flags & HFI1_S_TID_RETRY_TIMER) && 18183c6cb20aSKaike Wan (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { 18193c6cb20aSKaike Wan /* 18203c6cb20aSKaike Wan * The TID RDMA ACK packet could be received before this 18213c6cb20aSKaike Wan * function is called. Therefore, add the timer only if TID 18223c6cb20aSKaike Wan * RDMA ACK packets are actually pending. 18233c6cb20aSKaike Wan */ 18243c6cb20aSKaike Wan wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 18253c6cb20aSKaike Wan req = wqe_to_tid_req(wqe); 18263c6cb20aSKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && 18273c6cb20aSKaike Wan req->ack_seg < req->cur_seg) 18283c6cb20aSKaike Wan hfi1_add_tid_retry_timer(qp); 18293c6cb20aSKaike Wan } 18303c6cb20aSKaike Wan 1831f48ad614SDennis Dalessandro while (qp->s_last != qp->s_acked) { 1832f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_last); 1833f48ad614SDennis Dalessandro if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && 1834f48ad614SDennis Dalessandro cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) 1835f48ad614SDennis Dalessandro break; 183624b11923SKaike Wan trdma_clean_swqe(qp, wqe); 18374a9ceb7dSMike Marciniszyn trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); 18384a9ceb7dSMike Marciniszyn rvt_qp_complete_swqe(qp, 183943a474aaSMike Marciniszyn wqe, 184043a474aaSMike Marciniszyn ib_hfi1_wc_opcode[wqe->wr.opcode], 184143a474aaSMike Marciniszyn IB_WC_SUCCESS); 1842f48ad614SDennis Dalessandro } 1843f48ad614SDennis Dalessandro /* 1844f48ad614SDennis Dalessandro * If we were waiting for sends to complete before re-sending, 1845f48ad614SDennis Dalessandro * and they are now complete, restart sending. 1846f48ad614SDennis Dalessandro */ 1847462b6b21SSebastian Sanchez trace_hfi1_sendcomplete(qp, psn); 1848f48ad614SDennis Dalessandro if (qp->s_flags & RVT_S_WAIT_PSN && 1849f48ad614SDennis Dalessandro cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 1850f48ad614SDennis Dalessandro qp->s_flags &= ~RVT_S_WAIT_PSN; 1851f48ad614SDennis Dalessandro qp->s_sending_psn = qp->s_psn; 1852f48ad614SDennis Dalessandro qp->s_sending_hpsn = qp->s_psn - 1; 1853f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 1854f48ad614SDennis Dalessandro } 1855f48ad614SDennis Dalessandro } 1856f48ad614SDennis Dalessandro 1857f48ad614SDennis Dalessandro static inline void update_last_psn(struct rvt_qp *qp, u32 psn) 1858f48ad614SDennis Dalessandro { 1859f48ad614SDennis Dalessandro qp->s_last_psn = psn; 1860f48ad614SDennis Dalessandro } 1861f48ad614SDennis Dalessandro 1862f48ad614SDennis Dalessandro /* 1863f48ad614SDennis Dalessandro * Generate a SWQE completion. 1864f48ad614SDennis Dalessandro * This is similar to hfi1_send_complete but has to check to be sure 1865f48ad614SDennis Dalessandro * that the SGEs are not being referenced if the SWQE is being resent. 1866f48ad614SDennis Dalessandro */ 1867385156c5SKaike Wan struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, 1868f48ad614SDennis Dalessandro struct rvt_swqe *wqe, 1869f48ad614SDennis Dalessandro struct hfi1_ibport *ibp) 1870f48ad614SDennis Dalessandro { 1871a0b34f75SKaike Wan struct hfi1_qp_priv *priv = qp->priv; 1872a0b34f75SKaike Wan 187368e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 1874f48ad614SDennis Dalessandro /* 1875f48ad614SDennis Dalessandro * Don't decrement refcount and don't generate a 1876f48ad614SDennis Dalessandro * completion if the SWQE is being resent until the send 1877f48ad614SDennis Dalessandro * is finished. 1878f48ad614SDennis Dalessandro */ 18793ce5daa2SKaike Wan trace_hfi1_rc_completion(qp, wqe->lpsn); 1880f48ad614SDennis Dalessandro if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || 1881f48ad614SDennis Dalessandro cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { 188224b11923SKaike Wan trdma_clean_swqe(qp, wqe); 18834a9ceb7dSMike Marciniszyn trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); 18844a9ceb7dSMike Marciniszyn rvt_qp_complete_swqe(qp, 188543a474aaSMike Marciniszyn wqe, 188643a474aaSMike Marciniszyn ib_hfi1_wc_opcode[wqe->wr.opcode], 188743a474aaSMike Marciniszyn IB_WC_SUCCESS); 1888f48ad614SDennis Dalessandro } else { 1889f48ad614SDennis Dalessandro struct hfi1_pportdata *ppd = ppd_from_ibp(ibp); 1890f48ad614SDennis Dalessandro 1891f48ad614SDennis Dalessandro this_cpu_inc(*ibp->rvp.rc_delayed_comp); 1892f48ad614SDennis Dalessandro /* 1893f48ad614SDennis Dalessandro * If send progress not running attempt to progress 1894f48ad614SDennis Dalessandro * SDMA queue. 1895f48ad614SDennis Dalessandro */ 1896f48ad614SDennis Dalessandro if (ppd->dd->flags & HFI1_HAS_SEND_DMA) { 1897f48ad614SDennis Dalessandro struct sdma_engine *engine; 1898d8966fcdSDasaratharaman Chandramouli u8 sl = rdma_ah_get_sl(&qp->remote_ah_attr); 1899f48ad614SDennis Dalessandro u8 sc5; 1900f48ad614SDennis Dalessandro 1901f48ad614SDennis Dalessandro /* For now use sc to find engine */ 1902d8966fcdSDasaratharaman Chandramouli sc5 = ibp->sl_to_sc[sl]; 1903f48ad614SDennis Dalessandro engine = qp_to_sdma_engine(qp, sc5); 1904f48ad614SDennis Dalessandro sdma_engine_progress_schedule(engine); 1905f48ad614SDennis Dalessandro } 1906f48ad614SDennis Dalessandro } 1907f48ad614SDennis Dalessandro 1908f48ad614SDennis Dalessandro qp->s_retry = qp->s_retry_cnt; 19093c6cb20aSKaike Wan /* 19103c6cb20aSKaike Wan * Don't update the last PSN if the request being completed is 19113c6cb20aSKaike Wan * a TID RDMA WRITE request. 19123c6cb20aSKaike Wan * Completion of the TID RDMA WRITE requests are done by the 19133c6cb20aSKaike Wan * TID RDMA ACKs and as such could be for a request that has 19143c6cb20aSKaike Wan * already been ACKed as far as the IB state machine is 19153c6cb20aSKaike Wan * concerned. 19163c6cb20aSKaike Wan */ 19173c6cb20aSKaike Wan if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE) 1918f48ad614SDennis Dalessandro update_last_psn(qp, wqe->lpsn); 1919f48ad614SDennis Dalessandro 1920f48ad614SDennis Dalessandro /* 1921f48ad614SDennis Dalessandro * If we are completing a request which is in the process of 1922f48ad614SDennis Dalessandro * being resent, we can stop re-sending it since we know the 1923f48ad614SDennis Dalessandro * responder has already seen it. 1924f48ad614SDennis Dalessandro */ 1925f48ad614SDennis Dalessandro if (qp->s_acked == qp->s_cur) { 1926f48ad614SDennis Dalessandro if (++qp->s_cur >= qp->s_size) 1927f48ad614SDennis Dalessandro qp->s_cur = 0; 1928f48ad614SDennis Dalessandro qp->s_acked = qp->s_cur; 1929f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_cur); 1930f48ad614SDennis Dalessandro if (qp->s_acked != qp->s_tail) { 1931f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST); 1932f48ad614SDennis Dalessandro qp->s_psn = wqe->psn; 1933f48ad614SDennis Dalessandro } 1934f48ad614SDennis Dalessandro } else { 1935f48ad614SDennis Dalessandro if (++qp->s_acked >= qp->s_size) 1936f48ad614SDennis Dalessandro qp->s_acked = 0; 1937f48ad614SDennis Dalessandro if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) 1938f48ad614SDennis Dalessandro qp->s_draining = 0; 1939f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 1940f48ad614SDennis Dalessandro } 1941a0b34f75SKaike Wan if (priv->s_flags & HFI1_S_TID_WAIT_INTERLCK) { 1942a0b34f75SKaike Wan priv->s_flags &= ~HFI1_S_TID_WAIT_INTERLCK; 1943a0b34f75SKaike Wan hfi1_schedule_send(qp); 1944a0b34f75SKaike Wan } 1945f48ad614SDennis Dalessandro return wqe; 1946f48ad614SDennis Dalessandro } 1947f48ad614SDennis Dalessandro 19483c6cb20aSKaike Wan static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd) 19493c6cb20aSKaike Wan { 19503c6cb20aSKaike Wan /* Retry this request. */ 19513c6cb20aSKaike Wan if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { 19523c6cb20aSKaike Wan qp->r_flags |= RVT_R_RDMAR_SEQ; 19533c6cb20aSKaike Wan hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); 19543c6cb20aSKaike Wan if (list_empty(&qp->rspwait)) { 19553c6cb20aSKaike Wan qp->r_flags |= RVT_R_RSP_SEND; 19563c6cb20aSKaike Wan rvt_get_qp(qp); 19573c6cb20aSKaike Wan list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 19583c6cb20aSKaike Wan } 19593c6cb20aSKaike Wan } 19603c6cb20aSKaike Wan } 19613c6cb20aSKaike Wan 19623c6cb20aSKaike Wan /** 19633c6cb20aSKaike Wan * update_qp_retry_state - Update qp retry state. 19643c6cb20aSKaike Wan * @qp: the QP 19653c6cb20aSKaike Wan * @psn: the packet sequence number of the TID RDMA WRITE RESP. 19663c6cb20aSKaike Wan * @spsn: The start psn for the given TID RDMA WRITE swqe. 19673c6cb20aSKaike Wan * @lpsn: The last psn for the given TID RDMA WRITE swqe. 19683c6cb20aSKaike Wan * 19693c6cb20aSKaike Wan * This function is called to update the qp retry state upon 19703c6cb20aSKaike Wan * receiving a TID WRITE RESP after the qp is scheduled to retry 19713c6cb20aSKaike Wan * a request. 19723c6cb20aSKaike Wan */ 19733c6cb20aSKaike Wan static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn, 19743c6cb20aSKaike Wan u32 lpsn) 19753c6cb20aSKaike Wan { 19763c6cb20aSKaike Wan struct hfi1_qp_priv *qpriv = qp->priv; 19773c6cb20aSKaike Wan 19783c6cb20aSKaike Wan qp->s_psn = psn + 1; 19793c6cb20aSKaike Wan /* 19803c6cb20aSKaike Wan * If this is the first TID RDMA WRITE RESP packet for the current 19813c6cb20aSKaike Wan * request, change the s_state so that the retry will be processed 19823c6cb20aSKaike Wan * correctly. Similarly, if this is the last TID RDMA WRITE RESP 19833c6cb20aSKaike Wan * packet, change the s_state and advance the s_cur. 19843c6cb20aSKaike Wan */ 19853c6cb20aSKaike Wan if (cmp_psn(psn, lpsn) >= 0) { 19863c6cb20aSKaike Wan qp->s_cur = qpriv->s_tid_cur + 1; 19873c6cb20aSKaike Wan if (qp->s_cur >= qp->s_size) 19883c6cb20aSKaike Wan qp->s_cur = 0; 19893c6cb20aSKaike Wan qp->s_state = TID_OP(WRITE_REQ); 19903c6cb20aSKaike Wan } else if (!cmp_psn(psn, spsn)) { 19913c6cb20aSKaike Wan qp->s_cur = qpriv->s_tid_cur; 19923c6cb20aSKaike Wan qp->s_state = TID_OP(WRITE_RESP); 19933c6cb20aSKaike Wan } 19943c6cb20aSKaike Wan } 19953c6cb20aSKaike Wan 1996f48ad614SDennis Dalessandro /** 1997f48ad614SDennis Dalessandro * do_rc_ack - process an incoming RC ACK 1998f48ad614SDennis Dalessandro * @qp: the QP the ACK came in on 1999f48ad614SDennis Dalessandro * @psn: the packet sequence number of the ACK 2000f48ad614SDennis Dalessandro * @opcode: the opcode of the request that resulted in the ACK 2001f48ad614SDennis Dalessandro * 2002f48ad614SDennis Dalessandro * This is called from rc_rcv_resp() to process an incoming RC ACK 2003f48ad614SDennis Dalessandro * for the given QP. 2004f48ad614SDennis Dalessandro * May be called at interrupt level, with the QP s_lock held. 2005f48ad614SDennis Dalessandro * Returns 1 if OK, 0 if current operation should be aborted (NAK). 2006f48ad614SDennis Dalessandro */ 2007385156c5SKaike Wan int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, 2008f48ad614SDennis Dalessandro u64 val, struct hfi1_ctxtdata *rcd) 2009f48ad614SDennis Dalessandro { 2010f48ad614SDennis Dalessandro struct hfi1_ibport *ibp; 2011f48ad614SDennis Dalessandro enum ib_wc_status status; 201224b11923SKaike Wan struct hfi1_qp_priv *qpriv = qp->priv; 2013f48ad614SDennis Dalessandro struct rvt_swqe *wqe; 2014f48ad614SDennis Dalessandro int ret = 0; 2015f48ad614SDennis Dalessandro u32 ack_psn; 2016f48ad614SDennis Dalessandro int diff; 20173c6cb20aSKaike Wan struct rvt_dev_info *rdi; 2018f48ad614SDennis Dalessandro 201968e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 2020f48ad614SDennis Dalessandro /* 2021f48ad614SDennis Dalessandro * Note that NAKs implicitly ACK outstanding SEND and RDMA write 2022f48ad614SDennis Dalessandro * requests and implicitly NAK RDMA read and atomic requests issued 2023f48ad614SDennis Dalessandro * before the NAK'ed request. The MSN won't include the NAK'ed 2024f48ad614SDennis Dalessandro * request but will include an ACK'ed request(s). 2025f48ad614SDennis Dalessandro */ 2026f48ad614SDennis Dalessandro ack_psn = psn; 2027832666c1SDon Hiatt if (aeth >> IB_AETH_NAK_SHIFT) 2028f48ad614SDennis Dalessandro ack_psn--; 2029f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 2030f3e862cbSSebastian Sanchez ibp = rcd_to_iport(rcd); 2031f48ad614SDennis Dalessandro 2032f48ad614SDennis Dalessandro /* 2033f48ad614SDennis Dalessandro * The MSN might be for a later WQE than the PSN indicates so 2034f48ad614SDennis Dalessandro * only complete WQEs that the PSN finishes. 2035f48ad614SDennis Dalessandro */ 2036f48ad614SDennis Dalessandro while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) { 2037f48ad614SDennis Dalessandro /* 2038f48ad614SDennis Dalessandro * RDMA_READ_RESPONSE_ONLY is a special case since 2039f48ad614SDennis Dalessandro * we want to generate completion events for everything 2040f48ad614SDennis Dalessandro * before the RDMA read, copy the data, then generate 2041f48ad614SDennis Dalessandro * the completion for the read. 2042f48ad614SDennis Dalessandro */ 2043f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_RDMA_READ && 2044f48ad614SDennis Dalessandro opcode == OP(RDMA_READ_RESPONSE_ONLY) && 2045f48ad614SDennis Dalessandro diff == 0) { 2046f48ad614SDennis Dalessandro ret = 1; 2047f48ad614SDennis Dalessandro goto bail_stop; 2048f48ad614SDennis Dalessandro } 2049f48ad614SDennis Dalessandro /* 2050f48ad614SDennis Dalessandro * If this request is a RDMA read or atomic, and the ACK is 2051f48ad614SDennis Dalessandro * for a later operation, this ACK NAKs the RDMA read or 2052f48ad614SDennis Dalessandro * atomic. In other words, only a RDMA_READ_LAST or ONLY 2053f48ad614SDennis Dalessandro * can ACK a RDMA read and likewise for atomic ops. Note 2054f48ad614SDennis Dalessandro * that the NAK case can only happen if relaxed ordering is 2055f48ad614SDennis Dalessandro * used and requests are sent after an RDMA read or atomic 2056f48ad614SDennis Dalessandro * is sent but before the response is received. 2057f48ad614SDennis Dalessandro */ 2058f48ad614SDennis Dalessandro if ((wqe->wr.opcode == IB_WR_RDMA_READ && 2059f48ad614SDennis Dalessandro (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) || 206024b11923SKaike Wan (wqe->wr.opcode == IB_WR_TID_RDMA_READ && 206124b11923SKaike Wan (opcode != TID_OP(READ_RESP) || diff != 0)) || 2062f48ad614SDennis Dalessandro ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2063f48ad614SDennis Dalessandro wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && 20643c6cb20aSKaike Wan (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0)) || 20653c6cb20aSKaike Wan (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && 20663c6cb20aSKaike Wan (delta_psn(psn, qp->s_last_psn) != 1))) { 20673c6cb20aSKaike Wan set_restart_qp(qp, rcd); 2068f48ad614SDennis Dalessandro /* 2069f48ad614SDennis Dalessandro * No need to process the ACK/NAK since we are 2070f48ad614SDennis Dalessandro * restarting an earlier request. 2071f48ad614SDennis Dalessandro */ 2072f48ad614SDennis Dalessandro goto bail_stop; 2073f48ad614SDennis Dalessandro } 2074f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2075f48ad614SDennis Dalessandro wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { 2076f48ad614SDennis Dalessandro u64 *vaddr = wqe->sg_list[0].vaddr; 2077f48ad614SDennis Dalessandro *vaddr = val; 2078f48ad614SDennis Dalessandro } 207948a615dcSKaike Wan if (wqe->wr.opcode == IB_WR_OPFN) 208048a615dcSKaike Wan opfn_conn_reply(qp, val); 208148a615dcSKaike Wan 2082f48ad614SDennis Dalessandro if (qp->s_num_rd_atomic && 2083f48ad614SDennis Dalessandro (wqe->wr.opcode == IB_WR_RDMA_READ || 2084f48ad614SDennis Dalessandro wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2085f48ad614SDennis Dalessandro wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { 2086f48ad614SDennis Dalessandro qp->s_num_rd_atomic--; 2087f48ad614SDennis Dalessandro /* Restart sending task if fence is complete */ 2088f48ad614SDennis Dalessandro if ((qp->s_flags & RVT_S_WAIT_FENCE) && 2089f48ad614SDennis Dalessandro !qp->s_num_rd_atomic) { 2090f48ad614SDennis Dalessandro qp->s_flags &= ~(RVT_S_WAIT_FENCE | 2091f48ad614SDennis Dalessandro RVT_S_WAIT_ACK); 2092f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 2093f48ad614SDennis Dalessandro } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { 2094f48ad614SDennis Dalessandro qp->s_flags &= ~(RVT_S_WAIT_RDMAR | 2095f48ad614SDennis Dalessandro RVT_S_WAIT_ACK); 2096f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 2097f48ad614SDennis Dalessandro } 2098f48ad614SDennis Dalessandro } 20993c6cb20aSKaike Wan 21003c6cb20aSKaike Wan /* 21013c6cb20aSKaike Wan * TID RDMA WRITE requests will be completed by the TID RDMA 21023c6cb20aSKaike Wan * ACK packet handler (see tid_rdma.c). 21033c6cb20aSKaike Wan */ 21043c6cb20aSKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) 21053c6cb20aSKaike Wan break; 21063c6cb20aSKaike Wan 2107f48ad614SDennis Dalessandro wqe = do_rc_completion(qp, wqe, ibp); 2108f48ad614SDennis Dalessandro if (qp->s_acked == qp->s_tail) 2109f48ad614SDennis Dalessandro break; 2110f48ad614SDennis Dalessandro } 2111f48ad614SDennis Dalessandro 21123ce5daa2SKaike Wan trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); 21133ce5daa2SKaike Wan trace_hfi1_sender_do_rc_ack(qp); 2114832666c1SDon Hiatt switch (aeth >> IB_AETH_NAK_SHIFT) { 2115f48ad614SDennis Dalessandro case 0: /* ACK */ 2116f48ad614SDennis Dalessandro this_cpu_inc(*ibp->rvp.rc_acks); 211724b11923SKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) { 211824b11923SKaike Wan if (wqe_to_tid_req(wqe)->ack_pending) 211924b11923SKaike Wan rvt_mod_retry_timer_ext(qp, 212024b11923SKaike Wan qpriv->timeout_shift); 212124b11923SKaike Wan else 212224b11923SKaike Wan rvt_stop_rc_timers(qp); 212324b11923SKaike Wan } else if (qp->s_acked != qp->s_tail) { 21243c6cb20aSKaike Wan struct rvt_swqe *__w = NULL; 21253c6cb20aSKaike Wan 21263c6cb20aSKaike Wan if (qpriv->s_tid_cur != HFI1_QP_WQE_INVALID) 21273c6cb20aSKaike Wan __w = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); 21283c6cb20aSKaike Wan 21293c6cb20aSKaike Wan /* 21303c6cb20aSKaike Wan * Stop timers if we've received all of the TID RDMA 21313c6cb20aSKaike Wan * WRITE * responses. 21323c6cb20aSKaike Wan */ 21333c6cb20aSKaike Wan if (__w && __w->wr.opcode == IB_WR_TID_RDMA_WRITE && 21343c6cb20aSKaike Wan opcode == TID_OP(WRITE_RESP)) { 21353c6cb20aSKaike Wan /* 21363c6cb20aSKaike Wan * Normally, the loop above would correctly 21373c6cb20aSKaike Wan * process all WQEs from s_acked onward and 21383c6cb20aSKaike Wan * either complete them or check for correct 21393c6cb20aSKaike Wan * PSN sequencing. 21403c6cb20aSKaike Wan * However, for TID RDMA, due to pipelining, 21413c6cb20aSKaike Wan * the response may not be for the request at 21423c6cb20aSKaike Wan * s_acked so the above look would just be 21433c6cb20aSKaike Wan * skipped. This does not allow for checking 21443c6cb20aSKaike Wan * the PSN sequencing. It has to be done 21453c6cb20aSKaike Wan * separately. 21463c6cb20aSKaike Wan */ 21473c6cb20aSKaike Wan if (cmp_psn(psn, qp->s_last_psn + 1)) { 21483c6cb20aSKaike Wan set_restart_qp(qp, rcd); 21493c6cb20aSKaike Wan goto bail_stop; 21503c6cb20aSKaike Wan } 21513c6cb20aSKaike Wan /* 21523c6cb20aSKaike Wan * If the psn is being resent, stop the 21533c6cb20aSKaike Wan * resending. 21543c6cb20aSKaike Wan */ 21553c6cb20aSKaike Wan if (qp->s_cur != qp->s_tail && 21563c6cb20aSKaike Wan cmp_psn(qp->s_psn, psn) <= 0) 21573c6cb20aSKaike Wan update_qp_retry_state(qp, psn, 21583c6cb20aSKaike Wan __w->psn, 21593c6cb20aSKaike Wan __w->lpsn); 21603c6cb20aSKaike Wan else if (--qpriv->pending_tid_w_resp) 21613c6cb20aSKaike Wan rvt_mod_retry_timer(qp); 21623c6cb20aSKaike Wan else 21633c6cb20aSKaike Wan rvt_stop_rc_timers(qp); 21643c6cb20aSKaike Wan } else { 2165f48ad614SDennis Dalessandro /* 2166f48ad614SDennis Dalessandro * We are expecting more ACKs so 2167f48ad614SDennis Dalessandro * mod the retry timer. 2168f48ad614SDennis Dalessandro */ 216956acbbfbSVenkata Sandeep Dhanalakota rvt_mod_retry_timer(qp); 2170f48ad614SDennis Dalessandro /* 21713c6cb20aSKaike Wan * We can stop re-sending the earlier packets 21723c6cb20aSKaike Wan * and continue with the next packet the 21733c6cb20aSKaike Wan * receiver wants. 2174f48ad614SDennis Dalessandro */ 2175f48ad614SDennis Dalessandro if (cmp_psn(qp->s_psn, psn) <= 0) 2176f48ad614SDennis Dalessandro reset_psn(qp, psn + 1); 21773c6cb20aSKaike Wan } 2178f48ad614SDennis Dalessandro } else { 2179f48ad614SDennis Dalessandro /* No more acks - kill all timers */ 218056acbbfbSVenkata Sandeep Dhanalakota rvt_stop_rc_timers(qp); 2181f48ad614SDennis Dalessandro if (cmp_psn(qp->s_psn, psn) <= 0) { 2182f48ad614SDennis Dalessandro qp->s_state = OP(SEND_LAST); 2183f48ad614SDennis Dalessandro qp->s_psn = psn + 1; 2184f48ad614SDennis Dalessandro } 2185f48ad614SDennis Dalessandro } 2186f48ad614SDennis Dalessandro if (qp->s_flags & RVT_S_WAIT_ACK) { 2187f48ad614SDennis Dalessandro qp->s_flags &= ~RVT_S_WAIT_ACK; 2188f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 2189f48ad614SDennis Dalessandro } 2190696513e8SBrian Welty rvt_get_credit(qp, aeth); 2191f48ad614SDennis Dalessandro qp->s_rnr_retry = qp->s_rnr_retry_cnt; 2192f48ad614SDennis Dalessandro qp->s_retry = qp->s_retry_cnt; 21933c6cb20aSKaike Wan /* 21943c6cb20aSKaike Wan * If the current request is a TID RDMA WRITE request and the 21953c6cb20aSKaike Wan * response is not a TID RDMA WRITE RESP packet, s_last_psn 21963c6cb20aSKaike Wan * can't be advanced. 21973c6cb20aSKaike Wan */ 21983c6cb20aSKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE && 21993c6cb20aSKaike Wan opcode != TID_OP(WRITE_RESP) && 22003c6cb20aSKaike Wan cmp_psn(psn, wqe->psn) >= 0) 22013c6cb20aSKaike Wan return 1; 2202f48ad614SDennis Dalessandro update_last_psn(qp, psn); 2203f48ad614SDennis Dalessandro return 1; 2204f48ad614SDennis Dalessandro 2205f48ad614SDennis Dalessandro case 1: /* RNR NAK */ 2206f48ad614SDennis Dalessandro ibp->rvp.n_rnr_naks++; 2207f48ad614SDennis Dalessandro if (qp->s_acked == qp->s_tail) 2208f48ad614SDennis Dalessandro goto bail_stop; 2209f48ad614SDennis Dalessandro if (qp->s_flags & RVT_S_WAIT_RNR) 2210f48ad614SDennis Dalessandro goto bail_stop; 22113c6cb20aSKaike Wan rdi = ib_to_rvt(qp->ibqp.device); 2212ce8e8087SKaike Wan if (!(rdi->post_parms[wqe->wr.opcode].flags & 2213ce8e8087SKaike Wan RVT_OPERATION_IGN_RNR_CNT)) { 2214ce8e8087SKaike Wan if (qp->s_rnr_retry == 0) { 2215f48ad614SDennis Dalessandro status = IB_WC_RNR_RETRY_EXC_ERR; 2216f48ad614SDennis Dalessandro goto class_b; 2217f48ad614SDennis Dalessandro } 22183c6cb20aSKaike Wan if (qp->s_rnr_retry_cnt < 7 && qp->s_rnr_retry_cnt > 0) 2219f48ad614SDennis Dalessandro qp->s_rnr_retry--; 2220ce8e8087SKaike Wan } 2221f48ad614SDennis Dalessandro 22223c6cb20aSKaike Wan /* 22233c6cb20aSKaike Wan * The last valid PSN is the previous PSN. For TID RDMA WRITE 22243c6cb20aSKaike Wan * request, s_last_psn should be incremented only when a TID 22253c6cb20aSKaike Wan * RDMA WRITE RESP is received to avoid skipping lost TID RDMA 22263c6cb20aSKaike Wan * WRITE RESP packets. 22273c6cb20aSKaike Wan */ 22283c6cb20aSKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) { 22293c6cb20aSKaike Wan reset_psn(qp, qp->s_last_psn + 1); 22303c6cb20aSKaike Wan } else { 2231f48ad614SDennis Dalessandro update_last_psn(qp, psn - 1); 22323c6cb20aSKaike Wan reset_psn(qp, psn); 22333c6cb20aSKaike Wan } 2234f48ad614SDennis Dalessandro 2235f48ad614SDennis Dalessandro ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); 2236f48ad614SDennis Dalessandro qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); 223756acbbfbSVenkata Sandeep Dhanalakota rvt_stop_rc_timers(qp); 223856acbbfbSVenkata Sandeep Dhanalakota rvt_add_rnr_timer(qp, aeth); 2239f48ad614SDennis Dalessandro return 0; 2240f48ad614SDennis Dalessandro 2241f48ad614SDennis Dalessandro case 3: /* NAK */ 2242f48ad614SDennis Dalessandro if (qp->s_acked == qp->s_tail) 2243f48ad614SDennis Dalessandro goto bail_stop; 2244f48ad614SDennis Dalessandro /* The last valid PSN is the previous PSN. */ 2245f48ad614SDennis Dalessandro update_last_psn(qp, psn - 1); 2246832666c1SDon Hiatt switch ((aeth >> IB_AETH_CREDIT_SHIFT) & 2247832666c1SDon Hiatt IB_AETH_CREDIT_MASK) { 2248f48ad614SDennis Dalessandro case 0: /* PSN sequence error */ 2249f48ad614SDennis Dalessandro ibp->rvp.n_seq_naks++; 2250f48ad614SDennis Dalessandro /* 2251f48ad614SDennis Dalessandro * Back up to the responder's expected PSN. 2252f48ad614SDennis Dalessandro * Note that we might get a NAK in the middle of an 2253f48ad614SDennis Dalessandro * RDMA READ response which terminates the RDMA 2254f48ad614SDennis Dalessandro * READ. 2255f48ad614SDennis Dalessandro */ 225656acbbfbSVenkata Sandeep Dhanalakota hfi1_restart_rc(qp, psn, 0); 2257f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 2258f48ad614SDennis Dalessandro break; 2259f48ad614SDennis Dalessandro 2260f48ad614SDennis Dalessandro case 1: /* Invalid Request */ 2261f48ad614SDennis Dalessandro status = IB_WC_REM_INV_REQ_ERR; 2262f48ad614SDennis Dalessandro ibp->rvp.n_other_naks++; 2263f48ad614SDennis Dalessandro goto class_b; 2264f48ad614SDennis Dalessandro 2265f48ad614SDennis Dalessandro case 2: /* Remote Access Error */ 2266f48ad614SDennis Dalessandro status = IB_WC_REM_ACCESS_ERR; 2267f48ad614SDennis Dalessandro ibp->rvp.n_other_naks++; 2268f48ad614SDennis Dalessandro goto class_b; 2269f48ad614SDennis Dalessandro 2270f48ad614SDennis Dalessandro case 3: /* Remote Operation Error */ 2271f48ad614SDennis Dalessandro status = IB_WC_REM_OP_ERR; 2272f48ad614SDennis Dalessandro ibp->rvp.n_other_naks++; 2273f48ad614SDennis Dalessandro class_b: 2274f48ad614SDennis Dalessandro if (qp->s_last == qp->s_acked) { 227524b11923SKaike Wan if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) 227624b11923SKaike Wan hfi1_kern_read_tid_flow_free(qp); 227724b11923SKaike Wan 227824b11923SKaike Wan hfi1_trdma_send_complete(qp, wqe, status); 2279f48ad614SDennis Dalessandro rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 2280f48ad614SDennis Dalessandro } 2281f48ad614SDennis Dalessandro break; 2282f48ad614SDennis Dalessandro 2283f48ad614SDennis Dalessandro default: 2284f48ad614SDennis Dalessandro /* Ignore other reserved NAK error codes */ 2285f48ad614SDennis Dalessandro goto reserved; 2286f48ad614SDennis Dalessandro } 2287f48ad614SDennis Dalessandro qp->s_retry = qp->s_retry_cnt; 2288f48ad614SDennis Dalessandro qp->s_rnr_retry = qp->s_rnr_retry_cnt; 2289f48ad614SDennis Dalessandro goto bail_stop; 2290f48ad614SDennis Dalessandro 2291f48ad614SDennis Dalessandro default: /* 2: reserved */ 2292f48ad614SDennis Dalessandro reserved: 2293f48ad614SDennis Dalessandro /* Ignore reserved NAK codes. */ 2294f48ad614SDennis Dalessandro goto bail_stop; 2295f48ad614SDennis Dalessandro } 2296f48ad614SDennis Dalessandro /* cannot be reached */ 2297f48ad614SDennis Dalessandro bail_stop: 229856acbbfbSVenkata Sandeep Dhanalakota rvt_stop_rc_timers(qp); 2299f48ad614SDennis Dalessandro return ret; 2300f48ad614SDennis Dalessandro } 2301f48ad614SDennis Dalessandro 2302f48ad614SDennis Dalessandro /* 2303f48ad614SDennis Dalessandro * We have seen an out of sequence RDMA read middle or last packet. 2304f48ad614SDennis Dalessandro * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE. 2305f48ad614SDennis Dalessandro */ 2306f48ad614SDennis Dalessandro static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn, 2307f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd) 2308f48ad614SDennis Dalessandro { 2309f48ad614SDennis Dalessandro struct rvt_swqe *wqe; 2310f48ad614SDennis Dalessandro 231168e78b3dSMike Marciniszyn lockdep_assert_held(&qp->s_lock); 2312f48ad614SDennis Dalessandro /* Remove QP from retry timer */ 231356acbbfbSVenkata Sandeep Dhanalakota rvt_stop_rc_timers(qp); 2314f48ad614SDennis Dalessandro 2315f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 2316f48ad614SDennis Dalessandro 2317f48ad614SDennis Dalessandro while (cmp_psn(psn, wqe->lpsn) > 0) { 2318f48ad614SDennis Dalessandro if (wqe->wr.opcode == IB_WR_RDMA_READ || 231924b11923SKaike Wan wqe->wr.opcode == IB_WR_TID_RDMA_READ || 23203c6cb20aSKaike Wan wqe->wr.opcode == IB_WR_TID_RDMA_WRITE || 2321f48ad614SDennis Dalessandro wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || 2322f48ad614SDennis Dalessandro wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) 2323f48ad614SDennis Dalessandro break; 2324f48ad614SDennis Dalessandro wqe = do_rc_completion(qp, wqe, ibp); 2325f48ad614SDennis Dalessandro } 2326f48ad614SDennis Dalessandro 2327f48ad614SDennis Dalessandro ibp->rvp.n_rdma_seq++; 2328f48ad614SDennis Dalessandro qp->r_flags |= RVT_R_RDMAR_SEQ; 232956acbbfbSVenkata Sandeep Dhanalakota hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); 2330f48ad614SDennis Dalessandro if (list_empty(&qp->rspwait)) { 2331f48ad614SDennis Dalessandro qp->r_flags |= RVT_R_RSP_SEND; 23324d6f85c3SMike Marciniszyn rvt_get_qp(qp); 2333f48ad614SDennis Dalessandro list_add_tail(&qp->rspwait, &rcd->qp_wait_list); 2334f48ad614SDennis Dalessandro } 2335f48ad614SDennis Dalessandro } 2336f48ad614SDennis Dalessandro 2337f48ad614SDennis Dalessandro /** 2338f48ad614SDennis Dalessandro * rc_rcv_resp - process an incoming RC response packet 23395b6cabb0SDon Hiatt * @packet: data packet information 2340f48ad614SDennis Dalessandro * 2341f48ad614SDennis Dalessandro * This is called from hfi1_rc_rcv() to process an incoming RC response 2342f48ad614SDennis Dalessandro * packet for the given QP. 2343f48ad614SDennis Dalessandro * Called at interrupt level. 2344f48ad614SDennis Dalessandro */ 23455b6cabb0SDon Hiatt static void rc_rcv_resp(struct hfi1_packet *packet) 2346f48ad614SDennis Dalessandro { 23475b6cabb0SDon Hiatt struct hfi1_ctxtdata *rcd = packet->rcd; 23485b6cabb0SDon Hiatt void *data = packet->payload; 23495b6cabb0SDon Hiatt u32 tlen = packet->tlen; 23505b6cabb0SDon Hiatt struct rvt_qp *qp = packet->qp; 2351bdaf96f6SSebastian Sanchez struct hfi1_ibport *ibp; 23525b6cabb0SDon Hiatt struct ib_other_headers *ohdr = packet->ohdr; 2353f48ad614SDennis Dalessandro struct rvt_swqe *wqe; 2354f48ad614SDennis Dalessandro enum ib_wc_status status; 2355f48ad614SDennis Dalessandro unsigned long flags; 2356f48ad614SDennis Dalessandro int diff; 2357f48ad614SDennis Dalessandro u64 val; 23585b6cabb0SDon Hiatt u32 aeth; 23595b6cabb0SDon Hiatt u32 psn = ib_bth_get_psn(packet->ohdr); 23605b6cabb0SDon Hiatt u32 pmtu = qp->pmtu; 23615b6cabb0SDon Hiatt u16 hdrsize = packet->hlen; 23625b6cabb0SDon Hiatt u8 opcode = packet->opcode; 23635b6cabb0SDon Hiatt u8 pad = packet->pad; 23645b6cabb0SDon Hiatt u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2); 2365f48ad614SDennis Dalessandro 2366f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->s_lock, flags); 2367462b6b21SSebastian Sanchez trace_hfi1_ack(qp, psn); 2368f48ad614SDennis Dalessandro 2369f48ad614SDennis Dalessandro /* Ignore invalid responses. */ 2370eb04ff09SMike Marciniszyn if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0) 2371f48ad614SDennis Dalessandro goto ack_done; 2372f48ad614SDennis Dalessandro 2373f48ad614SDennis Dalessandro /* Ignore duplicate responses. */ 2374f48ad614SDennis Dalessandro diff = cmp_psn(psn, qp->s_last_psn); 2375f48ad614SDennis Dalessandro if (unlikely(diff <= 0)) { 2376f48ad614SDennis Dalessandro /* Update credits for "ghost" ACKs */ 2377f48ad614SDennis Dalessandro if (diff == 0 && opcode == OP(ACKNOWLEDGE)) { 2378f48ad614SDennis Dalessandro aeth = be32_to_cpu(ohdr->u.aeth); 2379832666c1SDon Hiatt if ((aeth >> IB_AETH_NAK_SHIFT) == 0) 2380696513e8SBrian Welty rvt_get_credit(qp, aeth); 2381f48ad614SDennis Dalessandro } 2382f48ad614SDennis Dalessandro goto ack_done; 2383f48ad614SDennis Dalessandro } 2384f48ad614SDennis Dalessandro 2385f48ad614SDennis Dalessandro /* 2386f48ad614SDennis Dalessandro * Skip everything other than the PSN we expect, if we are waiting 2387f48ad614SDennis Dalessandro * for a reply to a restarted RDMA read or atomic op. 2388f48ad614SDennis Dalessandro */ 2389f48ad614SDennis Dalessandro if (qp->r_flags & RVT_R_RDMAR_SEQ) { 2390f48ad614SDennis Dalessandro if (cmp_psn(psn, qp->s_last_psn + 1) != 0) 2391f48ad614SDennis Dalessandro goto ack_done; 2392f48ad614SDennis Dalessandro qp->r_flags &= ~RVT_R_RDMAR_SEQ; 2393f48ad614SDennis Dalessandro } 2394f48ad614SDennis Dalessandro 2395f48ad614SDennis Dalessandro if (unlikely(qp->s_acked == qp->s_tail)) 2396f48ad614SDennis Dalessandro goto ack_done; 2397f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 2398f48ad614SDennis Dalessandro status = IB_WC_SUCCESS; 2399f48ad614SDennis Dalessandro 2400f48ad614SDennis Dalessandro switch (opcode) { 2401f48ad614SDennis Dalessandro case OP(ACKNOWLEDGE): 2402f48ad614SDennis Dalessandro case OP(ATOMIC_ACKNOWLEDGE): 2403f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_FIRST): 2404f48ad614SDennis Dalessandro aeth = be32_to_cpu(ohdr->u.aeth); 2405261a4351SMike Marciniszyn if (opcode == OP(ATOMIC_ACKNOWLEDGE)) 2406261a4351SMike Marciniszyn val = ib_u64_get(&ohdr->u.at.atomic_ack_eth); 2407261a4351SMike Marciniszyn else 2408f48ad614SDennis Dalessandro val = 0; 2409f48ad614SDennis Dalessandro if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || 2410f48ad614SDennis Dalessandro opcode != OP(RDMA_READ_RESPONSE_FIRST)) 2411f48ad614SDennis Dalessandro goto ack_done; 2412f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 2413f48ad614SDennis Dalessandro if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 2414f48ad614SDennis Dalessandro goto ack_op_err; 2415f48ad614SDennis Dalessandro /* 2416f48ad614SDennis Dalessandro * If this is a response to a resent RDMA read, we 2417f48ad614SDennis Dalessandro * have to be careful to copy the data to the right 2418f48ad614SDennis Dalessandro * location. 2419f48ad614SDennis Dalessandro */ 2420f48ad614SDennis Dalessandro qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 2421f48ad614SDennis Dalessandro wqe, psn, pmtu); 2422f48ad614SDennis Dalessandro goto read_middle; 2423f48ad614SDennis Dalessandro 2424f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_MIDDLE): 2425f48ad614SDennis Dalessandro /* no AETH, no ACK */ 2426f48ad614SDennis Dalessandro if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) 2427f48ad614SDennis Dalessandro goto ack_seq_err; 2428f48ad614SDennis Dalessandro if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 2429f48ad614SDennis Dalessandro goto ack_op_err; 2430f48ad614SDennis Dalessandro read_middle: 24315b6cabb0SDon Hiatt if (unlikely(tlen != (hdrsize + pmtu + extra_bytes))) 2432f48ad614SDennis Dalessandro goto ack_len_err; 2433f48ad614SDennis Dalessandro if (unlikely(pmtu >= qp->s_rdma_read_len)) 2434f48ad614SDennis Dalessandro goto ack_len_err; 2435f48ad614SDennis Dalessandro 2436f48ad614SDennis Dalessandro /* 2437f48ad614SDennis Dalessandro * We got a response so update the timeout. 2438f48ad614SDennis Dalessandro * 4.096 usec. * (1 << qp->timeout) 2439f48ad614SDennis Dalessandro */ 244056acbbfbSVenkata Sandeep Dhanalakota rvt_mod_retry_timer(qp); 2441f48ad614SDennis Dalessandro if (qp->s_flags & RVT_S_WAIT_ACK) { 2442f48ad614SDennis Dalessandro qp->s_flags &= ~RVT_S_WAIT_ACK; 2443f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 2444f48ad614SDennis Dalessandro } 2445f48ad614SDennis Dalessandro 2446f48ad614SDennis Dalessandro if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE)) 2447f48ad614SDennis Dalessandro qp->s_retry = qp->s_retry_cnt; 2448f48ad614SDennis Dalessandro 2449f48ad614SDennis Dalessandro /* 2450f48ad614SDennis Dalessandro * Update the RDMA receive state but do the copy w/o 2451f48ad614SDennis Dalessandro * holding the locks and blocking interrupts. 2452f48ad614SDennis Dalessandro */ 2453f48ad614SDennis Dalessandro qp->s_rdma_read_len -= pmtu; 2454f48ad614SDennis Dalessandro update_last_psn(qp, psn); 2455f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 2456019f118bSBrian Welty rvt_copy_sge(qp, &qp->s_rdma_read_sge, 2457019f118bSBrian Welty data, pmtu, false, false); 2458f48ad614SDennis Dalessandro goto bail; 2459f48ad614SDennis Dalessandro 2460f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_ONLY): 2461f48ad614SDennis Dalessandro aeth = be32_to_cpu(ohdr->u.aeth); 2462f48ad614SDennis Dalessandro if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) 2463f48ad614SDennis Dalessandro goto ack_done; 2464f48ad614SDennis Dalessandro /* 2465f48ad614SDennis Dalessandro * Check that the data size is >= 0 && <= pmtu. 2466f48ad614SDennis Dalessandro * Remember to account for ICRC (4). 2467f48ad614SDennis Dalessandro */ 24685b6cabb0SDon Hiatt if (unlikely(tlen < (hdrsize + extra_bytes))) 2469f48ad614SDennis Dalessandro goto ack_len_err; 2470f48ad614SDennis Dalessandro /* 2471f48ad614SDennis Dalessandro * If this is a response to a resent RDMA read, we 2472f48ad614SDennis Dalessandro * have to be careful to copy the data to the right 2473f48ad614SDennis Dalessandro * location. 2474f48ad614SDennis Dalessandro */ 2475f48ad614SDennis Dalessandro wqe = rvt_get_swqe_ptr(qp, qp->s_acked); 2476f48ad614SDennis Dalessandro qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, 2477f48ad614SDennis Dalessandro wqe, psn, pmtu); 2478f48ad614SDennis Dalessandro goto read_last; 2479f48ad614SDennis Dalessandro 2480f48ad614SDennis Dalessandro case OP(RDMA_READ_RESPONSE_LAST): 2481f48ad614SDennis Dalessandro /* ACKs READ req. */ 2482f48ad614SDennis Dalessandro if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) 2483f48ad614SDennis Dalessandro goto ack_seq_err; 2484f48ad614SDennis Dalessandro if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) 2485f48ad614SDennis Dalessandro goto ack_op_err; 2486f48ad614SDennis Dalessandro /* 2487f48ad614SDennis Dalessandro * Check that the data size is >= 1 && <= pmtu. 2488f48ad614SDennis Dalessandro * Remember to account for ICRC (4). 2489f48ad614SDennis Dalessandro */ 24905b6cabb0SDon Hiatt if (unlikely(tlen <= (hdrsize + extra_bytes))) 2491f48ad614SDennis Dalessandro goto ack_len_err; 2492f48ad614SDennis Dalessandro read_last: 24935b6cabb0SDon Hiatt tlen -= hdrsize + extra_bytes; 2494f48ad614SDennis Dalessandro if (unlikely(tlen != qp->s_rdma_read_len)) 2495f48ad614SDennis Dalessandro goto ack_len_err; 2496f48ad614SDennis Dalessandro aeth = be32_to_cpu(ohdr->u.aeth); 2497019f118bSBrian Welty rvt_copy_sge(qp, &qp->s_rdma_read_sge, 2498019f118bSBrian Welty data, tlen, false, false); 2499f48ad614SDennis Dalessandro WARN_ON(qp->s_rdma_read_sge.num_sge); 2500f48ad614SDennis Dalessandro (void)do_rc_ack(qp, aeth, psn, 2501f48ad614SDennis Dalessandro OP(RDMA_READ_RESPONSE_LAST), 0, rcd); 2502f48ad614SDennis Dalessandro goto ack_done; 2503f48ad614SDennis Dalessandro } 2504f48ad614SDennis Dalessandro 2505f48ad614SDennis Dalessandro ack_op_err: 2506f48ad614SDennis Dalessandro status = IB_WC_LOC_QP_OP_ERR; 2507f48ad614SDennis Dalessandro goto ack_err; 2508f48ad614SDennis Dalessandro 2509f48ad614SDennis Dalessandro ack_seq_err: 2510bdaf96f6SSebastian Sanchez ibp = rcd_to_iport(rcd); 2511f48ad614SDennis Dalessandro rdma_seq_err(qp, ibp, psn, rcd); 2512f48ad614SDennis Dalessandro goto ack_done; 2513f48ad614SDennis Dalessandro 2514f48ad614SDennis Dalessandro ack_len_err: 2515f48ad614SDennis Dalessandro status = IB_WC_LOC_LEN_ERR; 2516f48ad614SDennis Dalessandro ack_err: 2517f48ad614SDennis Dalessandro if (qp->s_last == qp->s_acked) { 2518116aa033SVenkata Sandeep Dhanalakota rvt_send_complete(qp, wqe, status); 2519f48ad614SDennis Dalessandro rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); 2520f48ad614SDennis Dalessandro } 2521f48ad614SDennis Dalessandro ack_done: 2522f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 2523f48ad614SDennis Dalessandro bail: 2524f48ad614SDennis Dalessandro return; 2525f48ad614SDennis Dalessandro } 2526f48ad614SDennis Dalessandro 2527f48ad614SDennis Dalessandro static inline void rc_cancel_ack(struct rvt_qp *qp) 2528f48ad614SDennis Dalessandro { 2529688f21c0SMike Marciniszyn qp->r_adefered = 0; 2530f48ad614SDennis Dalessandro if (list_empty(&qp->rspwait)) 2531f48ad614SDennis Dalessandro return; 2532f48ad614SDennis Dalessandro list_del_init(&qp->rspwait); 2533f48ad614SDennis Dalessandro qp->r_flags &= ~RVT_R_RSP_NAK; 25344d6f85c3SMike Marciniszyn rvt_put_qp(qp); 2535f48ad614SDennis Dalessandro } 2536f48ad614SDennis Dalessandro 2537f48ad614SDennis Dalessandro /** 2538f48ad614SDennis Dalessandro * rc_rcv_error - process an incoming duplicate or error RC packet 2539f48ad614SDennis Dalessandro * @ohdr: the other headers for this packet 2540f48ad614SDennis Dalessandro * @data: the packet data 2541f48ad614SDennis Dalessandro * @qp: the QP for this packet 2542f48ad614SDennis Dalessandro * @opcode: the opcode for this packet 2543f48ad614SDennis Dalessandro * @psn: the packet sequence number for this packet 2544f48ad614SDennis Dalessandro * @diff: the difference between the PSN and the expected PSN 2545f48ad614SDennis Dalessandro * 2546f48ad614SDennis Dalessandro * This is called from hfi1_rc_rcv() to process an unexpected 2547f48ad614SDennis Dalessandro * incoming RC packet for the given QP. 2548f48ad614SDennis Dalessandro * Called at interrupt level. 2549f48ad614SDennis Dalessandro * Return 1 if no more processing is needed; otherwise return 0 to 2550f48ad614SDennis Dalessandro * schedule a response to be sent. 2551f48ad614SDennis Dalessandro */ 2552261a4351SMike Marciniszyn static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data, 2553f48ad614SDennis Dalessandro struct rvt_qp *qp, u32 opcode, u32 psn, 2554f48ad614SDennis Dalessandro int diff, struct hfi1_ctxtdata *rcd) 2555f48ad614SDennis Dalessandro { 2556f3e862cbSSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(rcd); 2557f48ad614SDennis Dalessandro struct rvt_ack_entry *e; 2558f48ad614SDennis Dalessandro unsigned long flags; 2559385156c5SKaike Wan u8 prev; 2560385156c5SKaike Wan u8 mra; /* most recent ACK */ 2561385156c5SKaike Wan bool old_req; 2562f48ad614SDennis Dalessandro 2563462b6b21SSebastian Sanchez trace_hfi1_rcv_error(qp, psn); 2564f48ad614SDennis Dalessandro if (diff > 0) { 2565f48ad614SDennis Dalessandro /* 2566f48ad614SDennis Dalessandro * Packet sequence error. 2567f48ad614SDennis Dalessandro * A NAK will ACK earlier sends and RDMA writes. 2568f48ad614SDennis Dalessandro * Don't queue the NAK if we already sent one. 2569f48ad614SDennis Dalessandro */ 2570f48ad614SDennis Dalessandro if (!qp->r_nak_state) { 2571f48ad614SDennis Dalessandro ibp->rvp.n_rc_seqnak++; 2572f48ad614SDennis Dalessandro qp->r_nak_state = IB_NAK_PSN_ERROR; 2573f48ad614SDennis Dalessandro /* Use the expected PSN. */ 2574f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn; 2575f48ad614SDennis Dalessandro /* 2576f48ad614SDennis Dalessandro * Wait to send the sequence NAK until all packets 2577f48ad614SDennis Dalessandro * in the receive queue have been processed. 2578f48ad614SDennis Dalessandro * Otherwise, we end up propagating congestion. 2579f48ad614SDennis Dalessandro */ 2580f48ad614SDennis Dalessandro rc_defered_ack(rcd, qp); 2581f48ad614SDennis Dalessandro } 2582f48ad614SDennis Dalessandro goto done; 2583f48ad614SDennis Dalessandro } 2584f48ad614SDennis Dalessandro 2585f48ad614SDennis Dalessandro /* 2586f48ad614SDennis Dalessandro * Handle a duplicate request. Don't re-execute SEND, RDMA 2587f48ad614SDennis Dalessandro * write or atomic op. Don't NAK errors, just silently drop 2588f48ad614SDennis Dalessandro * the duplicate request. Note that r_sge, r_len, and 2589f48ad614SDennis Dalessandro * r_rcv_len may be in use so don't modify them. 2590f48ad614SDennis Dalessandro * 2591f48ad614SDennis Dalessandro * We are supposed to ACK the earliest duplicate PSN but we 2592f48ad614SDennis Dalessandro * can coalesce an outstanding duplicate ACK. We have to 2593f48ad614SDennis Dalessandro * send the earliest so that RDMA reads can be restarted at 2594f48ad614SDennis Dalessandro * the requester's expected PSN. 2595f48ad614SDennis Dalessandro * 2596f48ad614SDennis Dalessandro * First, find where this duplicate PSN falls within the 2597f48ad614SDennis Dalessandro * ACKs previously sent. 2598f48ad614SDennis Dalessandro * old_req is true if there is an older response that is scheduled 2599f48ad614SDennis Dalessandro * to be sent before sending this one. 2600f48ad614SDennis Dalessandro */ 2601f48ad614SDennis Dalessandro e = NULL; 2602f48ad614SDennis Dalessandro old_req = 1; 2603f48ad614SDennis Dalessandro ibp->rvp.n_rc_dupreq++; 2604f48ad614SDennis Dalessandro 2605f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->s_lock, flags); 2606f48ad614SDennis Dalessandro 2607385156c5SKaike Wan e = find_prev_entry(qp, psn, &prev, &mra, &old_req); 2608385156c5SKaike Wan 2609f48ad614SDennis Dalessandro switch (opcode) { 2610f48ad614SDennis Dalessandro case OP(RDMA_READ_REQUEST): { 2611f48ad614SDennis Dalessandro struct ib_reth *reth; 2612f48ad614SDennis Dalessandro u32 offset; 2613f48ad614SDennis Dalessandro u32 len; 2614f48ad614SDennis Dalessandro 2615f48ad614SDennis Dalessandro /* 2616f48ad614SDennis Dalessandro * If we didn't find the RDMA read request in the ack queue, 2617f48ad614SDennis Dalessandro * we can ignore this request. 2618f48ad614SDennis Dalessandro */ 2619f48ad614SDennis Dalessandro if (!e || e->opcode != OP(RDMA_READ_REQUEST)) 2620f48ad614SDennis Dalessandro goto unlock_done; 2621f48ad614SDennis Dalessandro /* RETH comes after BTH */ 2622f48ad614SDennis Dalessandro reth = &ohdr->u.rc.reth; 2623f48ad614SDennis Dalessandro /* 2624f48ad614SDennis Dalessandro * Address range must be a subset of the original 2625f48ad614SDennis Dalessandro * request and start on pmtu boundaries. 2626f48ad614SDennis Dalessandro * We reuse the old ack_queue slot since the requester 2627f48ad614SDennis Dalessandro * should not back up and request an earlier PSN for the 2628f48ad614SDennis Dalessandro * same request. 2629f48ad614SDennis Dalessandro */ 2630f48ad614SDennis Dalessandro offset = delta_psn(psn, e->psn) * qp->pmtu; 2631f48ad614SDennis Dalessandro len = be32_to_cpu(reth->length); 2632f48ad614SDennis Dalessandro if (unlikely(offset + len != e->rdma_sge.sge_length)) 2633f48ad614SDennis Dalessandro goto unlock_done; 2634f6f3f532SKaike Wan release_rdma_sge_mr(e); 2635f48ad614SDennis Dalessandro if (len != 0) { 2636f48ad614SDennis Dalessandro u32 rkey = be32_to_cpu(reth->rkey); 2637261a4351SMike Marciniszyn u64 vaddr = get_ib_reth_vaddr(reth); 2638f48ad614SDennis Dalessandro int ok; 2639f48ad614SDennis Dalessandro 2640f48ad614SDennis Dalessandro ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, 2641f48ad614SDennis Dalessandro IB_ACCESS_REMOTE_READ); 2642f48ad614SDennis Dalessandro if (unlikely(!ok)) 2643f48ad614SDennis Dalessandro goto unlock_done; 2644f48ad614SDennis Dalessandro } else { 2645f48ad614SDennis Dalessandro e->rdma_sge.vaddr = NULL; 2646f48ad614SDennis Dalessandro e->rdma_sge.length = 0; 2647f48ad614SDennis Dalessandro e->rdma_sge.sge_length = 0; 2648f48ad614SDennis Dalessandro } 2649f48ad614SDennis Dalessandro e->psn = psn; 2650f48ad614SDennis Dalessandro if (old_req) 2651f48ad614SDennis Dalessandro goto unlock_done; 26524f9264d1SKaike Wan if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) 26534f9264d1SKaike Wan qp->s_acked_ack_queue = prev; 2654f48ad614SDennis Dalessandro qp->s_tail_ack_queue = prev; 2655f48ad614SDennis Dalessandro break; 2656f48ad614SDennis Dalessandro } 2657f48ad614SDennis Dalessandro 2658f48ad614SDennis Dalessandro case OP(COMPARE_SWAP): 2659f48ad614SDennis Dalessandro case OP(FETCH_ADD): { 2660f48ad614SDennis Dalessandro /* 2661f48ad614SDennis Dalessandro * If we didn't find the atomic request in the ack queue 2662ca00c62bSDennis Dalessandro * or the send engine is already backed up to send an 2663f48ad614SDennis Dalessandro * earlier entry, we can ignore this request. 2664f48ad614SDennis Dalessandro */ 2665f48ad614SDennis Dalessandro if (!e || e->opcode != (u8)opcode || old_req) 2666f48ad614SDennis Dalessandro goto unlock_done; 26674f9264d1SKaike Wan if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) 26684f9264d1SKaike Wan qp->s_acked_ack_queue = prev; 2669f48ad614SDennis Dalessandro qp->s_tail_ack_queue = prev; 2670f48ad614SDennis Dalessandro break; 2671f48ad614SDennis Dalessandro } 2672f48ad614SDennis Dalessandro 2673f48ad614SDennis Dalessandro default: 2674f48ad614SDennis Dalessandro /* 2675f48ad614SDennis Dalessandro * Ignore this operation if it doesn't request an ACK 2676f48ad614SDennis Dalessandro * or an earlier RDMA read or atomic is going to be resent. 2677f48ad614SDennis Dalessandro */ 2678f48ad614SDennis Dalessandro if (!(psn & IB_BTH_REQ_ACK) || old_req) 2679f48ad614SDennis Dalessandro goto unlock_done; 2680f48ad614SDennis Dalessandro /* 2681f48ad614SDennis Dalessandro * Resend the most recent ACK if this request is 2682f48ad614SDennis Dalessandro * after all the previous RDMA reads and atomics. 2683f48ad614SDennis Dalessandro */ 2684385156c5SKaike Wan if (mra == qp->r_head_ack_queue) { 2685f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 2686f48ad614SDennis Dalessandro qp->r_nak_state = 0; 2687f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn - 1; 2688f48ad614SDennis Dalessandro goto send_ack; 2689f48ad614SDennis Dalessandro } 2690f48ad614SDennis Dalessandro 2691f48ad614SDennis Dalessandro /* 2692f48ad614SDennis Dalessandro * Resend the RDMA read or atomic op which 2693f48ad614SDennis Dalessandro * ACKs this duplicate request. 2694f48ad614SDennis Dalessandro */ 26954f9264d1SKaike Wan if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) 26964f9264d1SKaike Wan qp->s_acked_ack_queue = mra; 2697385156c5SKaike Wan qp->s_tail_ack_queue = mra; 2698f48ad614SDennis Dalessandro break; 2699f48ad614SDennis Dalessandro } 2700f48ad614SDennis Dalessandro qp->s_ack_state = OP(ACKNOWLEDGE); 2701f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_RESP_PENDING; 2702f48ad614SDennis Dalessandro qp->r_nak_state = 0; 2703f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 2704f48ad614SDennis Dalessandro 2705f48ad614SDennis Dalessandro unlock_done: 2706f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 2707f48ad614SDennis Dalessandro done: 2708f48ad614SDennis Dalessandro return 1; 2709f48ad614SDennis Dalessandro 2710f48ad614SDennis Dalessandro send_ack: 2711f48ad614SDennis Dalessandro return 0; 2712f48ad614SDennis Dalessandro } 2713f48ad614SDennis Dalessandro 2714f48ad614SDennis Dalessandro static void log_cca_event(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, 2715f48ad614SDennis Dalessandro u32 lqpn, u32 rqpn, u8 svc_type) 2716f48ad614SDennis Dalessandro { 2717f48ad614SDennis Dalessandro struct opa_hfi1_cong_log_event_internal *cc_event; 2718f48ad614SDennis Dalessandro unsigned long flags; 2719f48ad614SDennis Dalessandro 2720f48ad614SDennis Dalessandro if (sl >= OPA_MAX_SLS) 2721f48ad614SDennis Dalessandro return; 2722f48ad614SDennis Dalessandro 2723f48ad614SDennis Dalessandro spin_lock_irqsave(&ppd->cc_log_lock, flags); 2724f48ad614SDennis Dalessandro 2725f48ad614SDennis Dalessandro ppd->threshold_cong_event_map[sl / 8] |= 1 << (sl % 8); 2726f48ad614SDennis Dalessandro ppd->threshold_event_counter++; 2727f48ad614SDennis Dalessandro 2728f48ad614SDennis Dalessandro cc_event = &ppd->cc_events[ppd->cc_log_idx++]; 2729f48ad614SDennis Dalessandro if (ppd->cc_log_idx == OPA_CONG_LOG_ELEMS) 2730f48ad614SDennis Dalessandro ppd->cc_log_idx = 0; 2731f48ad614SDennis Dalessandro cc_event->lqpn = lqpn & RVT_QPN_MASK; 2732f48ad614SDennis Dalessandro cc_event->rqpn = rqpn & RVT_QPN_MASK; 2733f48ad614SDennis Dalessandro cc_event->sl = sl; 2734f48ad614SDennis Dalessandro cc_event->svc_type = svc_type; 2735f48ad614SDennis Dalessandro cc_event->rlid = rlid; 2736f48ad614SDennis Dalessandro /* keep timestamp in units of 1.024 usec */ 2737d61ea075SMike Marciniszyn cc_event->timestamp = ktime_get_ns() / 1024; 2738f48ad614SDennis Dalessandro 2739f48ad614SDennis Dalessandro spin_unlock_irqrestore(&ppd->cc_log_lock, flags); 2740f48ad614SDennis Dalessandro } 2741f48ad614SDennis Dalessandro 27425b6cabb0SDon Hiatt void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn, 2743f48ad614SDennis Dalessandro u32 rqpn, u8 svc_type) 2744f48ad614SDennis Dalessandro { 2745f48ad614SDennis Dalessandro struct cca_timer *cca_timer; 2746f48ad614SDennis Dalessandro u16 ccti, ccti_incr, ccti_timer, ccti_limit; 2747f48ad614SDennis Dalessandro u8 trigger_threshold; 2748f48ad614SDennis Dalessandro struct cc_state *cc_state; 2749f48ad614SDennis Dalessandro unsigned long flags; 2750f48ad614SDennis Dalessandro 2751f48ad614SDennis Dalessandro if (sl >= OPA_MAX_SLS) 2752f48ad614SDennis Dalessandro return; 2753f48ad614SDennis Dalessandro 2754f48ad614SDennis Dalessandro cc_state = get_cc_state(ppd); 2755f48ad614SDennis Dalessandro 2756f48ad614SDennis Dalessandro if (!cc_state) 2757f48ad614SDennis Dalessandro return; 2758f48ad614SDennis Dalessandro 2759f48ad614SDennis Dalessandro /* 2760f48ad614SDennis Dalessandro * 1) increase CCTI (for this SL) 2761f48ad614SDennis Dalessandro * 2) select IPG (i.e., call set_link_ipg()) 2762f48ad614SDennis Dalessandro * 3) start timer 2763f48ad614SDennis Dalessandro */ 2764f48ad614SDennis Dalessandro ccti_limit = cc_state->cct.ccti_limit; 2765f48ad614SDennis Dalessandro ccti_incr = cc_state->cong_setting.entries[sl].ccti_increase; 2766f48ad614SDennis Dalessandro ccti_timer = cc_state->cong_setting.entries[sl].ccti_timer; 2767f48ad614SDennis Dalessandro trigger_threshold = 2768f48ad614SDennis Dalessandro cc_state->cong_setting.entries[sl].trigger_threshold; 2769f48ad614SDennis Dalessandro 2770f48ad614SDennis Dalessandro spin_lock_irqsave(&ppd->cca_timer_lock, flags); 2771f48ad614SDennis Dalessandro 2772f48ad614SDennis Dalessandro cca_timer = &ppd->cca_timer[sl]; 2773f48ad614SDennis Dalessandro if (cca_timer->ccti < ccti_limit) { 2774f48ad614SDennis Dalessandro if (cca_timer->ccti + ccti_incr <= ccti_limit) 2775f48ad614SDennis Dalessandro cca_timer->ccti += ccti_incr; 2776f48ad614SDennis Dalessandro else 2777f48ad614SDennis Dalessandro cca_timer->ccti = ccti_limit; 2778f48ad614SDennis Dalessandro set_link_ipg(ppd); 2779f48ad614SDennis Dalessandro } 2780f48ad614SDennis Dalessandro 2781f48ad614SDennis Dalessandro ccti = cca_timer->ccti; 2782f48ad614SDennis Dalessandro 2783f48ad614SDennis Dalessandro if (!hrtimer_active(&cca_timer->hrtimer)) { 2784f48ad614SDennis Dalessandro /* ccti_timer is in units of 1.024 usec */ 2785f48ad614SDennis Dalessandro unsigned long nsec = 1024 * ccti_timer; 2786f48ad614SDennis Dalessandro 2787f48ad614SDennis Dalessandro hrtimer_start(&cca_timer->hrtimer, ns_to_ktime(nsec), 27883ce459cdSMike Marciniszyn HRTIMER_MODE_REL_PINNED); 2789f48ad614SDennis Dalessandro } 2790f48ad614SDennis Dalessandro 2791f48ad614SDennis Dalessandro spin_unlock_irqrestore(&ppd->cca_timer_lock, flags); 2792f48ad614SDennis Dalessandro 2793f48ad614SDennis Dalessandro if ((trigger_threshold != 0) && (ccti >= trigger_threshold)) 2794f48ad614SDennis Dalessandro log_cca_event(ppd, sl, rlid, lqpn, rqpn, svc_type); 2795f48ad614SDennis Dalessandro } 2796f48ad614SDennis Dalessandro 2797f48ad614SDennis Dalessandro /** 2798f48ad614SDennis Dalessandro * hfi1_rc_rcv - process an incoming RC packet 27995b6cabb0SDon Hiatt * @packet: data packet information 2800f48ad614SDennis Dalessandro * 2801f48ad614SDennis Dalessandro * This is called from qp_rcv() to process an incoming RC packet 2802f48ad614SDennis Dalessandro * for the given QP. 2803f48ad614SDennis Dalessandro * May be called at interrupt level. 2804f48ad614SDennis Dalessandro */ 2805f48ad614SDennis Dalessandro void hfi1_rc_rcv(struct hfi1_packet *packet) 2806f48ad614SDennis Dalessandro { 2807f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd = packet->rcd; 280872c07e2bSDon Hiatt void *data = packet->payload; 2809f48ad614SDennis Dalessandro u32 tlen = packet->tlen; 2810f48ad614SDennis Dalessandro struct rvt_qp *qp = packet->qp; 281107b92370SKaike Wan struct hfi1_qp_priv *qpriv = qp->priv; 2812f3e862cbSSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(rcd); 2813261a4351SMike Marciniszyn struct ib_other_headers *ohdr = packet->ohdr; 28149039746cSDon Hiatt u32 opcode = packet->opcode; 2815f48ad614SDennis Dalessandro u32 hdrsize = packet->hlen; 28165b6cabb0SDon Hiatt u32 psn = ib_bth_get_psn(packet->ohdr); 28179039746cSDon Hiatt u32 pad = packet->pad; 2818f48ad614SDennis Dalessandro struct ib_wc wc; 2819f48ad614SDennis Dalessandro u32 pmtu = qp->pmtu; 2820f48ad614SDennis Dalessandro int diff; 2821f48ad614SDennis Dalessandro struct ib_reth *reth; 2822f48ad614SDennis Dalessandro unsigned long flags; 28234608e4c8SDennis Dalessandro int ret; 2824fe4dd423SMitko Haralanov bool copy_last = false, fecn; 2825a2df0c83SJianxin Xiong u32 rkey; 28265b6cabb0SDon Hiatt u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2); 2827f48ad614SDennis Dalessandro 282868e78b3dSMike Marciniszyn lockdep_assert_held(&qp->r_lock); 28299039746cSDon Hiatt 28309039746cSDon Hiatt if (hfi1_ruc_check_hdr(ibp, packet)) 2831f48ad614SDennis Dalessandro return; 2832f48ad614SDennis Dalessandro 2833fe4dd423SMitko Haralanov fecn = process_ecn(qp, packet); 283448a615dcSKaike Wan opfn_trigger_conn_request(qp, be32_to_cpu(ohdr->bth[1])); 2835f48ad614SDennis Dalessandro 2836f48ad614SDennis Dalessandro /* 2837f48ad614SDennis Dalessandro * Process responses (ACKs) before anything else. Note that the 2838f48ad614SDennis Dalessandro * packet sequence number will be for something in the send work 2839f48ad614SDennis Dalessandro * queue rather than the expected receive packet sequence number. 2840f48ad614SDennis Dalessandro * In other words, this QP is the requester. 2841f48ad614SDennis Dalessandro */ 2842f48ad614SDennis Dalessandro if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) && 2843f48ad614SDennis Dalessandro opcode <= OP(ATOMIC_ACKNOWLEDGE)) { 28445b6cabb0SDon Hiatt rc_rcv_resp(packet); 2845f48ad614SDennis Dalessandro return; 2846f48ad614SDennis Dalessandro } 2847f48ad614SDennis Dalessandro 2848f48ad614SDennis Dalessandro /* Compute 24 bits worth of difference. */ 2849f48ad614SDennis Dalessandro diff = delta_psn(psn, qp->r_psn); 2850f48ad614SDennis Dalessandro if (unlikely(diff)) { 2851f48ad614SDennis Dalessandro if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) 2852f48ad614SDennis Dalessandro return; 2853f48ad614SDennis Dalessandro goto send_ack; 2854f48ad614SDennis Dalessandro } 2855f48ad614SDennis Dalessandro 2856f48ad614SDennis Dalessandro /* Check for opcode sequence errors. */ 2857f48ad614SDennis Dalessandro switch (qp->r_state) { 2858f48ad614SDennis Dalessandro case OP(SEND_FIRST): 2859f48ad614SDennis Dalessandro case OP(SEND_MIDDLE): 2860f48ad614SDennis Dalessandro if (opcode == OP(SEND_MIDDLE) || 2861f48ad614SDennis Dalessandro opcode == OP(SEND_LAST) || 2862a2df0c83SJianxin Xiong opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 2863a2df0c83SJianxin Xiong opcode == OP(SEND_LAST_WITH_INVALIDATE)) 2864f48ad614SDennis Dalessandro break; 2865f48ad614SDennis Dalessandro goto nack_inv; 2866f48ad614SDennis Dalessandro 2867f48ad614SDennis Dalessandro case OP(RDMA_WRITE_FIRST): 2868f48ad614SDennis Dalessandro case OP(RDMA_WRITE_MIDDLE): 2869f48ad614SDennis Dalessandro if (opcode == OP(RDMA_WRITE_MIDDLE) || 2870f48ad614SDennis Dalessandro opcode == OP(RDMA_WRITE_LAST) || 2871f48ad614SDennis Dalessandro opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 2872f48ad614SDennis Dalessandro break; 2873f48ad614SDennis Dalessandro goto nack_inv; 2874f48ad614SDennis Dalessandro 2875f48ad614SDennis Dalessandro default: 2876f48ad614SDennis Dalessandro if (opcode == OP(SEND_MIDDLE) || 2877f48ad614SDennis Dalessandro opcode == OP(SEND_LAST) || 2878f48ad614SDennis Dalessandro opcode == OP(SEND_LAST_WITH_IMMEDIATE) || 2879a2df0c83SJianxin Xiong opcode == OP(SEND_LAST_WITH_INVALIDATE) || 2880f48ad614SDennis Dalessandro opcode == OP(RDMA_WRITE_MIDDLE) || 2881f48ad614SDennis Dalessandro opcode == OP(RDMA_WRITE_LAST) || 2882f48ad614SDennis Dalessandro opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE)) 2883f48ad614SDennis Dalessandro goto nack_inv; 2884f48ad614SDennis Dalessandro /* 2885f48ad614SDennis Dalessandro * Note that it is up to the requester to not send a new 2886f48ad614SDennis Dalessandro * RDMA read or atomic operation before receiving an ACK 2887f48ad614SDennis Dalessandro * for the previous operation. 2888f48ad614SDennis Dalessandro */ 2889f48ad614SDennis Dalessandro break; 2890f48ad614SDennis Dalessandro } 2891f48ad614SDennis Dalessandro 2892f48ad614SDennis Dalessandro if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) 2893beb5a042SBrian Welty rvt_comm_est(qp); 2894f48ad614SDennis Dalessandro 2895f48ad614SDennis Dalessandro /* OK, process the packet. */ 2896f48ad614SDennis Dalessandro switch (opcode) { 2897f48ad614SDennis Dalessandro case OP(SEND_FIRST): 2898832369faSBrian Welty ret = rvt_get_rwqe(qp, false); 2899f48ad614SDennis Dalessandro if (ret < 0) 2900f48ad614SDennis Dalessandro goto nack_op_err; 2901f48ad614SDennis Dalessandro if (!ret) 2902f48ad614SDennis Dalessandro goto rnr_nak; 2903f48ad614SDennis Dalessandro qp->r_rcv_len = 0; 2904f48ad614SDennis Dalessandro /* FALLTHROUGH */ 2905f48ad614SDennis Dalessandro case OP(SEND_MIDDLE): 2906f48ad614SDennis Dalessandro case OP(RDMA_WRITE_MIDDLE): 2907f48ad614SDennis Dalessandro send_middle: 2908f48ad614SDennis Dalessandro /* Check for invalid length PMTU or posted rwqe len. */ 29095b6cabb0SDon Hiatt /* 29105b6cabb0SDon Hiatt * There will be no padding for 9B packet but 16B packets 29115b6cabb0SDon Hiatt * will come in with some padding since we always add 29125b6cabb0SDon Hiatt * CRC and LT bytes which will need to be flit aligned 29135b6cabb0SDon Hiatt */ 29145b6cabb0SDon Hiatt if (unlikely(tlen != (hdrsize + pmtu + extra_bytes))) 2915f48ad614SDennis Dalessandro goto nack_inv; 2916f48ad614SDennis Dalessandro qp->r_rcv_len += pmtu; 2917f48ad614SDennis Dalessandro if (unlikely(qp->r_rcv_len > qp->r_len)) 2918f48ad614SDennis Dalessandro goto nack_inv; 2919019f118bSBrian Welty rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); 2920f48ad614SDennis Dalessandro break; 2921f48ad614SDennis Dalessandro 2922f48ad614SDennis Dalessandro case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE): 2923f48ad614SDennis Dalessandro /* consume RWQE */ 2924832369faSBrian Welty ret = rvt_get_rwqe(qp, true); 2925f48ad614SDennis Dalessandro if (ret < 0) 2926f48ad614SDennis Dalessandro goto nack_op_err; 2927f48ad614SDennis Dalessandro if (!ret) 2928f48ad614SDennis Dalessandro goto rnr_nak; 2929f48ad614SDennis Dalessandro goto send_last_imm; 2930f48ad614SDennis Dalessandro 2931f48ad614SDennis Dalessandro case OP(SEND_ONLY): 2932f48ad614SDennis Dalessandro case OP(SEND_ONLY_WITH_IMMEDIATE): 2933a2df0c83SJianxin Xiong case OP(SEND_ONLY_WITH_INVALIDATE): 2934832369faSBrian Welty ret = rvt_get_rwqe(qp, false); 2935f48ad614SDennis Dalessandro if (ret < 0) 2936f48ad614SDennis Dalessandro goto nack_op_err; 2937f48ad614SDennis Dalessandro if (!ret) 2938f48ad614SDennis Dalessandro goto rnr_nak; 2939f48ad614SDennis Dalessandro qp->r_rcv_len = 0; 2940f48ad614SDennis Dalessandro if (opcode == OP(SEND_ONLY)) 2941f48ad614SDennis Dalessandro goto no_immediate_data; 2942a2df0c83SJianxin Xiong if (opcode == OP(SEND_ONLY_WITH_INVALIDATE)) 2943a2df0c83SJianxin Xiong goto send_last_inv; 29446ffeb21fSBart Van Assche /* FALLTHROUGH -- for SEND_ONLY_WITH_IMMEDIATE */ 2945f48ad614SDennis Dalessandro case OP(SEND_LAST_WITH_IMMEDIATE): 2946f48ad614SDennis Dalessandro send_last_imm: 2947f48ad614SDennis Dalessandro wc.ex.imm_data = ohdr->u.imm_data; 2948f48ad614SDennis Dalessandro wc.wc_flags = IB_WC_WITH_IMM; 2949f48ad614SDennis Dalessandro goto send_last; 2950a2df0c83SJianxin Xiong case OP(SEND_LAST_WITH_INVALIDATE): 2951a2df0c83SJianxin Xiong send_last_inv: 2952a2df0c83SJianxin Xiong rkey = be32_to_cpu(ohdr->u.ieth); 2953a2df0c83SJianxin Xiong if (rvt_invalidate_rkey(qp, rkey)) 2954a2df0c83SJianxin Xiong goto no_immediate_data; 2955a2df0c83SJianxin Xiong wc.ex.invalidate_rkey = rkey; 2956a2df0c83SJianxin Xiong wc.wc_flags = IB_WC_WITH_INVALIDATE; 2957a2df0c83SJianxin Xiong goto send_last; 2958f48ad614SDennis Dalessandro case OP(RDMA_WRITE_LAST): 29590128fceaSBrian Welty copy_last = rvt_is_user_qp(qp); 2960f48ad614SDennis Dalessandro /* fall through */ 2961f48ad614SDennis Dalessandro case OP(SEND_LAST): 2962f48ad614SDennis Dalessandro no_immediate_data: 2963f48ad614SDennis Dalessandro wc.wc_flags = 0; 2964f48ad614SDennis Dalessandro wc.ex.imm_data = 0; 2965f48ad614SDennis Dalessandro send_last: 2966f48ad614SDennis Dalessandro /* Check for invalid length. */ 2967f48ad614SDennis Dalessandro /* LAST len should be >= 1 */ 29685b6cabb0SDon Hiatt if (unlikely(tlen < (hdrsize + extra_bytes))) 2969f48ad614SDennis Dalessandro goto nack_inv; 29705b6cabb0SDon Hiatt /* Don't count the CRC(and padding and LT byte for 16B). */ 29715b6cabb0SDon Hiatt tlen -= (hdrsize + extra_bytes); 2972f48ad614SDennis Dalessandro wc.byte_len = tlen + qp->r_rcv_len; 2973f48ad614SDennis Dalessandro if (unlikely(wc.byte_len > qp->r_len)) 2974f48ad614SDennis Dalessandro goto nack_inv; 2975019f118bSBrian Welty rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last); 2976f48ad614SDennis Dalessandro rvt_put_ss(&qp->r_sge); 2977f48ad614SDennis Dalessandro qp->r_msn++; 297853e91d26SSebastian Sanchez if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) 2979f48ad614SDennis Dalessandro break; 2980f48ad614SDennis Dalessandro wc.wr_id = qp->r_wr_id; 2981f48ad614SDennis Dalessandro wc.status = IB_WC_SUCCESS; 2982f48ad614SDennis Dalessandro if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) || 2983f48ad614SDennis Dalessandro opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE)) 2984f48ad614SDennis Dalessandro wc.opcode = IB_WC_RECV_RDMA_WITH_IMM; 2985f48ad614SDennis Dalessandro else 2986f48ad614SDennis Dalessandro wc.opcode = IB_WC_RECV; 2987f48ad614SDennis Dalessandro wc.qp = &qp->ibqp; 2988f48ad614SDennis Dalessandro wc.src_qp = qp->remote_qpn; 2989b64581adSDon Hiatt wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; 2990f48ad614SDennis Dalessandro /* 2991f48ad614SDennis Dalessandro * It seems that IB mandates the presence of an SL in a 2992f48ad614SDennis Dalessandro * work completion only for the UD transport (see section 2993f48ad614SDennis Dalessandro * 11.4.2 of IBTA Vol. 1). 2994f48ad614SDennis Dalessandro * 2995f48ad614SDennis Dalessandro * However, the way the SL is chosen below is consistent 2996f48ad614SDennis Dalessandro * with the way that IB/qib works and is trying avoid 2997f48ad614SDennis Dalessandro * introducing incompatibilities. 2998f48ad614SDennis Dalessandro * 2999f48ad614SDennis Dalessandro * See also OPA Vol. 1, section 9.7.6, and table 9-17. 3000f48ad614SDennis Dalessandro */ 3001d8966fcdSDasaratharaman Chandramouli wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); 3002f48ad614SDennis Dalessandro /* zero fields that are N/A */ 3003f48ad614SDennis Dalessandro wc.vendor_err = 0; 3004f48ad614SDennis Dalessandro wc.pkey_index = 0; 3005f48ad614SDennis Dalessandro wc.dlid_path_bits = 0; 3006f48ad614SDennis Dalessandro wc.port_num = 0; 3007f48ad614SDennis Dalessandro /* Signal completion event if the solicited bit is set. */ 30085136bfeaSKamenee Arumugam rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); 3009f48ad614SDennis Dalessandro break; 3010f48ad614SDennis Dalessandro 3011f48ad614SDennis Dalessandro case OP(RDMA_WRITE_ONLY): 30120128fceaSBrian Welty copy_last = rvt_is_user_qp(qp); 3013f48ad614SDennis Dalessandro /* fall through */ 3014f48ad614SDennis Dalessandro case OP(RDMA_WRITE_FIRST): 3015f48ad614SDennis Dalessandro case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE): 3016f48ad614SDennis Dalessandro if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) 3017f48ad614SDennis Dalessandro goto nack_inv; 3018f48ad614SDennis Dalessandro /* consume RWQE */ 3019f48ad614SDennis Dalessandro reth = &ohdr->u.rc.reth; 3020f48ad614SDennis Dalessandro qp->r_len = be32_to_cpu(reth->length); 3021f48ad614SDennis Dalessandro qp->r_rcv_len = 0; 3022f48ad614SDennis Dalessandro qp->r_sge.sg_list = NULL; 3023f48ad614SDennis Dalessandro if (qp->r_len != 0) { 3024f48ad614SDennis Dalessandro u32 rkey = be32_to_cpu(reth->rkey); 3025261a4351SMike Marciniszyn u64 vaddr = get_ib_reth_vaddr(reth); 3026f48ad614SDennis Dalessandro int ok; 3027f48ad614SDennis Dalessandro 3028f48ad614SDennis Dalessandro /* Check rkey & NAK */ 3029f48ad614SDennis Dalessandro ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, 3030f48ad614SDennis Dalessandro rkey, IB_ACCESS_REMOTE_WRITE); 3031f48ad614SDennis Dalessandro if (unlikely(!ok)) 3032f48ad614SDennis Dalessandro goto nack_acc; 3033f48ad614SDennis Dalessandro qp->r_sge.num_sge = 1; 3034f48ad614SDennis Dalessandro } else { 3035f48ad614SDennis Dalessandro qp->r_sge.num_sge = 0; 3036f48ad614SDennis Dalessandro qp->r_sge.sge.mr = NULL; 3037f48ad614SDennis Dalessandro qp->r_sge.sge.vaddr = NULL; 3038f48ad614SDennis Dalessandro qp->r_sge.sge.length = 0; 3039f48ad614SDennis Dalessandro qp->r_sge.sge.sge_length = 0; 3040f48ad614SDennis Dalessandro } 3041f48ad614SDennis Dalessandro if (opcode == OP(RDMA_WRITE_FIRST)) 3042f48ad614SDennis Dalessandro goto send_middle; 3043f48ad614SDennis Dalessandro else if (opcode == OP(RDMA_WRITE_ONLY)) 3044f48ad614SDennis Dalessandro goto no_immediate_data; 3045832369faSBrian Welty ret = rvt_get_rwqe(qp, true); 3046f48ad614SDennis Dalessandro if (ret < 0) 3047f48ad614SDennis Dalessandro goto nack_op_err; 30481feb4006SMike Marciniszyn if (!ret) { 30491feb4006SMike Marciniszyn /* peer will send again */ 30501feb4006SMike Marciniszyn rvt_put_ss(&qp->r_sge); 3051f48ad614SDennis Dalessandro goto rnr_nak; 30521feb4006SMike Marciniszyn } 3053f48ad614SDennis Dalessandro wc.ex.imm_data = ohdr->u.rc.imm_data; 3054f48ad614SDennis Dalessandro wc.wc_flags = IB_WC_WITH_IMM; 3055f48ad614SDennis Dalessandro goto send_last; 3056f48ad614SDennis Dalessandro 3057f48ad614SDennis Dalessandro case OP(RDMA_READ_REQUEST): { 3058f48ad614SDennis Dalessandro struct rvt_ack_entry *e; 3059f48ad614SDennis Dalessandro u32 len; 3060f48ad614SDennis Dalessandro u8 next; 3061f48ad614SDennis Dalessandro 3062f48ad614SDennis Dalessandro if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) 3063f48ad614SDennis Dalessandro goto nack_inv; 3064f48ad614SDennis Dalessandro next = qp->r_head_ack_queue + 1; 3065ddf922c3SKaike Wan /* s_ack_queue is size rvt_size_atomic()+1 so use > not >= */ 3066ddf922c3SKaike Wan if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) 3067f48ad614SDennis Dalessandro next = 0; 3068f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->s_lock, flags); 30694f9264d1SKaike Wan if (unlikely(next == qp->s_acked_ack_queue)) { 3070f48ad614SDennis Dalessandro if (!qp->s_ack_queue[next].sent) 3071f48ad614SDennis Dalessandro goto nack_inv_unlck; 3072f48ad614SDennis Dalessandro update_ack_queue(qp, next); 3073f48ad614SDennis Dalessandro } 3074f48ad614SDennis Dalessandro e = &qp->s_ack_queue[qp->r_head_ack_queue]; 3075f6f3f532SKaike Wan release_rdma_sge_mr(e); 3076f48ad614SDennis Dalessandro reth = &ohdr->u.rc.reth; 3077f48ad614SDennis Dalessandro len = be32_to_cpu(reth->length); 3078f48ad614SDennis Dalessandro if (len) { 3079f48ad614SDennis Dalessandro u32 rkey = be32_to_cpu(reth->rkey); 3080261a4351SMike Marciniszyn u64 vaddr = get_ib_reth_vaddr(reth); 3081f48ad614SDennis Dalessandro int ok; 3082f48ad614SDennis Dalessandro 3083f48ad614SDennis Dalessandro /* Check rkey & NAK */ 3084f48ad614SDennis Dalessandro ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, 3085f48ad614SDennis Dalessandro rkey, IB_ACCESS_REMOTE_READ); 3086f48ad614SDennis Dalessandro if (unlikely(!ok)) 3087f48ad614SDennis Dalessandro goto nack_acc_unlck; 3088f48ad614SDennis Dalessandro /* 3089f48ad614SDennis Dalessandro * Update the next expected PSN. We add 1 later 3090f48ad614SDennis Dalessandro * below, so only add the remainder here. 3091f48ad614SDennis Dalessandro */ 30925dc80605SMike Marciniszyn qp->r_psn += rvt_div_mtu(qp, len - 1); 3093f48ad614SDennis Dalessandro } else { 3094f48ad614SDennis Dalessandro e->rdma_sge.mr = NULL; 3095f48ad614SDennis Dalessandro e->rdma_sge.vaddr = NULL; 3096f48ad614SDennis Dalessandro e->rdma_sge.length = 0; 3097f48ad614SDennis Dalessandro e->rdma_sge.sge_length = 0; 3098f48ad614SDennis Dalessandro } 3099f48ad614SDennis Dalessandro e->opcode = opcode; 3100f48ad614SDennis Dalessandro e->sent = 0; 3101f48ad614SDennis Dalessandro e->psn = psn; 3102f48ad614SDennis Dalessandro e->lpsn = qp->r_psn; 3103f48ad614SDennis Dalessandro /* 3104f48ad614SDennis Dalessandro * We need to increment the MSN here instead of when we 3105f48ad614SDennis Dalessandro * finish sending the result since a duplicate request would 3106f48ad614SDennis Dalessandro * increment it more than once. 3107f48ad614SDennis Dalessandro */ 3108f48ad614SDennis Dalessandro qp->r_msn++; 3109f48ad614SDennis Dalessandro qp->r_psn++; 3110f48ad614SDennis Dalessandro qp->r_state = opcode; 3111f48ad614SDennis Dalessandro qp->r_nak_state = 0; 3112f48ad614SDennis Dalessandro qp->r_head_ack_queue = next; 311307b92370SKaike Wan qpriv->r_tid_alloc = qp->r_head_ack_queue; 3114f48ad614SDennis Dalessandro 3115ca00c62bSDennis Dalessandro /* Schedule the send engine. */ 3116f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_RESP_PENDING; 3117fe4dd423SMitko Haralanov if (fecn) 3118fe4dd423SMitko Haralanov qp->s_flags |= RVT_S_ECN; 3119f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 3120f48ad614SDennis Dalessandro 3121f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 3122f48ad614SDennis Dalessandro return; 3123f48ad614SDennis Dalessandro } 3124f48ad614SDennis Dalessandro 3125f48ad614SDennis Dalessandro case OP(COMPARE_SWAP): 3126f48ad614SDennis Dalessandro case OP(FETCH_ADD): { 312748a615dcSKaike Wan struct ib_atomic_eth *ateth = &ohdr->u.atomic_eth; 312848a615dcSKaike Wan u64 vaddr = get_ib_ateth_vaddr(ateth); 312948a615dcSKaike Wan bool opfn = opcode == OP(COMPARE_SWAP) && 313048a615dcSKaike Wan vaddr == HFI1_VERBS_E_ATOMIC_VADDR; 3131f48ad614SDennis Dalessandro struct rvt_ack_entry *e; 3132f48ad614SDennis Dalessandro atomic64_t *maddr; 3133f48ad614SDennis Dalessandro u64 sdata; 3134f48ad614SDennis Dalessandro u32 rkey; 3135f48ad614SDennis Dalessandro u8 next; 3136f48ad614SDennis Dalessandro 313748a615dcSKaike Wan if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) && 313848a615dcSKaike Wan !opfn)) 3139f48ad614SDennis Dalessandro goto nack_inv; 3140f48ad614SDennis Dalessandro next = qp->r_head_ack_queue + 1; 3141ddf922c3SKaike Wan if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) 3142f48ad614SDennis Dalessandro next = 0; 3143f48ad614SDennis Dalessandro spin_lock_irqsave(&qp->s_lock, flags); 31444f9264d1SKaike Wan if (unlikely(next == qp->s_acked_ack_queue)) { 3145f48ad614SDennis Dalessandro if (!qp->s_ack_queue[next].sent) 3146f48ad614SDennis Dalessandro goto nack_inv_unlck; 3147f48ad614SDennis Dalessandro update_ack_queue(qp, next); 3148f48ad614SDennis Dalessandro } 3149f48ad614SDennis Dalessandro e = &qp->s_ack_queue[qp->r_head_ack_queue]; 3150f6f3f532SKaike Wan release_rdma_sge_mr(e); 315148a615dcSKaike Wan /* Process OPFN special virtual address */ 315248a615dcSKaike Wan if (opfn) { 315348a615dcSKaike Wan opfn_conn_response(qp, e, ateth); 315448a615dcSKaike Wan goto ack; 315548a615dcSKaike Wan } 3156f48ad614SDennis Dalessandro if (unlikely(vaddr & (sizeof(u64) - 1))) 3157f48ad614SDennis Dalessandro goto nack_inv_unlck; 3158f48ad614SDennis Dalessandro rkey = be32_to_cpu(ateth->rkey); 3159f48ad614SDennis Dalessandro /* Check rkey & NAK */ 3160f48ad614SDennis Dalessandro if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), 3161f48ad614SDennis Dalessandro vaddr, rkey, 3162f48ad614SDennis Dalessandro IB_ACCESS_REMOTE_ATOMIC))) 3163f48ad614SDennis Dalessandro goto nack_acc_unlck; 3164f48ad614SDennis Dalessandro /* Perform atomic OP and save result. */ 3165f48ad614SDennis Dalessandro maddr = (atomic64_t *)qp->r_sge.sge.vaddr; 3166261a4351SMike Marciniszyn sdata = get_ib_ateth_swap(ateth); 3167f48ad614SDennis Dalessandro e->atomic_data = (opcode == OP(FETCH_ADD)) ? 3168f48ad614SDennis Dalessandro (u64)atomic64_add_return(sdata, maddr) - sdata : 3169f48ad614SDennis Dalessandro (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr, 3170261a4351SMike Marciniszyn get_ib_ateth_compare(ateth), 3171f48ad614SDennis Dalessandro sdata); 3172f48ad614SDennis Dalessandro rvt_put_mr(qp->r_sge.sge.mr); 3173f48ad614SDennis Dalessandro qp->r_sge.num_sge = 0; 317448a615dcSKaike Wan ack: 3175f48ad614SDennis Dalessandro e->opcode = opcode; 3176f48ad614SDennis Dalessandro e->sent = 0; 3177f48ad614SDennis Dalessandro e->psn = psn; 3178f48ad614SDennis Dalessandro e->lpsn = psn; 3179f48ad614SDennis Dalessandro qp->r_msn++; 3180f48ad614SDennis Dalessandro qp->r_psn++; 3181f48ad614SDennis Dalessandro qp->r_state = opcode; 3182f48ad614SDennis Dalessandro qp->r_nak_state = 0; 3183f48ad614SDennis Dalessandro qp->r_head_ack_queue = next; 318407b92370SKaike Wan qpriv->r_tid_alloc = qp->r_head_ack_queue; 3185f48ad614SDennis Dalessandro 3186ca00c62bSDennis Dalessandro /* Schedule the send engine. */ 3187f48ad614SDennis Dalessandro qp->s_flags |= RVT_S_RESP_PENDING; 3188fe4dd423SMitko Haralanov if (fecn) 3189fe4dd423SMitko Haralanov qp->s_flags |= RVT_S_ECN; 3190f48ad614SDennis Dalessandro hfi1_schedule_send(qp); 3191f48ad614SDennis Dalessandro 3192f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 3193f48ad614SDennis Dalessandro return; 3194f48ad614SDennis Dalessandro } 3195f48ad614SDennis Dalessandro 3196f48ad614SDennis Dalessandro default: 3197f48ad614SDennis Dalessandro /* NAK unknown opcodes. */ 3198f48ad614SDennis Dalessandro goto nack_inv; 3199f48ad614SDennis Dalessandro } 3200f48ad614SDennis Dalessandro qp->r_psn++; 3201f48ad614SDennis Dalessandro qp->r_state = opcode; 3202f48ad614SDennis Dalessandro qp->r_ack_psn = psn; 3203f48ad614SDennis Dalessandro qp->r_nak_state = 0; 3204f48ad614SDennis Dalessandro /* Send an ACK if requested or required. */ 3205fe4dd423SMitko Haralanov if (psn & IB_BTH_REQ_ACK || fecn) { 3206fe4dd423SMitko Haralanov if (packet->numpkt == 0 || fecn || 3207fe4dd423SMitko Haralanov qp->r_adefered >= HFI1_PSN_CREDIT) { 3208f48ad614SDennis Dalessandro rc_cancel_ack(qp); 3209f48ad614SDennis Dalessandro goto send_ack; 3210f48ad614SDennis Dalessandro } 3211688f21c0SMike Marciniszyn qp->r_adefered++; 3212f48ad614SDennis Dalessandro rc_defered_ack(rcd, qp); 3213f48ad614SDennis Dalessandro } 3214f48ad614SDennis Dalessandro return; 3215f48ad614SDennis Dalessandro 3216f48ad614SDennis Dalessandro rnr_nak: 3217f48ad614SDennis Dalessandro qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK; 3218f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn; 3219f48ad614SDennis Dalessandro /* Queue RNR NAK for later */ 3220f48ad614SDennis Dalessandro rc_defered_ack(rcd, qp); 3221f48ad614SDennis Dalessandro return; 3222f48ad614SDennis Dalessandro 3223f48ad614SDennis Dalessandro nack_op_err: 3224beb5a042SBrian Welty rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 3225f48ad614SDennis Dalessandro qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; 3226f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn; 3227f48ad614SDennis Dalessandro /* Queue NAK for later */ 3228f48ad614SDennis Dalessandro rc_defered_ack(rcd, qp); 3229f48ad614SDennis Dalessandro return; 3230f48ad614SDennis Dalessandro 3231f48ad614SDennis Dalessandro nack_inv_unlck: 3232f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 3233f48ad614SDennis Dalessandro nack_inv: 3234beb5a042SBrian Welty rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); 3235f48ad614SDennis Dalessandro qp->r_nak_state = IB_NAK_INVALID_REQUEST; 3236f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn; 3237f48ad614SDennis Dalessandro /* Queue NAK for later */ 3238f48ad614SDennis Dalessandro rc_defered_ack(rcd, qp); 3239f48ad614SDennis Dalessandro return; 3240f48ad614SDennis Dalessandro 3241f48ad614SDennis Dalessandro nack_acc_unlck: 3242f48ad614SDennis Dalessandro spin_unlock_irqrestore(&qp->s_lock, flags); 3243f48ad614SDennis Dalessandro nack_acc: 3244beb5a042SBrian Welty rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); 3245f48ad614SDennis Dalessandro qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; 3246f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn; 3247f48ad614SDennis Dalessandro send_ack: 3248fe4dd423SMitko Haralanov hfi1_send_rc_ack(packet, fecn); 3249f48ad614SDennis Dalessandro } 3250f48ad614SDennis Dalessandro 3251f48ad614SDennis Dalessandro void hfi1_rc_hdrerr( 3252f48ad614SDennis Dalessandro struct hfi1_ctxtdata *rcd, 32539039746cSDon Hiatt struct hfi1_packet *packet, 3254f48ad614SDennis Dalessandro struct rvt_qp *qp) 3255f48ad614SDennis Dalessandro { 3256f3e862cbSSebastian Sanchez struct hfi1_ibport *ibp = rcd_to_iport(rcd); 3257f48ad614SDennis Dalessandro int diff; 3258f48ad614SDennis Dalessandro u32 opcode; 32599039746cSDon Hiatt u32 psn; 3260f48ad614SDennis Dalessandro 32619039746cSDon Hiatt if (hfi1_ruc_check_hdr(ibp, packet)) 3262f48ad614SDennis Dalessandro return; 3263f48ad614SDennis Dalessandro 32649039746cSDon Hiatt psn = ib_bth_get_psn(packet->ohdr); 32659039746cSDon Hiatt opcode = ib_bth_get_opcode(packet->ohdr); 3266f48ad614SDennis Dalessandro 3267f48ad614SDennis Dalessandro /* Only deal with RDMA Writes for now */ 3268f48ad614SDennis Dalessandro if (opcode < IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) { 3269f48ad614SDennis Dalessandro diff = delta_psn(psn, qp->r_psn); 3270f48ad614SDennis Dalessandro if (!qp->r_nak_state && diff >= 0) { 3271f48ad614SDennis Dalessandro ibp->rvp.n_rc_seqnak++; 3272f48ad614SDennis Dalessandro qp->r_nak_state = IB_NAK_PSN_ERROR; 3273f48ad614SDennis Dalessandro /* Use the expected PSN. */ 3274f48ad614SDennis Dalessandro qp->r_ack_psn = qp->r_psn; 3275f48ad614SDennis Dalessandro /* 3276f48ad614SDennis Dalessandro * Wait to send the sequence 3277f48ad614SDennis Dalessandro * NAK until all packets 3278f48ad614SDennis Dalessandro * in the receive queue have 3279f48ad614SDennis Dalessandro * been processed. 3280f48ad614SDennis Dalessandro * Otherwise, we end up 3281f48ad614SDennis Dalessandro * propagating congestion. 3282f48ad614SDennis Dalessandro */ 3283f48ad614SDennis Dalessandro rc_defered_ack(rcd, qp); 3284f48ad614SDennis Dalessandro } /* Out of sequence NAK */ 3285f48ad614SDennis Dalessandro } /* QP Request NAKs */ 3286f48ad614SDennis Dalessandro } 3287