Lines Matching refs:qp
16 struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev, in find_prev_entry() argument
18 __must_hold(&qp->s_lock) in find_prev_entry()
24 for (i = qp->r_head_ack_queue; ; i = p) { in find_prev_entry()
25 if (i == qp->s_tail_ack_queue) in find_prev_entry()
30 p = rvt_size_atomic(ib_to_rvt(qp->ibqp.device)); in find_prev_entry()
31 if (p == qp->r_head_ack_queue) { in find_prev_entry()
35 e = &qp->s_ack_queue[p]; in find_prev_entry()
41 if (p == qp->s_tail_ack_queue && in find_prev_entry()
67 static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp, in make_rc_ack() argument
75 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); in make_rc_ack()
77 u32 pmtu = qp->pmtu; in make_rc_ack()
78 struct hfi1_qp_priv *qpriv = qp->priv; in make_rc_ack()
81 u8 next = qp->s_tail_ack_queue; in make_rc_ack()
84 trace_hfi1_rsp_make_rc_ack(qp, 0); in make_rc_ack()
85 lockdep_assert_held(&qp->s_lock); in make_rc_ack()
87 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in make_rc_ack()
97 switch (qp->s_ack_state) { in make_rc_ack()
100 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
115 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
117 qp->s_acked_ack_queue == qp->s_tail_ack_queue) in make_rc_ack()
118 qp->s_acked_ack_queue = next; in make_rc_ack()
119 qp->s_tail_ack_queue = next; in make_rc_ack()
120 trace_hfi1_rsp_make_rc_ack(qp, e->psn); in make_rc_ack()
125 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in make_rc_ack()
126 if (qp->s_flags & RVT_S_ACK_PENDING) in make_rc_ack()
131 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
134 hfi1_tid_rdma_ack_interlock(qp, e)) { in make_rc_ack()
147 if (qp->s_acked_ack_queue == in make_rc_ack()
148 qp->s_tail_ack_queue) in make_rc_ack()
149 qp->s_acked_ack_queue = in make_rc_ack()
150 qp->r_head_ack_queue; in make_rc_ack()
151 qp->s_tail_ack_queue = qp->r_head_ack_queue; in make_rc_ack()
158 qp->s_ack_rdma_sge.sge = e->rdma_sge; in make_rc_ack()
159 qp->s_ack_rdma_sge.num_sge = 1; in make_rc_ack()
160 ps->s_txreq->ss = &qp->s_ack_rdma_sge; in make_rc_ack()
163 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in make_rc_ack()
165 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in make_rc_ack()
168 ohdr->u.aeth = rvt_compute_aeth(qp); in make_rc_ack()
170 qp->s_ack_rdma_psn = e->psn; in make_rc_ack()
171 bth2 = mask_psn(qp->s_ack_rdma_psn++); in make_rc_ack()
184 qp->s_ack_state = TID_OP(WRITE_RESP); in make_rc_ack()
185 qp->s_ack_rdma_psn = mask_psn(e->psn + req->cur_seg); in make_rc_ack()
196 if (qp->s_acked_ack_queue == in make_rc_ack()
197 qp->s_tail_ack_queue) in make_rc_ack()
198 qp->s_acked_ack_queue = in make_rc_ack()
199 qp->r_head_ack_queue; in make_rc_ack()
200 qp->s_tail_ack_queue = qp->r_head_ack_queue; in make_rc_ack()
207 qp->s_ack_rdma_sge.sge = e->rdma_sge; in make_rc_ack()
208 qp->s_ack_rdma_sge.num_sge = 1; in make_rc_ack()
209 qp->s_ack_state = TID_OP(READ_RESP); in make_rc_ack()
215 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in make_rc_ack()
216 ohdr->u.at.aeth = rvt_compute_aeth(qp); in make_rc_ack()
222 trace_hfi1_tid_write_rsp_make_rc_ack(qp); in make_rc_ack()
223 bth0 = qp->s_ack_state << 24; in make_rc_ack()
227 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in make_rc_ack()
230 ps->s_txreq->ss = &qp->s_ack_rdma_sge; in make_rc_ack()
231 ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr; in make_rc_ack()
234 len = qp->s_ack_rdma_sge.sge.sge_length; in make_rc_ack()
239 ohdr->u.aeth = rvt_compute_aeth(qp); in make_rc_ack()
241 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in make_rc_ack()
242 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
245 bth0 = qp->s_ack_state << 24; in make_rc_ack()
246 bth2 = mask_psn(qp->s_ack_rdma_psn++); in make_rc_ack()
270 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
279 qp->s_tail_ack_queue == qpriv->r_tid_alloc && in make_rc_ack()
285 bth2 = mask_psn(qp->s_ack_rdma_psn); in make_rc_ack()
286 hdrlen = hfi1_build_tid_rdma_write_resp(qp, e, ohdr, &bth1, in make_rc_ack()
293 bth0 = qp->s_ack_state << 24; in make_rc_ack()
294 qp->s_ack_rdma_psn++; in make_rc_ack()
295 trace_hfi1_tid_req_make_rc_ack_write(qp, 0, e->opcode, e->psn, in make_rc_ack()
302 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in make_rc_ack()
307 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in make_rc_ack()
308 ps->s_txreq->ss = &qp->s_ack_rdma_sge; in make_rc_ack()
309 delta = hfi1_build_tid_rdma_read_resp(qp, e, ohdr, &bth0, in make_rc_ack()
321 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in make_rc_ack()
335 qp->s_ack_state = OP(SEND_ONLY); in make_rc_ack()
337 if (qp->s_nak_state) in make_rc_ack()
339 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in make_rc_ack()
340 (qp->s_nak_state << in make_rc_ack()
343 ohdr->u.aeth = rvt_compute_aeth(qp); in make_rc_ack()
347 bth2 = mask_psn(qp->s_ack_psn); in make_rc_ack()
348 qp->s_flags &= ~RVT_S_ACK_PENDING; in make_rc_ack()
352 qp->s_rdma_ack_cnt++; in make_rc_ack()
356 hfi1_make_ruc_header(qp, ohdr, bth0, bth1, bth2, middle, ps); in make_rc_ack()
359 spin_unlock_irqrestore(&qp->s_lock, ps->flags); in make_rc_ack()
360 spin_lock_irqsave(&qp->r_lock, ps->flags); in make_rc_ack()
361 spin_lock(&qp->s_lock); in make_rc_ack()
362 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in make_rc_ack()
363 spin_unlock(&qp->s_lock); in make_rc_ack()
364 spin_unlock_irqrestore(&qp->r_lock, ps->flags); in make_rc_ack()
365 spin_lock_irqsave(&qp->s_lock, ps->flags); in make_rc_ack()
367 qp->s_ack_state = OP(ACKNOWLEDGE); in make_rc_ack()
373 qp->s_flags &= ~(RVT_S_RESP_PENDING in make_rc_ack()
388 int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) in hfi1_make_rc_req() argument
390 struct hfi1_qp_priv *priv = qp->priv; in hfi1_make_rc_req()
391 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device); in hfi1_make_rc_req()
401 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT); in hfi1_make_rc_req()
402 u32 pmtu = qp->pmtu; in hfi1_make_rc_req()
409 trace_hfi1_sender_make_rc_req(qp); in hfi1_make_rc_req()
410 lockdep_assert_held(&qp->s_lock); in hfi1_make_rc_req()
411 ps->s_txreq = get_txreq(ps->dev, qp); in hfi1_make_rc_req()
418 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) in hfi1_make_rc_req()
425 if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && in hfi1_make_rc_req()
426 (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr)))) in hfi1_make_rc_req()
433 if ((qp->s_flags & RVT_S_RESP_PENDING) && in hfi1_make_rc_req()
434 make_rc_ack(dev, qp, ohdr, ps)) in hfi1_make_rc_req()
437 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { in hfi1_make_rc_req()
438 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) in hfi1_make_rc_req()
441 if (qp->s_last == READ_ONCE(qp->s_head)) in hfi1_make_rc_req()
445 qp->s_flags |= RVT_S_WAIT_DMA; in hfi1_make_rc_req()
448 clear_ahg(qp); in hfi1_make_rc_req()
449 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_make_rc_req()
450 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in hfi1_make_rc_req()
456 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK | HFI1_S_WAIT_HALT)) in hfi1_make_rc_req()
459 if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) { in hfi1_make_rc_req()
460 if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { in hfi1_make_rc_req()
461 qp->s_flags |= RVT_S_WAIT_PSN; in hfi1_make_rc_req()
464 qp->s_sending_psn = qp->s_psn; in hfi1_make_rc_req()
465 qp->s_sending_hpsn = qp->s_psn - 1; in hfi1_make_rc_req()
469 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in hfi1_make_rc_req()
471 switch (qp->s_state) { in hfi1_make_rc_req()
473 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) in hfi1_make_rc_req()
483 if (qp->s_cur == qp->s_tail) { in hfi1_make_rc_req()
485 if (qp->s_tail == READ_ONCE(qp->s_head)) { in hfi1_make_rc_req()
486 clear_ahg(qp); in hfi1_make_rc_req()
496 qp->s_num_rd_atomic && in hfi1_make_rc_req()
498 priv->pending_tid_r_segs < qp->s_num_rd_atomic)) { in hfi1_make_rc_req()
499 qp->s_flags |= RVT_S_WAIT_FENCE; in hfi1_make_rc_req()
511 if (qp->s_last != qp->s_cur) in hfi1_make_rc_req()
513 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
514 qp->s_cur = 0; in hfi1_make_rc_req()
515 if (++qp->s_tail == qp->s_size) in hfi1_make_rc_req()
516 qp->s_tail = 0; in hfi1_make_rc_req()
520 qp, in hfi1_make_rc_req()
524 rvt_send_complete(qp, wqe, in hfi1_make_rc_req()
528 atomic_dec(&qp->local_ops_pending); in hfi1_make_rc_req()
533 qp->s_psn = wqe->psn; in hfi1_make_rc_req()
541 ss = &qp->s_sge; in hfi1_make_rc_req()
542 bth2 = mask_psn(qp->s_psn); in hfi1_make_rc_req()
549 hfi1_tid_rdma_wqe_interlock(qp, wqe)) in hfi1_make_rc_req()
557 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
560 qp->s_state = OP(SEND_FIRST); in hfi1_make_rc_req()
565 qp->s_state = OP(SEND_ONLY); in hfi1_make_rc_req()
567 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in hfi1_make_rc_req()
572 qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE); in hfi1_make_rc_req()
581 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
582 qp->s_cur = 0; in hfi1_make_rc_req()
586 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
587 qp->s_lsn++; in hfi1_make_rc_req()
591 if (!rvt_rc_credit_avail(qp, wqe)) in hfi1_make_rc_req()
602 qp->s_state = OP(RDMA_WRITE_FIRST); in hfi1_make_rc_req()
607 qp->s_state = OP(RDMA_WRITE_ONLY); in hfi1_make_rc_req()
609 qp->s_state = in hfi1_make_rc_req()
618 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
619 qp->s_cur = 0; in hfi1_make_rc_req()
631 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
632 qp->s_lsn++; in hfi1_make_rc_req()
635 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, in hfi1_make_rc_req()
640 priv->s_tid_cur = qp->s_cur; in hfi1_make_rc_req()
642 priv->s_tid_tail = qp->s_cur; in hfi1_make_rc_req()
649 __w = rvt_get_swqe_ptr(qp, priv->s_tid_cur); in hfi1_make_rc_req()
682 priv->s_tid_tail = qp->s_cur; in hfi1_make_rc_req()
686 priv->s_tid_cur = qp->s_cur; in hfi1_make_rc_req()
698 if (priv->s_tid_tail == qp->s_cur && in hfi1_make_rc_req()
704 priv->s_tid_head = qp->s_cur; in hfi1_make_rc_req()
720 trace_hfi1_tid_write_sender_make_req(qp, newreq); in hfi1_make_rc_req()
721 trace_hfi1_tid_req_make_req_write(qp, newreq, in hfi1_make_rc_req()
725 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
726 qp->s_cur = 0; in hfi1_make_rc_req()
734 if (qp->s_num_rd_atomic >= in hfi1_make_rc_req()
735 qp->s_max_rd_atomic) { in hfi1_make_rc_req()
736 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
739 qp->s_num_rd_atomic++; in hfi1_make_rc_req()
740 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
741 qp->s_lsn++; in hfi1_make_rc_req()
748 qp->s_state = OP(RDMA_READ_REQUEST); in hfi1_make_rc_req()
753 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
754 qp->s_cur = 0; in hfi1_make_rc_req()
758 trace_hfi1_tid_read_sender_make_req(qp, newreq); in hfi1_make_rc_req()
761 trace_hfi1_tid_req_make_req_read(qp, newreq, in hfi1_make_rc_req()
765 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
777 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { in hfi1_make_rc_req()
778 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
792 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
793 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
794 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
795 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
796 qp->s_len = wqe->length; in hfi1_make_rc_req()
810 req->s_next_psn = qp->s_psn; in hfi1_make_rc_req()
814 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, in hfi1_make_rc_req()
821 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
822 qp->s_lsn++; in hfi1_make_rc_req()
827 ++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
828 qp->s_cur = 0; in hfi1_make_rc_req()
837 if (qp->s_num_rd_atomic >= in hfi1_make_rc_req()
838 qp->s_max_rd_atomic) { in hfi1_make_rc_req()
839 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
842 qp->s_num_rd_atomic++; in hfi1_make_rc_req()
845 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in hfi1_make_rc_req()
846 qp->s_lsn++; in hfi1_make_rc_req()
849 qp->s_state = OP(COMPARE_SWAP); in hfi1_make_rc_req()
855 qp->s_state = OP(FETCH_ADD); in hfi1_make_rc_req()
868 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
869 qp->s_cur = 0; in hfi1_make_rc_req()
876 qp->s_sge.sge = wqe->sg_list[0]; in hfi1_make_rc_req()
877 qp->s_sge.sg_list = wqe->sg_list + 1; in hfi1_make_rc_req()
878 qp->s_sge.num_sge = wqe->wr.num_sge; in hfi1_make_rc_req()
879 qp->s_sge.total_len = wqe->length; in hfi1_make_rc_req()
880 qp->s_len = wqe->length; in hfi1_make_rc_req()
883 qp->s_tail++; in hfi1_make_rc_req()
884 if (qp->s_tail >= qp->s_size) in hfi1_make_rc_req()
885 qp->s_tail = 0; in hfi1_make_rc_req()
889 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
891 qp->s_psn = req->s_next_psn; in hfi1_make_rc_req()
893 qp->s_psn++; in hfi1_make_rc_req()
906 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
909 qp->s_state = OP(SEND_MIDDLE); in hfi1_make_rc_req()
912 bth2 = mask_psn(qp->s_psn++); in hfi1_make_rc_req()
913 ss = &qp->s_sge; in hfi1_make_rc_req()
914 len = qp->s_len; in hfi1_make_rc_req()
921 qp->s_state = OP(SEND_LAST); in hfi1_make_rc_req()
923 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in hfi1_make_rc_req()
928 qp->s_state = OP(SEND_LAST_WITH_INVALIDATE); in hfi1_make_rc_req()
936 qp->s_cur++; in hfi1_make_rc_req()
937 if (qp->s_cur >= qp->s_size) in hfi1_make_rc_req()
938 qp->s_cur = 0; in hfi1_make_rc_req()
951 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in hfi1_make_rc_req()
954 qp->s_state = OP(RDMA_WRITE_MIDDLE); in hfi1_make_rc_req()
957 bth2 = mask_psn(qp->s_psn++); in hfi1_make_rc_req()
958 ss = &qp->s_sge; in hfi1_make_rc_req()
959 len = qp->s_len; in hfi1_make_rc_req()
966 qp->s_state = OP(RDMA_WRITE_LAST); in hfi1_make_rc_req()
968 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in hfi1_make_rc_req()
976 qp->s_cur++; in hfi1_make_rc_req()
977 if (qp->s_cur >= qp->s_size) in hfi1_make_rc_req()
978 qp->s_cur = 0; in hfi1_make_rc_req()
991 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu; in hfi1_make_rc_req()
998 qp->s_state = OP(RDMA_READ_REQUEST); in hfi1_make_rc_req()
1000 bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK; in hfi1_make_rc_req()
1001 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1004 qp->s_cur++; in hfi1_make_rc_req()
1005 if (qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1006 qp->s_cur = 0; in hfi1_make_rc_req()
1019 req->comp_seg = delta_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1023 bth2 = mask_psn(qp->s_psn); in hfi1_make_rc_req()
1024 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1026 qp->s_psn = wqe->lpsn + 1; in hfi1_make_rc_req()
1028 qp->s_state = TID_OP(WRITE_REQ); in hfi1_make_rc_req()
1030 priv->s_tid_cur = qp->s_cur; in hfi1_make_rc_req()
1031 if (++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1032 qp->s_cur = 0; in hfi1_make_rc_req()
1033 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1048 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps; in hfi1_make_rc_req()
1057 hfi1_tid_rdma_restart_req(qp, wqe, &bth2); in hfi1_make_rc_req()
1064 hfi1_kern_clear_hw_flow(priv->rcd, qp); in hfi1_make_rc_req()
1066 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR); in hfi1_make_rc_req()
1085 ++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1086 qp->s_cur = 0; in hfi1_make_rc_req()
1087 qp->s_psn = req->s_next_psn; in hfi1_make_rc_req()
1088 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1093 delta = cmp_psn(qp->s_psn, wqe->psn); in hfi1_make_rc_req()
1100 qp->s_cur == qp->s_tail) { in hfi1_make_rc_req()
1101 qp->s_state = OP(RDMA_READ_REQUEST); in hfi1_make_rc_req()
1102 if (delta == 0 || qp->s_cur == qp->s_tail) in hfi1_make_rc_req()
1109 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) { in hfi1_make_rc_req()
1110 qp->s_flags |= RVT_S_WAIT_RDMAR; in hfi1_make_rc_req()
1118 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1, in hfi1_make_rc_req()
1128 ++qp->s_cur == qp->s_size) in hfi1_make_rc_req()
1129 qp->s_cur = 0; in hfi1_make_rc_req()
1130 qp->s_psn = req->s_next_psn; in hfi1_make_rc_req()
1131 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode, in hfi1_make_rc_req()
1135 qp->s_sending_hpsn = bth2; in hfi1_make_rc_req()
1140 if (qp->s_flags & RVT_S_SEND_ONE) { in hfi1_make_rc_req()
1141 qp->s_flags &= ~RVT_S_SEND_ONE; in hfi1_make_rc_req()
1142 qp->s_flags |= RVT_S_WAIT_ACK; in hfi1_make_rc_req()
1145 qp->s_len -= len; in hfi1_make_rc_req()
1151 qp, in hfi1_make_rc_req()
1153 bth0 | (qp->s_state << 24), in hfi1_make_rc_req()
1170 qp->s_flags &= ~RVT_S_BUSY; in hfi1_make_rc_req()
1180 static inline void hfi1_make_bth_aeth(struct rvt_qp *qp, in hfi1_make_bth_aeth() argument
1184 if (qp->r_nak_state) in hfi1_make_bth_aeth()
1185 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in hfi1_make_bth_aeth()
1186 (qp->r_nak_state << in hfi1_make_bth_aeth()
1189 ohdr->u.aeth = rvt_compute_aeth(qp); in hfi1_make_bth_aeth()
1192 ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn); in hfi1_make_bth_aeth()
1193 ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn)); in hfi1_make_bth_aeth()
1198 struct rvt_qp *qp = packet->qp; in hfi1_queue_rc_ack() local
1202 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_queue_rc_ack()
1203 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in hfi1_queue_rc_ack()
1207 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; in hfi1_queue_rc_ack()
1208 qp->s_nak_state = qp->r_nak_state; in hfi1_queue_rc_ack()
1209 qp->s_ack_psn = qp->r_ack_psn; in hfi1_queue_rc_ack()
1211 qp->s_flags |= RVT_S_ECN; in hfi1_queue_rc_ack()
1214 hfi1_schedule_send(qp); in hfi1_queue_rc_ack()
1216 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_queue_rc_ack()
1225 struct rvt_qp *qp = packet->qp; in hfi1_make_rc_ack_9B() local
1239 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) { in hfi1_make_rc_ack_9B()
1241 rdma_ah_read_grh(&qp->remote_ah_attr), in hfi1_make_rc_ack_9B()
1250 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); in hfi1_make_rc_ack_9B()
1253 (rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) << in hfi1_make_rc_ack_9B()
1257 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B), in hfi1_make_rc_ack_9B()
1258 ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr)); in hfi1_make_rc_ack_9B()
1261 if (qp->s_mig_state == IB_MIG_MIGRATED) in hfi1_make_rc_ack_9B()
1269 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); in hfi1_make_rc_ack_9B()
1278 struct rvt_qp *qp = packet->qp; in hfi1_make_rc_ack_16B() local
1296 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) && in hfi1_make_rc_ack_16B()
1297 hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) { in hfi1_make_rc_ack_16B()
1299 rdma_ah_read_grh(&qp->remote_ah_attr), in hfi1_make_rc_ack_16B()
1307 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); in hfi1_make_rc_ack_16B()
1313 (rdma_ah_get_path_bits(&qp->remote_ah_attr) & in hfi1_make_rc_ack_16B()
1315 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), in hfi1_make_rc_ack_16B()
1320 if (qp->s_mig_state == IB_MIG_MIGRATED) in hfi1_make_rc_ack_16B()
1322 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1); in hfi1_make_rc_ack_16B()
1347 struct rvt_qp *qp = packet->qp; in hfi1_send_rc_ack() local
1349 struct hfi1_qp_priv *priv = qp->priv; in hfi1_send_rc_ack()
1351 u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)]; in hfi1_send_rc_ack()
1360 qp->r_adefered = 0; in hfi1_send_rc_ack()
1363 if (qp->s_flags & RVT_S_RESP_PENDING) { in hfi1_send_rc_ack()
1369 if (qp->s_rdma_ack_cnt) { in hfi1_send_rc_ack()
1383 pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, in hfi1_send_rc_ack()
1396 trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), in hfi1_send_rc_ack()
1417 static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn, in update_num_rd_atomic() argument
1425 qp->s_num_rd_atomic++; in update_num_rd_atomic()
1428 struct hfi1_qp_priv *priv = qp->priv; in update_num_rd_atomic()
1436 qp->s_num_rd_atomic += req->ack_pending; in update_num_rd_atomic()
1437 trace_hfi1_tid_req_update_num_rd_atomic(qp, 0, in update_num_rd_atomic()
1444 qp->s_num_rd_atomic += req->total_segs; in update_num_rd_atomic()
1458 static void reset_psn(struct rvt_qp *qp, u32 psn) in reset_psn() argument
1460 u32 n = qp->s_acked; in reset_psn()
1461 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
1463 struct hfi1_qp_priv *priv = qp->priv; in reset_psn()
1465 lockdep_assert_held(&qp->s_lock); in reset_psn()
1466 qp->s_cur = n; in reset_psn()
1469 qp->s_num_rd_atomic = 0; in reset_psn()
1476 qp->s_state = OP(SEND_LAST); in reset_psn()
1479 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1485 if (++n == qp->s_size) in reset_psn()
1487 if (n == qp->s_tail) in reset_psn()
1489 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
1493 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in reset_psn()
1496 qp->s_cur = n; in reset_psn()
1502 qp->s_state = OP(SEND_LAST); in reset_psn()
1506 update_num_rd_atomic(qp, psn, wqe); in reset_psn()
1518 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
1523 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
1527 qp->s_state = TID_OP(WRITE_RESP); in reset_psn()
1531 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
1535 qp->s_state = TID_OP(READ_RESP); in reset_psn()
1543 qp->s_state = OP(SEND_LAST); in reset_psn()
1547 qp->s_psn = psn; in reset_psn()
1553 if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) && in reset_psn()
1554 (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) in reset_psn()
1555 qp->s_flags |= RVT_S_WAIT_PSN; in reset_psn()
1556 qp->s_flags &= ~HFI1_S_AHG_VALID; in reset_psn()
1557 trace_hfi1_sender_reset_psn(qp); in reset_psn()
1564 void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait) in hfi1_restart_rc() argument
1566 struct hfi1_qp_priv *priv = qp->priv; in hfi1_restart_rc()
1567 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_restart_rc()
1570 lockdep_assert_held(&qp->r_lock); in hfi1_restart_rc()
1571 lockdep_assert_held(&qp->s_lock); in hfi1_restart_rc()
1572 trace_hfi1_sender_restart_rc(qp); in hfi1_restart_rc()
1573 if (qp->s_retry == 0) { in hfi1_restart_rc()
1574 if (qp->s_mig_state == IB_MIG_ARMED) { in hfi1_restart_rc()
1575 hfi1_migrate_qp(qp); in hfi1_restart_rc()
1576 qp->s_retry = qp->s_retry_cnt; in hfi1_restart_rc()
1577 } else if (qp->s_last == qp->s_acked) { in hfi1_restart_rc()
1584 to_iport(qp->ibqp.device, qp->port_num); in hfi1_restart_rc()
1590 opfn_conn_reply(qp, priv->opfn.curr); in hfi1_restart_rc()
1591 wqe = do_rc_completion(qp, wqe, ibp); in hfi1_restart_rc()
1592 qp->s_flags &= ~RVT_S_WAIT_ACK; in hfi1_restart_rc()
1594 trace_hfi1_tid_write_sender_restart_rc(qp, 0); in hfi1_restart_rc()
1600 hfi1_kern_clear_hw_flow(priv->rcd, qp); in hfi1_restart_rc()
1603 hfi1_trdma_send_complete(qp, wqe, in hfi1_restart_rc()
1605 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in hfi1_restart_rc()
1612 qp->s_retry--; in hfi1_restart_rc()
1615 ibp = to_iport(qp->ibqp.device, qp->port_num); in hfi1_restart_rc()
1620 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); in hfi1_restart_rc()
1622 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | in hfi1_restart_rc()
1626 qp->s_flags |= RVT_S_SEND_ONE; in hfi1_restart_rc()
1627 reset_psn(qp, psn); in hfi1_restart_rc()
1635 static void reset_sending_psn(struct rvt_qp *qp, u32 psn) in reset_sending_psn() argument
1638 u32 n = qp->s_last; in reset_sending_psn()
1640 lockdep_assert_held(&qp->s_lock); in reset_sending_psn()
1643 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
1648 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
1650 qp->s_sending_psn = psn + 1; in reset_sending_psn()
1653 if (++n == qp->s_size) in reset_sending_psn()
1655 if (n == qp->s_tail) in reset_sending_psn()
1672 void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah) in hfi1_rc_verbs_aborted() argument
1687 qp->s_flags |= RVT_S_SEND_ONE; in hfi1_rc_verbs_aborted()
1693 void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) in hfi1_rc_send_complete() argument
1696 struct hfi1_qp_priv *priv = qp->priv; in hfi1_rc_send_complete()
1702 lockdep_assert_held(&qp->s_lock); in hfi1_rc_send_complete()
1703 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) in hfi1_rc_send_complete()
1712 WARN_ON(!qp->s_rdma_ack_cnt); in hfi1_rc_send_complete()
1713 qp->s_rdma_ack_cnt--; in hfi1_rc_send_complete()
1725 reset_sending_psn(qp, psn); in hfi1_rc_send_complete()
1740 wqe = rvt_get_swqe_ptr(qp, tail); in hfi1_rc_send_complete()
1744 tail = qp->s_size - 1; in hfi1_rc_send_complete()
1749 head = qp->s_tail; in hfi1_rc_send_complete()
1750 tail = qp->s_acked; in hfi1_rc_send_complete()
1760 !(qp->s_flags & in hfi1_rc_send_complete()
1762 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { in hfi1_rc_send_complete()
1764 rvt_add_retry_timer_ext(qp, priv->timeout_shift); in hfi1_rc_send_complete()
1766 rvt_add_retry_timer(qp); in hfi1_rc_send_complete()
1775 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) { in hfi1_rc_send_complete()
1781 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in hfi1_rc_send_complete()
1785 hfi1_add_tid_retry_timer(qp); in hfi1_rc_send_complete()
1788 while (qp->s_last != qp->s_acked) { in hfi1_rc_send_complete()
1789 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in hfi1_rc_send_complete()
1790 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && in hfi1_rc_send_complete()
1791 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) in hfi1_rc_send_complete()
1793 trdma_clean_swqe(qp, wqe); in hfi1_rc_send_complete()
1794 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in hfi1_rc_send_complete()
1795 rvt_qp_complete_swqe(qp, in hfi1_rc_send_complete()
1804 trace_hfi1_sendcomplete(qp, psn); in hfi1_rc_send_complete()
1805 if (qp->s_flags & RVT_S_WAIT_PSN && in hfi1_rc_send_complete()
1806 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in hfi1_rc_send_complete()
1807 qp->s_flags &= ~RVT_S_WAIT_PSN; in hfi1_rc_send_complete()
1808 qp->s_sending_psn = qp->s_psn; in hfi1_rc_send_complete()
1809 qp->s_sending_hpsn = qp->s_psn - 1; in hfi1_rc_send_complete()
1810 hfi1_schedule_send(qp); in hfi1_rc_send_complete()
1814 static inline void update_last_psn(struct rvt_qp *qp, u32 psn) in update_last_psn() argument
1816 qp->s_last_psn = psn; in update_last_psn()
1824 struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, in do_rc_completion() argument
1828 struct hfi1_qp_priv *priv = qp->priv; in do_rc_completion()
1830 lockdep_assert_held(&qp->s_lock); in do_rc_completion()
1836 trace_hfi1_rc_completion(qp, wqe->lpsn); in do_rc_completion()
1837 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
1838 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in do_rc_completion()
1839 trdma_clean_swqe(qp, wqe); in do_rc_completion()
1840 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); in do_rc_completion()
1841 rvt_qp_complete_swqe(qp, in do_rc_completion()
1855 u8 sl = rdma_ah_get_sl(&qp->remote_ah_attr); in do_rc_completion()
1860 engine = qp_to_sdma_engine(qp, sc5); in do_rc_completion()
1865 qp->s_retry = qp->s_retry_cnt; in do_rc_completion()
1875 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
1882 if (qp->s_acked == qp->s_cur) { in do_rc_completion()
1883 if (++qp->s_cur >= qp->s_size) in do_rc_completion()
1884 qp->s_cur = 0; in do_rc_completion()
1885 qp->s_acked = qp->s_cur; in do_rc_completion()
1886 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
1887 if (qp->s_acked != qp->s_tail) { in do_rc_completion()
1888 qp->s_state = OP(SEND_LAST); in do_rc_completion()
1889 qp->s_psn = wqe->psn; in do_rc_completion()
1892 if (++qp->s_acked >= qp->s_size) in do_rc_completion()
1893 qp->s_acked = 0; in do_rc_completion()
1894 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) in do_rc_completion()
1895 qp->s_draining = 0; in do_rc_completion()
1896 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
1900 hfi1_schedule_send(qp); in do_rc_completion()
1905 static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd) in set_restart_qp() argument
1908 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { in set_restart_qp()
1909 qp->r_flags |= RVT_R_RDMAR_SEQ; in set_restart_qp()
1910 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); in set_restart_qp()
1911 if (list_empty(&qp->rspwait)) { in set_restart_qp()
1912 qp->r_flags |= RVT_R_RSP_SEND; in set_restart_qp()
1913 rvt_get_qp(qp); in set_restart_qp()
1914 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in set_restart_qp()
1930 static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn, in update_qp_retry_state() argument
1933 struct hfi1_qp_priv *qpriv = qp->priv; in update_qp_retry_state()
1935 qp->s_psn = psn + 1; in update_qp_retry_state()
1943 qp->s_cur = qpriv->s_tid_cur + 1; in update_qp_retry_state()
1944 if (qp->s_cur >= qp->s_size) in update_qp_retry_state()
1945 qp->s_cur = 0; in update_qp_retry_state()
1946 qp->s_state = TID_OP(WRITE_REQ); in update_qp_retry_state()
1948 qp->s_cur = qpriv->s_tid_cur; in update_qp_retry_state()
1949 qp->s_state = TID_OP(WRITE_RESP); in update_qp_retry_state()
1964 int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, in do_rc_ack() argument
1969 struct hfi1_qp_priv *qpriv = qp->priv; in do_rc_ack()
1976 lockdep_assert_held(&qp->s_lock); in do_rc_ack()
1986 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
2023 (delta_psn(psn, qp->s_last_psn) != 1))) { in do_rc_ack()
2024 set_restart_qp(qp, rcd); in do_rc_ack()
2037 opfn_conn_reply(qp, val); in do_rc_ack()
2039 if (qp->s_num_rd_atomic && in do_rc_ack()
2043 qp->s_num_rd_atomic--; in do_rc_ack()
2045 if ((qp->s_flags & RVT_S_WAIT_FENCE) && in do_rc_ack()
2046 !qp->s_num_rd_atomic) { in do_rc_ack()
2047 qp->s_flags &= ~(RVT_S_WAIT_FENCE | in do_rc_ack()
2049 hfi1_schedule_send(qp); in do_rc_ack()
2050 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { in do_rc_ack()
2051 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | in do_rc_ack()
2053 hfi1_schedule_send(qp); in do_rc_ack()
2064 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
2065 if (qp->s_acked == qp->s_tail) in do_rc_ack()
2069 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe); in do_rc_ack()
2070 trace_hfi1_sender_do_rc_ack(qp); in do_rc_ack()
2076 rvt_mod_retry_timer_ext(qp, in do_rc_ack()
2079 rvt_stop_rc_timers(qp); in do_rc_ack()
2080 } else if (qp->s_acked != qp->s_tail) { in do_rc_ack()
2084 __w = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur); in do_rc_ack()
2104 if (cmp_psn(psn, qp->s_last_psn + 1)) { in do_rc_ack()
2105 set_restart_qp(qp, rcd); in do_rc_ack()
2112 if (qp->s_cur != qp->s_tail && in do_rc_ack()
2113 cmp_psn(qp->s_psn, psn) <= 0) in do_rc_ack()
2114 update_qp_retry_state(qp, psn, in do_rc_ack()
2118 rvt_mod_retry_timer(qp); in do_rc_ack()
2120 rvt_stop_rc_timers(qp); in do_rc_ack()
2126 rvt_mod_retry_timer(qp); in do_rc_ack()
2132 if (cmp_psn(qp->s_psn, psn) <= 0) in do_rc_ack()
2133 reset_psn(qp, psn + 1); in do_rc_ack()
2137 rvt_stop_rc_timers(qp); in do_rc_ack()
2138 if (cmp_psn(qp->s_psn, psn) <= 0) { in do_rc_ack()
2139 qp->s_state = OP(SEND_LAST); in do_rc_ack()
2140 qp->s_psn = psn + 1; in do_rc_ack()
2143 if (qp->s_flags & RVT_S_WAIT_ACK) { in do_rc_ack()
2144 qp->s_flags &= ~RVT_S_WAIT_ACK; in do_rc_ack()
2145 hfi1_schedule_send(qp); in do_rc_ack()
2147 rvt_get_credit(qp, aeth); in do_rc_ack()
2148 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
2149 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
2159 update_last_psn(qp, psn); in do_rc_ack()
2164 if (qp->s_acked == qp->s_tail) in do_rc_ack()
2166 if (qp->s_flags & RVT_S_WAIT_RNR) in do_rc_ack()
2168 rdi = ib_to_rvt(qp->ibqp.device); in do_rc_ack()
2171 if (qp->s_rnr_retry == 0) { in do_rc_ack()
2175 if (qp->s_rnr_retry_cnt < 7 && qp->s_rnr_retry_cnt > 0) in do_rc_ack()
2176 qp->s_rnr_retry--; in do_rc_ack()
2186 reset_psn(qp, qp->s_last_psn + 1); in do_rc_ack()
2188 update_last_psn(qp, psn - 1); in do_rc_ack()
2189 reset_psn(qp, psn); in do_rc_ack()
2192 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn); in do_rc_ack()
2193 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); in do_rc_ack()
2194 rvt_stop_rc_timers(qp); in do_rc_ack()
2195 rvt_add_rnr_timer(qp, aeth); in do_rc_ack()
2199 if (qp->s_acked == qp->s_tail) in do_rc_ack()
2202 update_last_psn(qp, psn - 1); in do_rc_ack()
2213 hfi1_restart_rc(qp, psn, 0); in do_rc_ack()
2214 hfi1_schedule_send(qp); in do_rc_ack()
2231 if (qp->s_last == qp->s_acked) { in do_rc_ack()
2233 hfi1_kern_read_tid_flow_free(qp); in do_rc_ack()
2235 hfi1_trdma_send_complete(qp, wqe, status); in do_rc_ack()
2236 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in do_rc_ack()
2244 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
2245 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
2255 rvt_stop_rc_timers(qp); in do_rc_ack()
2263 static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn, in rdma_seq_err() argument
2268 lockdep_assert_held(&qp->s_lock); in rdma_seq_err()
2270 rvt_stop_rc_timers(qp); in rdma_seq_err()
2272 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
2281 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
2285 qp->r_flags |= RVT_R_RDMAR_SEQ; in rdma_seq_err()
2286 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0); in rdma_seq_err()
2287 if (list_empty(&qp->rspwait)) { in rdma_seq_err()
2288 qp->r_flags |= RVT_R_RSP_SEND; in rdma_seq_err()
2289 rvt_get_qp(qp); in rdma_seq_err()
2290 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in rdma_seq_err()
2307 struct rvt_qp *qp = packet->qp; in rc_rcv_resp() local
2317 u32 pmtu = qp->pmtu; in rc_rcv_resp()
2323 spin_lock_irqsave(&qp->s_lock, flags); in rc_rcv_resp()
2324 trace_hfi1_ack(qp, psn); in rc_rcv_resp()
2327 if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0) in rc_rcv_resp()
2331 diff = cmp_psn(psn, qp->s_last_psn); in rc_rcv_resp()
2337 rvt_get_credit(qp, aeth); in rc_rcv_resp()
2346 if (qp->r_flags & RVT_R_RDMAR_SEQ) { in rc_rcv_resp()
2347 if (cmp_psn(psn, qp->s_last_psn + 1) != 0) in rc_rcv_resp()
2349 qp->r_flags &= ~RVT_R_RDMAR_SEQ; in rc_rcv_resp()
2352 if (unlikely(qp->s_acked == qp->s_tail)) in rc_rcv_resp()
2354 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2366 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || in rc_rcv_resp()
2369 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2377 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in rc_rcv_resp()
2383 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) in rc_rcv_resp()
2390 if (unlikely(pmtu >= qp->s_rdma_read_len)) in rc_rcv_resp()
2397 rvt_mod_retry_timer(qp); in rc_rcv_resp()
2398 if (qp->s_flags & RVT_S_WAIT_ACK) { in rc_rcv_resp()
2399 qp->s_flags &= ~RVT_S_WAIT_ACK; in rc_rcv_resp()
2400 hfi1_schedule_send(qp); in rc_rcv_resp()
2404 qp->s_retry = qp->s_retry_cnt; in rc_rcv_resp()
2410 qp->s_rdma_read_len -= pmtu; in rc_rcv_resp()
2411 update_last_psn(qp, psn); in rc_rcv_resp()
2412 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_resp()
2413 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in rc_rcv_resp()
2419 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) in rc_rcv_resp()
2432 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rc_rcv_resp()
2433 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in rc_rcv_resp()
2439 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1))) in rc_rcv_resp()
2451 if (unlikely(tlen != qp->s_rdma_read_len)) in rc_rcv_resp()
2454 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in rc_rcv_resp()
2456 WARN_ON(qp->s_rdma_read_sge.num_sge); in rc_rcv_resp()
2457 (void)do_rc_ack(qp, aeth, psn, in rc_rcv_resp()
2468 rdma_seq_err(qp, ibp, psn, rcd); in rc_rcv_resp()
2474 if (qp->s_last == qp->s_acked) { in rc_rcv_resp()
2475 rvt_send_complete(qp, wqe, status); in rc_rcv_resp()
2476 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in rc_rcv_resp()
2479 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_resp()
2484 static inline void rc_cancel_ack(struct rvt_qp *qp) in rc_cancel_ack() argument
2486 qp->r_adefered = 0; in rc_cancel_ack()
2487 if (list_empty(&qp->rspwait)) in rc_cancel_ack()
2489 list_del_init(&qp->rspwait); in rc_cancel_ack()
2490 qp->r_flags &= ~RVT_R_RSP_NAK; in rc_cancel_ack()
2491 rvt_put_qp(qp); in rc_cancel_ack()
2511 struct rvt_qp *qp, u32 opcode, u32 psn, in rc_rcv_error() argument
2521 trace_hfi1_rcv_error(qp, psn); in rc_rcv_error()
2528 if (!qp->r_nak_state) { in rc_rcv_error()
2530 qp->r_nak_state = IB_NAK_PSN_ERROR; in rc_rcv_error()
2532 qp->r_ack_psn = qp->r_psn; in rc_rcv_error()
2538 rc_defered_ack(rcd, qp); in rc_rcv_error()
2563 spin_lock_irqsave(&qp->s_lock, flags); in rc_rcv_error()
2565 e = find_prev_entry(qp, psn, &prev, &mra, &old_req); in rc_rcv_error()
2588 offset = delta_psn(psn, e->psn) * qp->pmtu; in rc_rcv_error()
2598 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in rc_rcv_error()
2610 if (qp->s_acked_ack_queue == qp->s_tail_ack_queue) in rc_rcv_error()
2611 qp->s_acked_ack_queue = prev; in rc_rcv_error()
2612 qp->s_tail_ack_queue = prev; in rc_rcv_error()
2625 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) in rc_rcv_error()
2626 qp->s_acked_ack_queue = prev; in rc_rcv_error()
2627 qp->s_tail_ack_queue = prev; in rc_rcv_error()
2642 if (mra == qp->r_head_ack_queue) { in rc_rcv_error()
2643 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_error()
2644 qp->r_nak_state = 0; in rc_rcv_error()
2645 qp->r_ack_psn = qp->r_psn - 1; in rc_rcv_error()
2653 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue) in rc_rcv_error()
2654 qp->s_acked_ack_queue = mra; in rc_rcv_error()
2655 qp->s_tail_ack_queue = mra; in rc_rcv_error()
2658 qp->s_ack_state = OP(ACKNOWLEDGE); in rc_rcv_error()
2659 qp->s_flags |= RVT_S_RESP_PENDING; in rc_rcv_error()
2660 qp->r_nak_state = 0; in rc_rcv_error()
2661 hfi1_schedule_send(qp); in rc_rcv_error()
2664 spin_unlock_irqrestore(&qp->s_lock, flags); in rc_rcv_error()
2768 struct rvt_qp *qp = packet->qp; in hfi1_rc_rcv() local
2769 struct hfi1_qp_priv *qpriv = qp->priv; in hfi1_rc_rcv()
2777 u32 pmtu = qp->pmtu; in hfi1_rc_rcv()
2786 lockdep_assert_held(&qp->r_lock); in hfi1_rc_rcv()
2791 fecn = process_ecn(qp, packet); in hfi1_rc_rcv()
2792 opfn_trigger_conn_request(qp, be32_to_cpu(ohdr->bth[1])); in hfi1_rc_rcv()
2807 diff = delta_psn(psn, qp->r_psn); in hfi1_rc_rcv()
2809 if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) in hfi1_rc_rcv()
2815 switch (qp->r_state) { in hfi1_rc_rcv()
2850 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in hfi1_rc_rcv()
2851 rvt_comm_est(qp); in hfi1_rc_rcv()
2856 ret = rvt_get_rwqe(qp, false); in hfi1_rc_rcv()
2861 qp->r_rcv_len = 0; in hfi1_rc_rcv()
2874 qp->r_rcv_len += pmtu; in hfi1_rc_rcv()
2875 if (unlikely(qp->r_rcv_len > qp->r_len)) in hfi1_rc_rcv()
2877 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); in hfi1_rc_rcv()
2882 ret = rvt_get_rwqe(qp, true); in hfi1_rc_rcv()
2892 ret = rvt_get_rwqe(qp, false); in hfi1_rc_rcv()
2897 qp->r_rcv_len = 0; in hfi1_rc_rcv()
2911 if (rvt_invalidate_rkey(qp, rkey)) in hfi1_rc_rcv()
2917 copy_last = rvt_is_user_qp(qp); in hfi1_rc_rcv()
2930 wc.byte_len = tlen + qp->r_rcv_len; in hfi1_rc_rcv()
2931 if (unlikely(wc.byte_len > qp->r_len)) in hfi1_rc_rcv()
2933 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last); in hfi1_rc_rcv()
2934 rvt_put_ss(&qp->r_sge); in hfi1_rc_rcv()
2935 qp->r_msn++; in hfi1_rc_rcv()
2936 if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) in hfi1_rc_rcv()
2938 wc.wr_id = qp->r_wr_id; in hfi1_rc_rcv()
2945 wc.qp = &qp->ibqp; in hfi1_rc_rcv()
2946 wc.src_qp = qp->remote_qpn; in hfi1_rc_rcv()
2947 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX; in hfi1_rc_rcv()
2959 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); in hfi1_rc_rcv()
2966 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); in hfi1_rc_rcv()
2970 copy_last = rvt_is_user_qp(qp); in hfi1_rc_rcv()
2974 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in hfi1_rc_rcv()
2978 qp->r_len = be32_to_cpu(reth->length); in hfi1_rc_rcv()
2979 qp->r_rcv_len = 0; in hfi1_rc_rcv()
2980 qp->r_sge.sg_list = NULL; in hfi1_rc_rcv()
2981 if (qp->r_len != 0) { in hfi1_rc_rcv()
2987 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, in hfi1_rc_rcv()
2991 qp->r_sge.num_sge = 1; in hfi1_rc_rcv()
2993 qp->r_sge.num_sge = 0; in hfi1_rc_rcv()
2994 qp->r_sge.sge.mr = NULL; in hfi1_rc_rcv()
2995 qp->r_sge.sge.vaddr = NULL; in hfi1_rc_rcv()
2996 qp->r_sge.sge.length = 0; in hfi1_rc_rcv()
2997 qp->r_sge.sge.sge_length = 0; in hfi1_rc_rcv()
3003 ret = rvt_get_rwqe(qp, true); in hfi1_rc_rcv()
3008 rvt_put_ss(&qp->r_sge); in hfi1_rc_rcv()
3020 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in hfi1_rc_rcv()
3022 next = qp->r_head_ack_queue + 1; in hfi1_rc_rcv()
3024 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_rc_rcv()
3026 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv()
3027 if (unlikely(next == qp->s_acked_ack_queue)) { in hfi1_rc_rcv()
3028 if (!qp->s_ack_queue[next].sent) in hfi1_rc_rcv()
3030 update_ack_queue(qp, next); in hfi1_rc_rcv()
3032 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in hfi1_rc_rcv()
3042 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, in hfi1_rc_rcv()
3050 qp->r_psn += rvt_div_mtu(qp, len - 1); in hfi1_rc_rcv()
3060 e->lpsn = qp->r_psn; in hfi1_rc_rcv()
3066 qp->r_msn++; in hfi1_rc_rcv()
3067 qp->r_psn++; in hfi1_rc_rcv()
3068 qp->r_state = opcode; in hfi1_rc_rcv()
3069 qp->r_nak_state = 0; in hfi1_rc_rcv()
3070 qp->r_head_ack_queue = next; in hfi1_rc_rcv()
3071 qpriv->r_tid_alloc = qp->r_head_ack_queue; in hfi1_rc_rcv()
3074 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv()
3076 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv()
3077 hfi1_schedule_send(qp); in hfi1_rc_rcv()
3079 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3095 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) && in hfi1_rc_rcv()
3098 next = qp->r_head_ack_queue + 1; in hfi1_rc_rcv()
3099 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device))) in hfi1_rc_rcv()
3101 spin_lock_irqsave(&qp->s_lock, flags); in hfi1_rc_rcv()
3102 if (unlikely(next == qp->s_acked_ack_queue)) { in hfi1_rc_rcv()
3103 if (!qp->s_ack_queue[next].sent) in hfi1_rc_rcv()
3105 update_ack_queue(qp, next); in hfi1_rc_rcv()
3107 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in hfi1_rc_rcv()
3111 opfn_conn_response(qp, e, ateth); in hfi1_rc_rcv()
3118 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), in hfi1_rc_rcv()
3123 maddr = (atomic64_t *)qp->r_sge.sge.vaddr; in hfi1_rc_rcv()
3127 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr, in hfi1_rc_rcv()
3130 rvt_put_mr(qp->r_sge.sge.mr); in hfi1_rc_rcv()
3131 qp->r_sge.num_sge = 0; in hfi1_rc_rcv()
3137 qp->r_msn++; in hfi1_rc_rcv()
3138 qp->r_psn++; in hfi1_rc_rcv()
3139 qp->r_state = opcode; in hfi1_rc_rcv()
3140 qp->r_nak_state = 0; in hfi1_rc_rcv()
3141 qp->r_head_ack_queue = next; in hfi1_rc_rcv()
3142 qpriv->r_tid_alloc = qp->r_head_ack_queue; in hfi1_rc_rcv()
3145 qp->s_flags |= RVT_S_RESP_PENDING; in hfi1_rc_rcv()
3147 qp->s_flags |= RVT_S_ECN; in hfi1_rc_rcv()
3148 hfi1_schedule_send(qp); in hfi1_rc_rcv()
3150 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3158 qp->r_psn++; in hfi1_rc_rcv()
3159 qp->r_state = opcode; in hfi1_rc_rcv()
3160 qp->r_ack_psn = psn; in hfi1_rc_rcv()
3161 qp->r_nak_state = 0; in hfi1_rc_rcv()
3165 qp->r_adefered >= HFI1_PSN_CREDIT) { in hfi1_rc_rcv()
3166 rc_cancel_ack(qp); in hfi1_rc_rcv()
3169 qp->r_adefered++; in hfi1_rc_rcv()
3170 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3175 qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK; in hfi1_rc_rcv()
3176 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3178 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3182 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in hfi1_rc_rcv()
3183 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; in hfi1_rc_rcv()
3184 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3186 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3190 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3192 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in hfi1_rc_rcv()
3193 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in hfi1_rc_rcv()
3194 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3196 rc_defered_ack(rcd, qp); in hfi1_rc_rcv()
3200 spin_unlock_irqrestore(&qp->s_lock, flags); in hfi1_rc_rcv()
3202 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); in hfi1_rc_rcv()
3203 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in hfi1_rc_rcv()
3204 qp->r_ack_psn = qp->r_psn; in hfi1_rc_rcv()
3212 struct rvt_qp *qp) in hfi1_rc_hdrerr() argument
3227 diff = delta_psn(psn, qp->r_psn); in hfi1_rc_hdrerr()
3228 if (!qp->r_nak_state && diff >= 0) { in hfi1_rc_hdrerr()
3230 qp->r_nak_state = IB_NAK_PSN_ERROR; in hfi1_rc_hdrerr()
3232 qp->r_ack_psn = qp->r_psn; in hfi1_rc_hdrerr()
3241 rc_defered_ack(rcd, qp); in hfi1_rc_hdrerr()