Lines Matching refs:qp
62 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp, in qib_make_rc_ack() argument
72 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_make_rc_ack()
78 switch (qp->s_ack_state) { in qib_make_rc_ack()
81 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
93 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) in qib_make_rc_ack()
94 qp->s_tail_ack_queue = 0; in qib_make_rc_ack()
99 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in qib_make_rc_ack()
100 if (qp->s_flags & RVT_S_ACK_PENDING) in qib_make_rc_ack()
105 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
115 qp->s_tail_ack_queue = qp->r_head_ack_queue; in qib_make_rc_ack()
119 qp->s_rdma_mr = e->rdma_sge.mr; in qib_make_rc_ack()
120 if (qp->s_rdma_mr) in qib_make_rc_ack()
121 rvt_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
122 qp->s_ack_rdma_sge.sge = e->rdma_sge; in qib_make_rc_ack()
123 qp->s_ack_rdma_sge.num_sge = 1; in qib_make_rc_ack()
124 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
127 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in qib_make_rc_ack()
129 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in qib_make_rc_ack()
132 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
134 qp->s_ack_rdma_psn = e->psn; in qib_make_rc_ack()
135 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
138 qp->s_cur_sge = NULL; in qib_make_rc_ack()
140 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in qib_make_rc_ack()
141 ohdr->u.at.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
147 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
151 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in qib_make_rc_ack()
154 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
155 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; in qib_make_rc_ack()
156 if (qp->s_rdma_mr) in qib_make_rc_ack()
157 rvt_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
158 len = qp->s_ack_rdma_sge.sge.sge_length; in qib_make_rc_ack()
162 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
164 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in qib_make_rc_ack()
165 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
168 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
169 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
180 qp->s_ack_state = OP(SEND_ONLY); in qib_make_rc_ack()
181 qp->s_flags &= ~RVT_S_ACK_PENDING; in qib_make_rc_ack()
182 qp->s_cur_sge = NULL; in qib_make_rc_ack()
183 if (qp->s_nak_state) in qib_make_rc_ack()
185 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in qib_make_rc_ack()
186 (qp->s_nak_state << in qib_make_rc_ack()
189 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
193 bth2 = qp->s_ack_psn & QIB_PSN_MASK; in qib_make_rc_ack()
195 qp->s_rdma_ack_cnt++; in qib_make_rc_ack()
196 qp->s_hdrwords = hwords; in qib_make_rc_ack()
197 qp->s_cur_size = len; in qib_make_rc_ack()
198 qib_make_ruc_header(qp, ohdr, bth0, bth2); in qib_make_rc_ack()
202 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_make_rc_ack()
203 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING); in qib_make_rc_ack()
216 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags) in qib_make_rc_req() argument
218 struct qib_qp_priv *priv = qp->priv; in qib_make_rc_req()
219 struct qib_ibdev *dev = to_idev(qp->ibqp.device); in qib_make_rc_req()
227 u32 pmtu = qp->pmtu; in qib_make_rc_req()
233 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) in qib_make_rc_req()
237 if ((qp->s_flags & RVT_S_RESP_PENDING) && in qib_make_rc_req()
238 qib_make_rc_ack(dev, qp, ohdr, pmtu)) in qib_make_rc_req()
241 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { in qib_make_rc_req()
242 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) in qib_make_rc_req()
245 if (qp->s_last == READ_ONCE(qp->s_head)) in qib_make_rc_req()
249 qp->s_flags |= RVT_S_WAIT_DMA; in qib_make_rc_req()
252 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_make_rc_req()
253 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in qib_make_rc_req()
259 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK)) in qib_make_rc_req()
262 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
263 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
264 qp->s_flags |= RVT_S_WAIT_PSN; in qib_make_rc_req()
267 qp->s_sending_psn = qp->s_psn; in qib_make_rc_req()
268 qp->s_sending_hpsn = qp->s_psn - 1; in qib_make_rc_req()
276 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in qib_make_rc_req()
277 switch (qp->s_state) { in qib_make_rc_req()
279 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) in qib_make_rc_req()
289 if (qp->s_cur == qp->s_tail) { in qib_make_rc_req()
291 if (qp->s_tail == READ_ONCE(qp->s_head)) in qib_make_rc_req()
298 qp->s_num_rd_atomic) { in qib_make_rc_req()
299 qp->s_flags |= RVT_S_WAIT_FENCE; in qib_make_rc_req()
303 qp->s_psn = wqe->psn; in qib_make_rc_req()
311 ss = &qp->s_sge; in qib_make_rc_req()
312 bth2 = qp->s_psn & QIB_PSN_MASK; in qib_make_rc_req()
317 if (!rvt_rc_credit_avail(qp, wqe)) in qib_make_rc_req()
320 qp->s_state = OP(SEND_FIRST); in qib_make_rc_req()
325 qp->s_state = OP(SEND_ONLY); in qib_make_rc_req()
327 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
335 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
336 qp->s_cur = 0; in qib_make_rc_req()
340 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
341 qp->s_lsn++; in qib_make_rc_req()
345 if (!rvt_rc_credit_avail(qp, wqe)) in qib_make_rc_req()
355 qp->s_state = OP(RDMA_WRITE_FIRST); in qib_make_rc_req()
360 qp->s_state = OP(RDMA_WRITE_ONLY); in qib_make_rc_req()
362 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
371 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
372 qp->s_cur = 0; in qib_make_rc_req()
381 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
382 qp->s_max_rd_atomic) { in qib_make_rc_req()
383 qp->s_flags |= RVT_S_WAIT_RDMAR; in qib_make_rc_req()
386 qp->s_num_rd_atomic++; in qib_make_rc_req()
387 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
388 qp->s_lsn++; in qib_make_rc_req()
396 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
401 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
402 qp->s_cur = 0; in qib_make_rc_req()
412 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
413 qp->s_max_rd_atomic) { in qib_make_rc_req()
414 qp->s_flags |= RVT_S_WAIT_RDMAR; in qib_make_rc_req()
417 qp->s_num_rd_atomic++; in qib_make_rc_req()
418 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
419 qp->s_lsn++; in qib_make_rc_req()
422 qp->s_state = OP(COMPARE_SWAP); in qib_make_rc_req()
428 qp->s_state = OP(FETCH_ADD); in qib_make_rc_req()
441 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
442 qp->s_cur = 0; in qib_make_rc_req()
448 qp->s_sge.sge = wqe->sg_list[0]; in qib_make_rc_req()
449 qp->s_sge.sg_list = wqe->sg_list + 1; in qib_make_rc_req()
450 qp->s_sge.num_sge = wqe->wr.num_sge; in qib_make_rc_req()
451 qp->s_sge.total_len = wqe->length; in qib_make_rc_req()
452 qp->s_len = wqe->length; in qib_make_rc_req()
454 qp->s_tail++; in qib_make_rc_req()
455 if (qp->s_tail >= qp->s_size) in qib_make_rc_req()
456 qp->s_tail = 0; in qib_make_rc_req()
459 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
461 qp->s_psn++; in qib_make_rc_req()
474 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
477 qp->s_state = OP(SEND_MIDDLE); in qib_make_rc_req()
480 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
481 ss = &qp->s_sge; in qib_make_rc_req()
482 len = qp->s_len; in qib_make_rc_req()
488 qp->s_state = OP(SEND_LAST); in qib_make_rc_req()
490 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
498 qp->s_cur++; in qib_make_rc_req()
499 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
500 qp->s_cur = 0; in qib_make_rc_req()
513 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
516 qp->s_state = OP(RDMA_WRITE_MIDDLE); in qib_make_rc_req()
519 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
520 ss = &qp->s_sge; in qib_make_rc_req()
521 len = qp->s_len; in qib_make_rc_req()
527 qp->s_state = OP(RDMA_WRITE_LAST); in qib_make_rc_req()
529 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
537 qp->s_cur++; in qib_make_rc_req()
538 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
539 qp->s_cur = 0; in qib_make_rc_req()
552 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in qib_make_rc_req()
558 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
560 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; in qib_make_rc_req()
561 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
564 qp->s_cur++; in qib_make_rc_req()
565 if (qp->s_cur == qp->s_size) in qib_make_rc_req()
566 qp->s_cur = 0; in qib_make_rc_req()
569 qp->s_sending_hpsn = bth2; in qib_make_rc_req()
573 if (qp->s_flags & RVT_S_SEND_ONE) { in qib_make_rc_req()
574 qp->s_flags &= ~RVT_S_SEND_ONE; in qib_make_rc_req()
575 qp->s_flags |= RVT_S_WAIT_ACK; in qib_make_rc_req()
578 qp->s_len -= len; in qib_make_rc_req()
579 qp->s_hdrwords = hwords; in qib_make_rc_req()
580 qp->s_cur_sge = ss; in qib_make_rc_req()
581 qp->s_cur_size = len; in qib_make_rc_req()
582 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); in qib_make_rc_req()
586 qp->s_flags &= ~RVT_S_BUSY; in qib_make_rc_req()
598 void qib_send_rc_ack(struct rvt_qp *qp) in qib_send_rc_ack() argument
600 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); in qib_send_rc_ack()
601 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_send_rc_ack()
614 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
616 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_send_rc_ack()
620 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt) in qib_send_rc_ack()
628 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & in qib_send_rc_ack()
631 rdma_ah_read_grh(&qp->remote_ah_attr), in qib_send_rc_ack()
637 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); in qib_send_rc_ack()
638 if (qp->s_mig_state == IB_MIG_MIGRATED) in qib_send_rc_ack()
640 if (qp->r_nak_state) in qib_send_rc_ack()
641 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in qib_send_rc_ack()
642 (qp->r_nak_state << in qib_send_rc_ack()
645 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_send_rc_ack()
646 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 | in qib_send_rc_ack()
647 rdma_ah_get_sl(&qp->remote_ah_attr) << 4; in qib_send_rc_ack()
649 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); in qib_send_rc_ack()
652 rdma_ah_get_path_bits(&qp->remote_ah_attr)); in qib_send_rc_ack()
654 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); in qib_send_rc_ack()
655 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); in qib_send_rc_ack()
657 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
664 qp->s_srate, lrh0 >> 12); in qib_send_rc_ack()
677 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
712 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { in qib_send_rc_ack()
714 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; in qib_send_rc_ack()
715 qp->s_nak_state = qp->r_nak_state; in qib_send_rc_ack()
716 qp->s_ack_psn = qp->r_ack_psn; in qib_send_rc_ack()
719 qib_schedule_send(qp); in qib_send_rc_ack()
722 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
736 static void reset_psn(struct rvt_qp *qp, u32 psn) in reset_psn() argument
738 u32 n = qp->s_acked; in reset_psn()
739 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
742 qp->s_cur = n; in reset_psn()
749 qp->s_state = OP(SEND_LAST); in reset_psn()
758 if (++n == qp->s_size) in reset_psn()
760 if (n == qp->s_tail) in reset_psn()
762 wqe = rvt_get_swqe_ptr(qp, n); in reset_psn()
766 qp->s_cur = n; in reset_psn()
772 qp->s_state = OP(SEND_LAST); in reset_psn()
786 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
791 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
795 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
803 qp->s_state = OP(SEND_LAST); in reset_psn()
806 qp->s_psn = psn; in reset_psn()
812 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && in reset_psn()
813 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) in reset_psn()
814 qp->s_flags |= RVT_S_WAIT_PSN; in reset_psn()
821 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait) in qib_restart_rc() argument
823 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_restart_rc()
826 if (qp->s_retry == 0) { in qib_restart_rc()
827 if (qp->s_mig_state == IB_MIG_ARMED) { in qib_restart_rc()
828 qib_migrate_qp(qp); in qib_restart_rc()
829 qp->s_retry = qp->s_retry_cnt; in qib_restart_rc()
830 } else if (qp->s_last == qp->s_acked) { in qib_restart_rc()
831 rvt_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR); in qib_restart_rc()
832 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in qib_restart_rc()
837 qp->s_retry--; in qib_restart_rc()
839 ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_restart_rc()
843 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in qib_restart_rc()
845 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | in qib_restart_rc()
849 qp->s_flags |= RVT_S_SEND_ONE; in qib_restart_rc()
850 reset_psn(qp, psn); in qib_restart_rc()
857 static void reset_sending_psn(struct rvt_qp *qp, u32 psn) in reset_sending_psn() argument
860 u32 n = qp->s_last; in reset_sending_psn()
864 wqe = rvt_get_swqe_ptr(qp, n); in reset_sending_psn()
867 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
869 qp->s_sending_psn = psn + 1; in reset_sending_psn()
872 if (++n == qp->s_size) in reset_sending_psn()
874 if (n == qp->s_tail) in reset_sending_psn()
882 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) in qib_rc_send_complete() argument
889 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) in qib_rc_send_complete()
901 WARN_ON(!qp->s_rdma_ack_cnt); in qib_rc_send_complete()
902 qp->s_rdma_ack_cnt--; in qib_rc_send_complete()
907 reset_sending_psn(qp, psn); in qib_rc_send_complete()
913 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && in qib_rc_send_complete()
914 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) && in qib_rc_send_complete()
915 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_rc_send_complete()
916 rvt_add_retry_timer(qp); in qib_rc_send_complete()
918 while (qp->s_last != qp->s_acked) { in qib_rc_send_complete()
919 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_rc_send_complete()
920 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && in qib_rc_send_complete()
921 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) in qib_rc_send_complete()
923 rvt_qp_complete_swqe(qp, in qib_rc_send_complete()
932 if (qp->s_flags & RVT_S_WAIT_PSN && in qib_rc_send_complete()
933 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in qib_rc_send_complete()
934 qp->s_flags &= ~RVT_S_WAIT_PSN; in qib_rc_send_complete()
935 qp->s_sending_psn = qp->s_psn; in qib_rc_send_complete()
936 qp->s_sending_hpsn = qp->s_psn - 1; in qib_rc_send_complete()
937 qib_schedule_send(qp); in qib_rc_send_complete()
941 static inline void update_last_psn(struct rvt_qp *qp, u32 psn) in update_last_psn() argument
943 qp->s_last_psn = psn; in update_last_psn()
951 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, in do_rc_completion() argument
960 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
961 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) in do_rc_completion()
962 rvt_qp_complete_swqe(qp, in do_rc_completion()
969 qp->s_retry = qp->s_retry_cnt; in do_rc_completion()
970 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
977 if (qp->s_acked == qp->s_cur) { in do_rc_completion()
978 if (++qp->s_cur >= qp->s_size) in do_rc_completion()
979 qp->s_cur = 0; in do_rc_completion()
980 qp->s_acked = qp->s_cur; in do_rc_completion()
981 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
982 if (qp->s_acked != qp->s_tail) { in do_rc_completion()
983 qp->s_state = OP(SEND_LAST); in do_rc_completion()
984 qp->s_psn = wqe->psn; in do_rc_completion()
987 if (++qp->s_acked >= qp->s_size) in do_rc_completion()
988 qp->s_acked = 0; in do_rc_completion()
989 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) in do_rc_completion()
990 qp->s_draining = 0; in do_rc_completion()
991 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
1007 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode, in do_rc_ack() argument
1026 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
1027 ibp = to_iport(qp->ibqp.device, qp->port_num); in do_rc_ack()
1061 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { in do_rc_ack()
1062 qp->r_flags |= RVT_R_RDMAR_SEQ; in do_rc_ack()
1063 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in do_rc_ack()
1064 if (list_empty(&qp->rspwait)) { in do_rc_ack()
1065 qp->r_flags |= RVT_R_RSP_SEND; in do_rc_ack()
1066 rvt_get_qp(qp); in do_rc_ack()
1067 list_add_tail(&qp->rspwait, in do_rc_ack()
1082 if (qp->s_num_rd_atomic && in do_rc_ack()
1086 qp->s_num_rd_atomic--; in do_rc_ack()
1088 if ((qp->s_flags & RVT_S_WAIT_FENCE) && in do_rc_ack()
1089 !qp->s_num_rd_atomic) { in do_rc_ack()
1090 qp->s_flags &= ~(RVT_S_WAIT_FENCE | in do_rc_ack()
1092 qib_schedule_send(qp); in do_rc_ack()
1093 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { in do_rc_ack()
1094 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | in do_rc_ack()
1096 qib_schedule_send(qp); in do_rc_ack()
1099 wqe = do_rc_completion(qp, wqe, ibp); in do_rc_ack()
1100 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1107 if (qp->s_acked != qp->s_tail) { in do_rc_ack()
1112 rvt_mod_retry_timer(qp); in do_rc_ack()
1117 if (qib_cmp24(qp->s_psn, psn) <= 0) in do_rc_ack()
1118 reset_psn(qp, psn + 1); in do_rc_ack()
1121 rvt_stop_rc_timers(qp); in do_rc_ack()
1122 if (qib_cmp24(qp->s_psn, psn) <= 0) { in do_rc_ack()
1123 qp->s_state = OP(SEND_LAST); in do_rc_ack()
1124 qp->s_psn = psn + 1; in do_rc_ack()
1127 if (qp->s_flags & RVT_S_WAIT_ACK) { in do_rc_ack()
1128 qp->s_flags &= ~RVT_S_WAIT_ACK; in do_rc_ack()
1129 qib_schedule_send(qp); in do_rc_ack()
1131 rvt_get_credit(qp, aeth); in do_rc_ack()
1132 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1133 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1134 update_last_psn(qp, psn); in do_rc_ack()
1139 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1141 if (qp->s_flags & RVT_S_WAIT_RNR) in do_rc_ack()
1143 if (qp->s_rnr_retry == 0) { in do_rc_ack()
1147 if (qp->s_rnr_retry_cnt < 7) in do_rc_ack()
1148 qp->s_rnr_retry--; in do_rc_ack()
1151 update_last_psn(qp, psn - 1); in do_rc_ack()
1153 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in do_rc_ack()
1155 reset_psn(qp, psn); in do_rc_ack()
1157 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); in do_rc_ack()
1158 rvt_stop_rc_timers(qp); in do_rc_ack()
1159 rvt_add_rnr_timer(qp, aeth); in do_rc_ack()
1163 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1166 update_last_psn(qp, psn - 1); in do_rc_ack()
1177 qib_restart_rc(qp, psn, 0); in do_rc_ack()
1178 qib_schedule_send(qp); in do_rc_ack()
1195 if (qp->s_last == qp->s_acked) { in do_rc_ack()
1196 rvt_send_complete(qp, wqe, status); in do_rc_ack()
1197 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in do_rc_ack()
1205 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1206 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1216 rvt_stop_rc_timers(qp); in do_rc_ack()
1224 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn, in rdma_seq_err() argument
1230 rvt_stop_rc_timers(qp); in rdma_seq_err()
1232 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
1239 wqe = do_rc_completion(qp, wqe, ibp); in rdma_seq_err()
1243 qp->r_flags |= RVT_R_RDMAR_SEQ; in rdma_seq_err()
1244 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in rdma_seq_err()
1245 if (list_empty(&qp->rspwait)) { in rdma_seq_err()
1246 qp->r_flags |= RVT_R_RSP_SEND; in rdma_seq_err()
1247 rvt_get_qp(qp); in rdma_seq_err()
1248 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in rdma_seq_err()
1272 struct rvt_qp *qp, in qib_rc_rcv_resp() argument
1291 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && in qib_rc_rcv_resp()
1292 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { in qib_rc_rcv_resp()
1298 if (!(qp->s_flags & RVT_S_BUSY)) { in qib_rc_rcv_resp()
1309 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_resp()
1310 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_rc_rcv_resp()
1314 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0) in qib_rc_rcv_resp()
1318 diff = qib_cmp24(psn, qp->s_last_psn); in qib_rc_rcv_resp()
1324 rvt_get_credit(qp, aeth); in qib_rc_rcv_resp()
1333 if (qp->r_flags & RVT_R_RDMAR_SEQ) { in qib_rc_rcv_resp()
1334 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) in qib_rc_rcv_resp()
1336 qp->r_flags &= ~RVT_R_RDMAR_SEQ; in qib_rc_rcv_resp()
1339 if (unlikely(qp->s_acked == qp->s_tail)) in qib_rc_rcv_resp()
1341 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1353 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) || in qib_rc_rcv_resp()
1357 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1365 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1371 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1378 if (unlikely(pmtu >= qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1385 rvt_mod_retry_timer(qp); in qib_rc_rcv_resp()
1386 if (qp->s_flags & RVT_S_WAIT_ACK) { in qib_rc_rcv_resp()
1387 qp->s_flags &= ~RVT_S_WAIT_ACK; in qib_rc_rcv_resp()
1388 qib_schedule_send(qp); in qib_rc_rcv_resp()
1392 qp->s_retry = qp->s_retry_cnt; in qib_rc_rcv_resp()
1398 qp->s_rdma_read_len -= pmtu; in qib_rc_rcv_resp()
1399 update_last_psn(qp, psn); in qib_rc_rcv_resp()
1400 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1401 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1407 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd)) in qib_rc_rcv_resp()
1423 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1424 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1430 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1445 if (unlikely(tlen != qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1448 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1450 WARN_ON(qp->s_rdma_read_sge.num_sge); in qib_rc_rcv_resp()
1451 (void) do_rc_ack(qp, aeth, psn, in qib_rc_rcv_resp()
1461 rdma_seq_err(qp, ibp, psn, rcd); in qib_rc_rcv_resp()
1467 if (qp->s_last == qp->s_acked) { in qib_rc_rcv_resp()
1468 rvt_send_complete(qp, wqe, status); in qib_rc_rcv_resp()
1469 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR); in qib_rc_rcv_resp()
1472 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1495 struct rvt_qp *qp, in qib_rc_rcv_error() argument
1501 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_rc_rcv_error()
1513 if (!qp->r_nak_state) { in qib_rc_rcv_error()
1515 qp->r_nak_state = IB_NAK_PSN_ERROR; in qib_rc_rcv_error()
1517 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv_error()
1523 if (list_empty(&qp->rspwait)) { in qib_rc_rcv_error()
1524 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv_error()
1525 rvt_get_qp(qp); in qib_rc_rcv_error()
1526 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv_error()
1552 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_error()
1554 for (i = qp->r_head_ack_queue; ; i = prev) { in qib_rc_rcv_error()
1555 if (i == qp->s_tail_ack_queue) in qib_rc_rcv_error()
1561 if (prev == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1565 e = &qp->s_ack_queue[prev]; in qib_rc_rcv_error()
1571 if (prev == qp->s_tail_ack_queue && in qib_rc_rcv_error()
1599 qp->pmtu; in qib_rc_rcv_error()
1612 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in qib_rc_rcv_error()
1624 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1637 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1652 if (i == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1653 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1654 qp->r_nak_state = 0; in qib_rc_rcv_error()
1655 qp->r_ack_psn = qp->r_psn - 1; in qib_rc_rcv_error()
1663 if (!(qp->s_flags & RVT_S_RESP_PENDING)) { in qib_rc_rcv_error()
1664 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1665 qp->r_nak_state = 0; in qib_rc_rcv_error()
1666 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; in qib_rc_rcv_error()
1673 qp->s_tail_ack_queue = i; in qib_rc_rcv_error()
1676 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_rc_rcv_error()
1677 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv_error()
1678 qp->r_nak_state = 0; in qib_rc_rcv_error()
1679 qib_schedule_send(qp); in qib_rc_rcv_error()
1682 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1690 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n) in qib_update_ack_queue() argument
1697 qp->s_tail_ack_queue = next; in qib_update_ack_queue()
1698 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_update_ack_queue()
1715 int has_grh, void *data, u32 tlen, struct rvt_qp *qp) in qib_rc_rcv() argument
1724 u32 pmtu = qp->pmtu; in qib_rc_rcv()
1740 if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode)) in qib_rc_rcv()
1754 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, in qib_rc_rcv()
1760 diff = qib_cmp24(psn, qp->r_psn); in qib_rc_rcv()
1762 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) in qib_rc_rcv()
1768 switch (qp->r_state) { in qib_rc_rcv()
1801 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in qib_rc_rcv()
1802 rvt_comm_est(qp); in qib_rc_rcv()
1807 ret = rvt_get_rwqe(qp, false); in qib_rc_rcv()
1812 qp->r_rcv_len = 0; in qib_rc_rcv()
1820 qp->r_rcv_len += pmtu; in qib_rc_rcv()
1821 if (unlikely(qp->r_rcv_len > qp->r_len)) in qib_rc_rcv()
1823 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); in qib_rc_rcv()
1828 ret = rvt_get_rwqe(qp, true); in qib_rc_rcv()
1837 ret = rvt_get_rwqe(qp, false); in qib_rc_rcv()
1842 qp->r_rcv_len = 0; in qib_rc_rcv()
1866 wc.byte_len = tlen + qp->r_rcv_len; in qib_rc_rcv()
1867 if (unlikely(wc.byte_len > qp->r_len)) in qib_rc_rcv()
1869 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); in qib_rc_rcv()
1870 rvt_put_ss(&qp->r_sge); in qib_rc_rcv()
1871 qp->r_msn++; in qib_rc_rcv()
1872 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) in qib_rc_rcv()
1874 wc.wr_id = qp->r_wr_id; in qib_rc_rcv()
1881 wc.qp = &qp->ibqp; in qib_rc_rcv()
1882 wc.src_qp = qp->remote_qpn; in qib_rc_rcv()
1883 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); in qib_rc_rcv()
1884 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); in qib_rc_rcv()
1891 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); in qib_rc_rcv()
1897 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in qib_rc_rcv()
1902 qp->r_len = be32_to_cpu(reth->length); in qib_rc_rcv()
1903 qp->r_rcv_len = 0; in qib_rc_rcv()
1904 qp->r_sge.sg_list = NULL; in qib_rc_rcv()
1905 if (qp->r_len != 0) { in qib_rc_rcv()
1911 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, in qib_rc_rcv()
1915 qp->r_sge.num_sge = 1; in qib_rc_rcv()
1917 qp->r_sge.num_sge = 0; in qib_rc_rcv()
1918 qp->r_sge.sge.mr = NULL; in qib_rc_rcv()
1919 qp->r_sge.sge.vaddr = NULL; in qib_rc_rcv()
1920 qp->r_sge.sge.length = 0; in qib_rc_rcv()
1921 qp->r_sge.sge.sge_length = 0; in qib_rc_rcv()
1927 ret = rvt_get_rwqe(qp, true); in qib_rc_rcv()
1931 rvt_put_ss(&qp->r_sge); in qib_rc_rcv()
1944 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in qib_rc_rcv()
1946 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
1950 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
1951 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
1952 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
1954 qib_update_ack_queue(qp, next); in qib_rc_rcv()
1956 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
1969 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, in qib_rc_rcv()
1977 qp->r_psn += rvt_div_mtu(qp, len - 1); in qib_rc_rcv()
1987 e->lpsn = qp->r_psn; in qib_rc_rcv()
1993 qp->r_msn++; in qib_rc_rcv()
1994 qp->r_psn++; in qib_rc_rcv()
1995 qp->r_state = opcode; in qib_rc_rcv()
1996 qp->r_nak_state = 0; in qib_rc_rcv()
1997 qp->r_head_ack_queue = next; in qib_rc_rcv()
2000 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv()
2001 qib_schedule_send(qp); in qib_rc_rcv()
2016 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) in qib_rc_rcv()
2018 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
2021 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
2022 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
2023 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
2025 qib_update_ack_queue(qp, next); in qib_rc_rcv()
2027 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
2038 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), in qib_rc_rcv()
2043 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; in qib_rc_rcv()
2047 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, in qib_rc_rcv()
2050 rvt_put_mr(qp->r_sge.sge.mr); in qib_rc_rcv()
2051 qp->r_sge.num_sge = 0; in qib_rc_rcv()
2056 qp->r_msn++; in qib_rc_rcv()
2057 qp->r_psn++; in qib_rc_rcv()
2058 qp->r_state = opcode; in qib_rc_rcv()
2059 qp->r_nak_state = 0; in qib_rc_rcv()
2060 qp->r_head_ack_queue = next; in qib_rc_rcv()
2063 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv()
2064 qib_schedule_send(qp); in qib_rc_rcv()
2073 qp->r_psn++; in qib_rc_rcv()
2074 qp->r_state = opcode; in qib_rc_rcv()
2075 qp->r_ack_psn = psn; in qib_rc_rcv()
2076 qp->r_nak_state = 0; in qib_rc_rcv()
2083 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; in qib_rc_rcv()
2084 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2086 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2087 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2088 rvt_get_qp(qp); in qib_rc_rcv()
2089 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2094 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in qib_rc_rcv()
2095 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; in qib_rc_rcv()
2096 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2098 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2099 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2100 rvt_get_qp(qp); in qib_rc_rcv()
2101 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2106 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2108 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR); in qib_rc_rcv()
2109 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in qib_rc_rcv()
2110 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2112 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2113 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2114 rvt_get_qp(qp); in qib_rc_rcv()
2115 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2120 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2122 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR); in qib_rc_rcv()
2123 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in qib_rc_rcv()
2124 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2126 qib_send_rc_ack(qp); in qib_rc_rcv()
2130 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()