Lines Matching refs:qp
14 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
17 static inline void retry_first_write_send(struct rxe_qp *qp, in retry_first_write_send() argument
23 int to_send = (wqe->dma.resid > qp->mtu) ? in retry_first_write_send()
24 qp->mtu : wqe->dma.resid; in retry_first_write_send()
26 qp->req.opcode = next_opcode(qp, wqe, in retry_first_write_send()
38 static void req_retry(struct rxe_qp *qp) in req_retry() argument
45 struct rxe_queue *q = qp->sq.queue; in req_retry()
52 qp->req.wqe_index = cons; in req_retry()
53 qp->req.psn = qp->comp.psn; in req_retry()
54 qp->req.opcode = -1; in req_retry()
58 wqe = queue_addr_from_index(qp->sq.queue, wqe_index); in req_retry()
59 mask = wr_opcode_mask(wqe->wr.opcode, qp); in req_retry()
83 npsn = (qp->comp.psn - wqe->first_psn) & in req_retry()
85 retry_first_write_send(qp, wqe, npsn); in req_retry()
90 qp->mtu; in req_retry()
91 wqe->iova += npsn * qp->mtu; in req_retry()
101 struct rxe_qp *qp = from_timer(qp, t, rnr_nak_timer); in rnr_nak_timer() local
104 rxe_dbg_qp(qp, "nak timer fired\n"); in rnr_nak_timer()
106 spin_lock_irqsave(&qp->state_lock, flags); in rnr_nak_timer()
107 if (qp->valid) { in rnr_nak_timer()
109 qp->req.need_retry = 1; in rnr_nak_timer()
110 qp->req.wait_for_rnr_timer = 0; in rnr_nak_timer()
111 rxe_sched_task(&qp->req.task); in rnr_nak_timer()
113 spin_unlock_irqrestore(&qp->state_lock, flags); in rnr_nak_timer()
116 static void req_check_sq_drain_done(struct rxe_qp *qp) in req_check_sq_drain_done() argument
124 spin_lock_irqsave(&qp->state_lock, flags); in req_check_sq_drain_done()
125 if (qp_state(qp) == IB_QPS_SQD) { in req_check_sq_drain_done()
126 q = qp->sq.queue; in req_check_sq_drain_done()
127 index = qp->req.wqe_index; in req_check_sq_drain_done()
135 if (!qp->attr.sq_draining) in req_check_sq_drain_done()
144 qp->attr.sq_draining = 0; in req_check_sq_drain_done()
145 spin_unlock_irqrestore(&qp->state_lock, flags); in req_check_sq_drain_done()
147 if (qp->ibqp.event_handler) { in req_check_sq_drain_done()
150 ev.device = qp->ibqp.device; in req_check_sq_drain_done()
151 ev.element.qp = &qp->ibqp; in req_check_sq_drain_done()
153 qp->ibqp.event_handler(&ev, in req_check_sq_drain_done()
154 qp->ibqp.qp_context); in req_check_sq_drain_done()
159 spin_unlock_irqrestore(&qp->state_lock, flags); in req_check_sq_drain_done()
162 static struct rxe_send_wqe *__req_next_wqe(struct rxe_qp *qp) in __req_next_wqe() argument
164 struct rxe_queue *q = qp->sq.queue; in __req_next_wqe()
165 unsigned int index = qp->req.wqe_index; in __req_next_wqe()
175 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp) in req_next_wqe() argument
180 req_check_sq_drain_done(qp); in req_next_wqe()
182 wqe = __req_next_wqe(qp); in req_next_wqe()
186 spin_lock_irqsave(&qp->state_lock, flags); in req_next_wqe()
187 if (unlikely((qp_state(qp) == IB_QPS_SQD) && in req_next_wqe()
189 spin_unlock_irqrestore(&qp->state_lock, flags); in req_next_wqe()
192 spin_unlock_irqrestore(&qp->state_lock, flags); in req_next_wqe()
194 wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp); in req_next_wqe()
206 static int rxe_wqe_is_fenced(struct rxe_qp *qp, struct rxe_send_wqe *wqe) in rxe_wqe_is_fenced() argument
213 return qp->req.wqe_index != queue_get_consumer(qp->sq.queue, in rxe_wqe_is_fenced()
221 atomic_read(&qp->req.rd_atomic) != qp->attr.max_rd_atomic; in rxe_wqe_is_fenced()
224 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits) in next_opcode_rc() argument
228 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || in next_opcode_rc()
229 qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) in next_opcode_rc()
239 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST || in next_opcode_rc()
240 qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE) in next_opcode_rc()
250 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
251 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
261 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
262 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
284 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST || in next_opcode_rc()
285 qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE) in next_opcode_rc()
303 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits) in next_opcode_uc() argument
307 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || in next_opcode_uc()
308 qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) in next_opcode_uc()
318 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST || in next_opcode_uc()
319 qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE) in next_opcode_uc()
329 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || in next_opcode_uc()
330 qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) in next_opcode_uc()
340 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST || in next_opcode_uc()
341 qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE) in next_opcode_uc()
354 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe, in next_opcode() argument
357 int fits = (wqe->dma.resid <= qp->mtu); in next_opcode()
359 switch (qp_type(qp)) { in next_opcode()
361 return next_opcode_rc(qp, opcode, fits); in next_opcode()
364 return next_opcode_uc(qp, opcode, fits); in next_opcode()
384 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe) in check_init_depth() argument
391 qp->req.need_rd_atomic = 1; in check_init_depth()
392 depth = atomic_dec_return(&qp->req.rd_atomic); in check_init_depth()
395 qp->req.need_rd_atomic = 0; in check_init_depth()
400 atomic_inc(&qp->req.rd_atomic); in check_init_depth()
404 static inline int get_mtu(struct rxe_qp *qp) in get_mtu() argument
406 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in get_mtu()
408 if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC)) in get_mtu()
409 return qp->mtu; in get_mtu()
414 static struct sk_buff *init_req_packet(struct rxe_qp *qp, in init_req_packet() argument
420 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in init_req_packet()
446 qp->attr.dest_qp_num; in init_req_packet()
448 if (qp_type(qp) != IB_QPT_UD && qp_type(qp) != IB_QPT_UC) in init_req_packet()
450 (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK)); in init_req_packet()
452 qp->req.noack_pkts = 0; in init_req_packet()
489 if (qp->ibqp.qp_num == 1) in init_req_packet()
493 deth_set_sqp(pkt, qp->ibqp.qp_num); in init_req_packet()
499 static int finish_packet(struct rxe_qp *qp, struct rxe_av *av, in finish_packet() argument
518 err = copy_data(qp->pd, 0, &wqe->dma, in finish_packet()
542 static void update_wqe_state(struct rxe_qp *qp, in update_wqe_state() argument
547 if (qp_type(qp) == IB_QPT_RC) in update_wqe_state()
554 static void update_wqe_psn(struct rxe_qp *qp, in update_wqe_psn() argument
560 int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu; in update_wqe_psn()
567 wqe->first_psn = qp->req.psn; in update_wqe_psn()
568 wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK; in update_wqe_psn()
572 qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK; in update_wqe_psn()
574 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; in update_wqe_psn()
578 struct rxe_qp *qp, in save_state() argument
586 *rollback_psn = qp->req.psn; in save_state()
590 struct rxe_qp *qp, in rollback_state() argument
598 qp->req.psn = rollback_psn; in rollback_state()
601 static void update_state(struct rxe_qp *qp, struct rxe_pkt_info *pkt) in update_state() argument
603 qp->req.opcode = pkt->opcode; in update_state()
606 qp->req.wqe_index = queue_next_index(qp->sq.queue, in update_state()
607 qp->req.wqe_index); in update_state()
609 qp->need_req_skb = 0; in update_state()
611 if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer)) in update_state()
612 mod_timer(&qp->retrans_timer, in update_state()
613 jiffies + qp->qp_timeout_jiffies); in update_state()
616 static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe) in rxe_do_local_ops() argument
626 ret = rxe_invalidate_mw(qp, rkey); in rxe_do_local_ops()
628 ret = rxe_invalidate_mr(qp, rkey); in rxe_do_local_ops()
636 ret = rxe_reg_fast_mr(qp, wqe); in rxe_do_local_ops()
643 ret = rxe_bind_mw(qp, wqe); in rxe_do_local_ops()
650 rxe_dbg_qp(qp, "Unexpected send wqe opcode %d\n", opcode); in rxe_do_local_ops()
657 qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index); in rxe_do_local_ops()
663 rxe_sched_task(&qp->comp.task); in rxe_do_local_ops()
668 int rxe_requester(struct rxe_qp *qp) in rxe_requester() argument
670 struct rxe_dev *rxe = to_rdev(qp->ibqp.device); in rxe_requester()
682 struct rxe_queue *q = qp->sq.queue; in rxe_requester()
687 spin_lock_irqsave(&qp->state_lock, flags); in rxe_requester()
688 if (unlikely(!qp->valid)) { in rxe_requester()
689 spin_unlock_irqrestore(&qp->state_lock, flags); in rxe_requester()
693 if (unlikely(qp_state(qp) == IB_QPS_ERR)) { in rxe_requester()
694 wqe = __req_next_wqe(qp); in rxe_requester()
695 spin_unlock_irqrestore(&qp->state_lock, flags); in rxe_requester()
702 if (unlikely(qp_state(qp) == IB_QPS_RESET)) { in rxe_requester()
703 qp->req.wqe_index = queue_get_consumer(q, in rxe_requester()
705 qp->req.opcode = -1; in rxe_requester()
706 qp->req.need_rd_atomic = 0; in rxe_requester()
707 qp->req.wait_psn = 0; in rxe_requester()
708 qp->req.need_retry = 0; in rxe_requester()
709 qp->req.wait_for_rnr_timer = 0; in rxe_requester()
710 spin_unlock_irqrestore(&qp->state_lock, flags); in rxe_requester()
713 spin_unlock_irqrestore(&qp->state_lock, flags); in rxe_requester()
721 if (unlikely(qp->req.need_retry && !qp->req.wait_for_rnr_timer)) { in rxe_requester()
722 req_retry(qp); in rxe_requester()
723 qp->req.need_retry = 0; in rxe_requester()
726 wqe = req_next_wqe(qp); in rxe_requester()
730 if (rxe_wqe_is_fenced(qp, wqe)) { in rxe_requester()
731 qp->req.wait_fence = 1; in rxe_requester()
736 err = rxe_do_local_ops(qp, wqe); in rxe_requester()
743 if (unlikely(qp_type(qp) == IB_QPT_RC && in rxe_requester()
744 psn_compare(qp->req.psn, (qp->comp.psn + in rxe_requester()
746 qp->req.wait_psn = 1; in rxe_requester()
751 if (unlikely(atomic_read(&qp->skb_out) > in rxe_requester()
753 qp->need_req_skb = 1; in rxe_requester()
757 opcode = next_opcode(qp, wqe, wqe->wr.opcode); in rxe_requester()
766 if (check_init_depth(qp, wqe)) in rxe_requester()
770 mtu = get_mtu(qp); in rxe_requester()
774 if (qp_type(qp) == IB_QPT_UD) { in rxe_requester()
782 wqe->first_psn = qp->req.psn; in rxe_requester()
783 wqe->last_psn = qp->req.psn; in rxe_requester()
784 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK; in rxe_requester()
785 qp->req.opcode = IB_OPCODE_UD_SEND_ONLY; in rxe_requester()
786 qp->req.wqe_index = queue_next_index(qp->sq.queue, in rxe_requester()
787 qp->req.wqe_index); in rxe_requester()
790 rxe_sched_task(&qp->comp.task); in rxe_requester()
798 pkt.qp = qp; in rxe_requester()
799 pkt.psn = qp->req.psn; in rxe_requester()
804 save_state(wqe, qp, &rollback_wqe, &rollback_psn); in rxe_requester()
808 rxe_dbg_qp(qp, "Failed no address vector\n"); in rxe_requester()
813 skb = init_req_packet(qp, av, wqe, opcode, payload, &pkt); in rxe_requester()
815 rxe_dbg_qp(qp, "Failed allocating skb\n"); in rxe_requester()
822 err = finish_packet(qp, av, wqe, &pkt, skb, payload); in rxe_requester()
824 rxe_dbg_qp(qp, "Error during finish packet\n"); in rxe_requester()
839 update_wqe_state(qp, wqe, &pkt); in rxe_requester()
840 update_wqe_psn(qp, wqe, &pkt, payload); in rxe_requester()
842 err = rxe_xmit_packet(qp, &pkt, skb); in rxe_requester()
852 rollback_state(wqe, qp, &rollback_wqe, rollback_psn); in rxe_requester()
857 qp->need_req_skb = 1; in rxe_requester()
859 rxe_sched_task(&qp->req.task); in rxe_requester()
863 update_state(qp, &pkt); in rxe_requester()
874 qp->req.wqe_index = queue_next_index(qp->sq.queue, qp->req.wqe_index); in rxe_requester()
876 rxe_qp_error(qp); in rxe_requester()