96 	struct siw_qp *qp;  in siw_qp_llp_data_ready()  local
105 	qp = sk_to_qp(sk);  in siw_qp_llp_data_ready()
107 	if (likely(!qp->rx_stream.rx_suspend &&  in siw_qp_llp_data_ready()
108 		   down_read_trylock(&qp->state_lock))) {  in siw_qp_llp_data_ready()
109 		read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };  in siw_qp_llp_data_ready()
111 		if (likely(qp->attrs.state == SIW_QP_STATE_RTS))  in siw_qp_llp_data_ready()
120 		up_read(&qp->state_lock);  in siw_qp_llp_data_ready()
122 		siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",  in siw_qp_llp_data_ready()
123 			   qp->rx_stream.rx_suspend);  in siw_qp_llp_data_ready()
129 void siw_qp_llp_close(struct siw_qp *qp)  in siw_qp_llp_close()  argument
131 	siw_dbg_qp(qp, "enter llp close, state = %s\n",  in siw_qp_llp_close()
132 		   siw_qp_state_to_string[qp->attrs.state]);  in siw_qp_llp_close()
134 	down_write(&qp->state_lock);  in siw_qp_llp_close()
136 	qp->rx_stream.rx_suspend = 1;  in siw_qp_llp_close()
137 	qp->tx_ctx.tx_suspend = 1;  in siw_qp_llp_close()
138 	qp->attrs.sk = NULL;  in siw_qp_llp_close()
140 	switch (qp->attrs.state) {  in siw_qp_llp_close()
145 		qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_llp_close()
154 		if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)  in siw_qp_llp_close()
155 			qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_llp_close()
157 			qp->attrs.state = SIW_QP_STATE_IDLE;  in siw_qp_llp_close()
161 		siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",  in siw_qp_llp_close()
162 			   siw_qp_state_to_string[qp->attrs.state]);  in siw_qp_llp_close()
165 	siw_sq_flush(qp);  in siw_qp_llp_close()
166 	siw_rq_flush(qp);  in siw_qp_llp_close()
171 	if (qp->cep) {  in siw_qp_llp_close()
172 		siw_cep_put(qp->cep);  in siw_qp_llp_close()
173 		qp->cep = NULL;  in siw_qp_llp_close()
176 	up_write(&qp->state_lock);  in siw_qp_llp_close()
178 	siw_dbg_qp(qp, "llp close exit: state %s\n",  in siw_qp_llp_close()
179 		   siw_qp_state_to_string[qp->attrs.state]);  in siw_qp_llp_close()
197 			(void)siw_sq_start(cep->qp);  in siw_qp_llp_write_space()
203 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)  in siw_qp_readq_init()  argument
207 		qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe));  in siw_qp_readq_init()
208 		if (!qp->irq) {  in siw_qp_readq_init()
209 			qp->attrs.irq_size = 0;  in siw_qp_readq_init()
215 		qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe));  in siw_qp_readq_init()
216 		if (!qp->orq) {  in siw_qp_readq_init()
217 			qp->attrs.orq_size = 0;  in siw_qp_readq_init()
218 			qp->attrs.irq_size = 0;  in siw_qp_readq_init()
219 			vfree(qp->irq);  in siw_qp_readq_init()
223 	qp->attrs.irq_size = irq_size;  in siw_qp_readq_init()
224 	qp->attrs.orq_size = orq_size;  in siw_qp_readq_init()
225 	siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);  in siw_qp_readq_init()
229 static int siw_qp_enable_crc(struct siw_qp *qp)  in siw_qp_enable_crc()  argument
231 	struct siw_rx_stream *c_rx = &qp->rx_stream;  in siw_qp_enable_crc()
232 	struct siw_iwarp_tx *c_tx = &qp->tx_ctx;  in siw_qp_enable_crc()
263 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)  in siw_qp_mpa_rts()  argument
265 	struct siw_wqe *wqe = tx_wqe(qp);  in siw_qp_mpa_rts()
269 	spin_lock_irqsave(&qp->sq_lock, flags);  in siw_qp_mpa_rts()
272 		spin_unlock_irqrestore(&qp->sq_lock, flags);  in siw_qp_mpa_rts()
298 		spin_lock(&qp->orq_lock);  in siw_qp_mpa_rts()
300 		if (qp->attrs.orq_size)  in siw_qp_mpa_rts()
301 			rreq = orq_get_free(qp);  in siw_qp_mpa_rts()
304 			qp->orq_put++;  in siw_qp_mpa_rts()
308 		spin_unlock(&qp->orq_lock);  in siw_qp_mpa_rts()
315 	spin_unlock_irqrestore(&qp->sq_lock, flags);  in siw_qp_mpa_rts()
318 		rv = siw_sq_start(qp);  in siw_qp_mpa_rts()
367 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,  in siw_init_terminate()  argument
370 	if (!qp->term_info.valid) {  in siw_init_terminate()
371 		memset(&qp->term_info, 0, sizeof(qp->term_info));  in siw_init_terminate()
372 		qp->term_info.layer = layer;  in siw_init_terminate()
373 		qp->term_info.etype = etype;  in siw_init_terminate()
374 		qp->term_info.ecode = ecode;  in siw_init_terminate()
375 		qp->term_info.in_tx = in_tx;  in siw_init_terminate()
376 		qp->term_info.valid = 1;  in siw_init_terminate()
378 	siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",  in siw_init_terminate()
390 void siw_send_terminate(struct siw_qp *qp)  in siw_send_terminate()  argument
396 	struct socket *s = qp->attrs.sk;  in siw_send_terminate()
397 	struct siw_rx_stream *srx = &qp->rx_stream;  in siw_send_terminate()
402 	if (!qp->term_info.valid)  in siw_send_terminate()
405 	qp->term_info.valid = 0;  in siw_send_terminate()
407 	if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {  in siw_send_terminate()
408 		siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",  in siw_send_terminate()
409 			   tx_type(tx_wqe(qp)));  in siw_send_terminate()
412 	if (!s && qp->cep)  in siw_send_terminate()
414 		s = qp->cep->sock;  in siw_send_terminate()
417 		siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");  in siw_send_terminate()
432 	if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||  in siw_send_terminate()
433 	    ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&  in siw_send_terminate()
434 	     (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {  in siw_send_terminate()
444 	__rdmap_term_set_layer(term, qp->term_info.layer);  in siw_send_terminate()
445 	__rdmap_term_set_etype(term, qp->term_info.etype);  in siw_send_terminate()
446 	__rdmap_term_set_ecode(term, qp->term_info.ecode);  in siw_send_terminate()
448 	switch (qp->term_info.layer) {  in siw_send_terminate()
450 		if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)  in siw_send_terminate()
454 		if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {  in siw_send_terminate()
463 			if (qp->term_info.in_tx) {  in siw_send_terminate()
465 				struct siw_wqe *wqe = tx_wqe(qp);  in siw_send_terminate()
515 			if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||  in siw_send_terminate()
516 			    (qp->term_info.ecode == RDMAP_ECODE_OPCODE))  in siw_send_terminate()
543 		if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&  in siw_send_terminate()
544 		     (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||  in siw_send_terminate()
545 		    ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&  in siw_send_terminate()
546 		     (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))  in siw_send_terminate()
586 	if (qp->tx_ctx.mpa_crc_hd) {  in siw_send_terminate()
587 		crypto_shash_init(qp->tx_ctx.mpa_crc_hd);  in siw_send_terminate()
588 		if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,  in siw_send_terminate()
594 			if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,  in siw_send_terminate()
599 		crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc);  in siw_send_terminate()
603 	siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",  in siw_send_terminate()
615 static void siw_qp_modify_nonstate(struct siw_qp *qp,  in siw_qp_modify_nonstate()  argument
621 			qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;  in siw_qp_modify_nonstate()
623 			qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;  in siw_qp_modify_nonstate()
626 			qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;  in siw_qp_modify_nonstate()
628 			qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;  in siw_qp_modify_nonstate()
631 			qp->attrs.flags |= SIW_RDMA_READ_ENABLED;  in siw_qp_modify_nonstate()
633 			qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;  in siw_qp_modify_nonstate()
637 static int siw_qp_nextstate_from_idle(struct siw_qp *qp,  in siw_qp_nextstate_from_idle()  argument
646 			rv = siw_qp_enable_crc(qp);  in siw_qp_nextstate_from_idle()
651 			siw_dbg_qp(qp, "no socket\n");  in siw_qp_nextstate_from_idle()
656 			siw_dbg_qp(qp, "no MPA\n");  in siw_qp_nextstate_from_idle()
663 		qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;  in siw_qp_nextstate_from_idle()
664 		qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;  in siw_qp_nextstate_from_idle()
665 		qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;  in siw_qp_nextstate_from_idle()
670 		qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;  in siw_qp_nextstate_from_idle()
671 		qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;  in siw_qp_nextstate_from_idle()
672 		qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;  in siw_qp_nextstate_from_idle()
678 		rv = siw_qp_readq_init(qp, attrs->irq_size,  in siw_qp_nextstate_from_idle()
683 		qp->attrs.sk = attrs->sk;  in siw_qp_nextstate_from_idle()
684 		qp->attrs.state = SIW_QP_STATE_RTS;  in siw_qp_nextstate_from_idle()
686 		siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",  in siw_qp_nextstate_from_idle()
688 			   qp->attrs.orq_size, qp->attrs.irq_size);  in siw_qp_nextstate_from_idle()
692 		siw_rq_flush(qp);  in siw_qp_nextstate_from_idle()
693 		qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_nextstate_from_idle()
694 		if (qp->cep) {  in siw_qp_nextstate_from_idle()
695 			siw_cep_put(qp->cep);  in siw_qp_nextstate_from_idle()
696 			qp->cep = NULL;  in siw_qp_nextstate_from_idle()
706 static int siw_qp_nextstate_from_rts(struct siw_qp *qp,  in siw_qp_nextstate_from_rts()  argument
721 		if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {  in siw_qp_nextstate_from_rts()
722 			qp->attrs.state = SIW_QP_STATE_CLOSING;  in siw_qp_nextstate_from_rts()
724 			qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_nextstate_from_rts()
725 			siw_sq_flush(qp);  in siw_qp_nextstate_from_rts()
727 		siw_rq_flush(qp);  in siw_qp_nextstate_from_rts()
733 		qp->attrs.state = SIW_QP_STATE_TERMINATE;  in siw_qp_nextstate_from_rts()
735 		siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,  in siw_qp_nextstate_from_rts()
755 		siw_sq_flush(qp);  in siw_qp_nextstate_from_rts()
756 		siw_rq_flush(qp);  in siw_qp_nextstate_from_rts()
757 		qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_nextstate_from_rts()
767 static void siw_qp_nextstate_from_term(struct siw_qp *qp,  in siw_qp_nextstate_from_term()  argument
772 		siw_rq_flush(qp);  in siw_qp_nextstate_from_term()
773 		qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_nextstate_from_term()
775 		if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)  in siw_qp_nextstate_from_term()
776 			siw_sq_flush(qp);  in siw_qp_nextstate_from_term()
784 static int siw_qp_nextstate_from_close(struct siw_qp *qp,  in siw_qp_nextstate_from_close()  argument
791 		WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);  in siw_qp_nextstate_from_close()
792 		qp->attrs.state = SIW_QP_STATE_IDLE;  in siw_qp_nextstate_from_close()
807 		qp->attrs.state = SIW_QP_STATE_ERROR;  in siw_qp_nextstate_from_close()
809 		if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)  in siw_qp_nextstate_from_close()
810 			siw_sq_flush(qp);  in siw_qp_nextstate_from_close()
812 		siw_rq_flush(qp);  in siw_qp_nextstate_from_close()
816 		siw_dbg_qp(qp, "state transition undefined: %s => %s\n",  in siw_qp_nextstate_from_close()
817 			   siw_qp_state_to_string[qp->attrs.state],  in siw_qp_nextstate_from_close()
828 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,  in siw_qp_modify()  argument
836 	siw_dbg_qp(qp, "state: %s => %s\n",  in siw_qp_modify()
837 		   siw_qp_state_to_string[qp->attrs.state],  in siw_qp_modify()
841 		siw_qp_modify_nonstate(qp, attrs, mask);  in siw_qp_modify()
846 	switch (qp->attrs.state) {  in siw_qp_modify()
849 		rv = siw_qp_nextstate_from_idle(qp, attrs, mask);  in siw_qp_modify()
853 		drop_conn = siw_qp_nextstate_from_rts(qp, attrs);  in siw_qp_modify()
857 		siw_qp_nextstate_from_term(qp, attrs);  in siw_qp_modify()
861 		siw_qp_nextstate_from_close(qp, attrs);  in siw_qp_modify()
867 		siw_qp_cm_drop(qp, 0);  in siw_qp_modify()
884 static int siw_activate_tx_from_sq(struct siw_qp *qp)  in siw_activate_tx_from_sq()  argument
887 	struct siw_wqe *wqe = tx_wqe(qp);  in siw_activate_tx_from_sq()
890 	sqe = sq_get_next(qp);  in siw_activate_tx_from_sq()
923 			siw_dbg_qp(qp, "cannot fence read\n");  in siw_activate_tx_from_sq()
927 		spin_lock(&qp->orq_lock);  in siw_activate_tx_from_sq()
929 		if (qp->attrs.orq_size && !siw_orq_empty(qp)) {  in siw_activate_tx_from_sq()
930 			qp->tx_ctx.orq_fence = 1;  in siw_activate_tx_from_sq()
933 		spin_unlock(&qp->orq_lock);  in siw_activate_tx_from_sq()
939 		if (unlikely(!qp->attrs.orq_size)) {  in siw_activate_tx_from_sq()
946 		spin_lock(&qp->orq_lock);  in siw_activate_tx_from_sq()
948 		rreq = orq_get_free(qp);  in siw_activate_tx_from_sq()
955 			qp->orq_put++;  in siw_activate_tx_from_sq()
957 			qp->tx_ctx.orq_fence = 1;  in siw_activate_tx_from_sq()
960 		spin_unlock(&qp->orq_lock);  in siw_activate_tx_from_sq()
965 	qp->sq_get++;  in siw_activate_tx_from_sq()
968 		siw_dbg_qp(qp, "error %d\n", rv);  in siw_activate_tx_from_sq()
980 int siw_activate_tx(struct siw_qp *qp)  in siw_activate_tx()  argument
983 	struct siw_wqe *wqe = tx_wqe(qp);  in siw_activate_tx()
985 	if (!qp->attrs.irq_size)  in siw_activate_tx()
986 		return siw_activate_tx_from_sq(qp);  in siw_activate_tx()
988 	irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];  in siw_activate_tx()
991 		return siw_activate_tx_from_sq(qp);  in siw_activate_tx()
997 	if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {  in siw_activate_tx()
998 		qp->irq_burst = 0;  in siw_activate_tx()
999 		return siw_activate_tx_from_sq(qp);  in siw_activate_tx()
1025 	qp->irq_get++;  in siw_activate_tx()
1063 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,  in siw_sqe_complete()  argument
1066 	struct siw_cq *cq = qp->scq;  in siw_sqe_complete()
1090 				cqe->base_qp = &qp->base_qp;  in siw_sqe_complete()
1092 				cqe->qp_id = qp_id(qp);  in siw_sqe_complete()
1121 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,  in siw_rqe_complete()  argument
1124 	struct siw_cq *cq = qp->rcq;  in siw_rqe_complete()
1148 				cqe->base_qp = &qp->base_qp;  in siw_rqe_complete()
1154 				cqe->qp_id = qp_id(qp);  in siw_rqe_complete()
1191 void siw_sq_flush(struct siw_qp *qp)  in siw_sq_flush()  argument
1194 	struct siw_wqe *wqe = tx_wqe(qp);  in siw_sq_flush()
1200 	while (qp->attrs.orq_size) {  in siw_sq_flush()
1201 		sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];  in siw_sq_flush()
1205 		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_sq_flush()
1209 		qp->orq_get++;  in siw_sq_flush()
1215 		siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",  in siw_sq_flush()
1228 			siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,  in siw_sq_flush()
1236 	while (qp->attrs.sq_size) {  in siw_sq_flush()
1237 		sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];  in siw_sq_flush()
1242 		if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_sq_flush()
1250 		qp->sq_get++;  in siw_sq_flush()
1253 		siw_qp_event(qp, IB_EVENT_SQ_DRAINED);  in siw_sq_flush()
1267 void siw_rq_flush(struct siw_qp *qp)  in siw_rq_flush()  argument
1269 	struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;  in siw_rq_flush()
1275 		siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",  in siw_rq_flush()
1281 			siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,  in siw_rq_flush()
1286 			siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);  in siw_rq_flush()
1290 	wqe = &qp->rx_tagged.wqe_active;  in siw_rq_flush()
1299 	while (qp->attrs.rq_size) {  in siw_rq_flush()
1301 			&qp->recvq[qp->rq_get % qp->attrs.rq_size];  in siw_rq_flush()
1306 		if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)  in siw_rq_flush()
1310 		qp->rq_get++;  in siw_rq_flush()
1314 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)  in siw_qp_add()  argument
1316 	int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,  in siw_qp_add()
1320 		kref_init(&qp->ref);  in siw_qp_add()
1321 		qp->sdev = sdev;  in siw_qp_add()
1322 		siw_dbg_qp(qp, "new QP\n");  in siw_qp_add()
1329 	struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);  in siw_free_qp()  local
1330 	struct siw_device *sdev = qp->sdev;  in siw_free_qp()
1333 	if (qp->cep)  in siw_free_qp()
1334 		siw_cep_put(qp->cep);  in siw_free_qp()
1336 	found = xa_erase(&sdev->qp_xa, qp_id(qp));  in siw_free_qp()
1337 	WARN_ON(found != qp);  in siw_free_qp()
1339 	list_del(&qp->devq);  in siw_free_qp()
1342 	vfree(qp->sendq);  in siw_free_qp()
1343 	vfree(qp->recvq);  in siw_free_qp()
1344 	vfree(qp->irq);  in siw_free_qp()
1345 	vfree(qp->orq);  in siw_free_qp()
1347 	siw_put_tx_cpu(qp->tx_cpu);  in siw_free_qp()
1348 	complete(&qp->qp_free);  in siw_free_qp()