115505577SCheng Xu // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
215505577SCheng Xu
315505577SCheng Xu /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */
415505577SCheng Xu /* Kai Shen <kaishen@linux.alibaba.com> */
515505577SCheng Xu /* Copyright (c) 2020-2021, Alibaba Group */
615505577SCheng Xu /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
715505577SCheng Xu /* Copyright (c) 2008-2019, IBM Corporation */
815505577SCheng Xu
915505577SCheng Xu #include "erdma_cm.h"
1015505577SCheng Xu #include "erdma_verbs.h"
1115505577SCheng Xu
erdma_qp_llp_close(struct erdma_qp * qp)1215505577SCheng Xu void erdma_qp_llp_close(struct erdma_qp *qp)
1315505577SCheng Xu {
1415505577SCheng Xu struct erdma_qp_attrs qp_attrs;
1515505577SCheng Xu
1615505577SCheng Xu down_write(&qp->state_lock);
1715505577SCheng Xu
1815505577SCheng Xu switch (qp->attrs.state) {
1915505577SCheng Xu case ERDMA_QP_STATE_RTS:
2015505577SCheng Xu case ERDMA_QP_STATE_RTR:
2115505577SCheng Xu case ERDMA_QP_STATE_IDLE:
2215505577SCheng Xu case ERDMA_QP_STATE_TERMINATE:
2315505577SCheng Xu qp_attrs.state = ERDMA_QP_STATE_CLOSING;
2415505577SCheng Xu erdma_modify_qp_internal(qp, &qp_attrs, ERDMA_QP_ATTR_STATE);
2515505577SCheng Xu break;
2615505577SCheng Xu case ERDMA_QP_STATE_CLOSING:
2715505577SCheng Xu qp->attrs.state = ERDMA_QP_STATE_IDLE;
2815505577SCheng Xu break;
2915505577SCheng Xu default:
3015505577SCheng Xu break;
3115505577SCheng Xu }
3215505577SCheng Xu
3315505577SCheng Xu if (qp->cep) {
3415505577SCheng Xu erdma_cep_put(qp->cep);
3515505577SCheng Xu qp->cep = NULL;
3615505577SCheng Xu }
3715505577SCheng Xu
3815505577SCheng Xu up_write(&qp->state_lock);
3915505577SCheng Xu }
4015505577SCheng Xu
erdma_get_ibqp(struct ib_device * ibdev,int id)4115505577SCheng Xu struct ib_qp *erdma_get_ibqp(struct ib_device *ibdev, int id)
4215505577SCheng Xu {
4315505577SCheng Xu struct erdma_qp *qp = find_qp_by_qpn(to_edev(ibdev), id);
4415505577SCheng Xu
4515505577SCheng Xu if (qp)
4615505577SCheng Xu return &qp->ibqp;
4715505577SCheng Xu
4815505577SCheng Xu return NULL;
4915505577SCheng Xu }
5015505577SCheng Xu
erdma_modify_qp_state_to_rts(struct erdma_qp * qp,struct erdma_qp_attrs * attrs,enum erdma_qp_attr_mask mask)5115505577SCheng Xu static int erdma_modify_qp_state_to_rts(struct erdma_qp *qp,
5215505577SCheng Xu struct erdma_qp_attrs *attrs,
5315505577SCheng Xu enum erdma_qp_attr_mask mask)
5415505577SCheng Xu {
5515505577SCheng Xu int ret;
5615505577SCheng Xu struct erdma_dev *dev = qp->dev;
5715505577SCheng Xu struct erdma_cmdq_modify_qp_req req;
5815505577SCheng Xu struct tcp_sock *tp;
5915505577SCheng Xu struct erdma_cep *cep = qp->cep;
6015505577SCheng Xu struct sockaddr_storage local_addr, remote_addr;
6115505577SCheng Xu
6215505577SCheng Xu if (!(mask & ERDMA_QP_ATTR_LLP_HANDLE))
6315505577SCheng Xu return -EINVAL;
6415505577SCheng Xu
6515505577SCheng Xu if (!(mask & ERDMA_QP_ATTR_MPA))
6615505577SCheng Xu return -EINVAL;
6715505577SCheng Xu
6815505577SCheng Xu ret = getname_local(cep->sock, &local_addr);
6915505577SCheng Xu if (ret < 0)
7015505577SCheng Xu return ret;
7115505577SCheng Xu
7215505577SCheng Xu ret = getname_peer(cep->sock, &remote_addr);
7315505577SCheng Xu if (ret < 0)
7415505577SCheng Xu return ret;
7515505577SCheng Xu
7615505577SCheng Xu qp->attrs.state = ERDMA_QP_STATE_RTS;
7715505577SCheng Xu
7815505577SCheng Xu tp = tcp_sk(qp->cep->sock->sk);
7915505577SCheng Xu
8015505577SCheng Xu erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
8115505577SCheng Xu CMDQ_OPCODE_MODIFY_QP);
8215505577SCheng Xu
8315505577SCheng Xu req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, qp->attrs.state) |
8415505577SCheng Xu FIELD_PREP(ERDMA_CMD_MODIFY_QP_CC_MASK, qp->attrs.cc) |
8515505577SCheng Xu FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));
8615505577SCheng Xu
8715505577SCheng Xu req.cookie = be32_to_cpu(qp->cep->mpa.ext_data.cookie);
8815505577SCheng Xu req.dip = to_sockaddr_in(remote_addr).sin_addr.s_addr;
8915505577SCheng Xu req.sip = to_sockaddr_in(local_addr).sin_addr.s_addr;
9015505577SCheng Xu req.dport = to_sockaddr_in(remote_addr).sin_port;
9115505577SCheng Xu req.sport = to_sockaddr_in(local_addr).sin_port;
9215505577SCheng Xu
9315505577SCheng Xu req.send_nxt = tp->snd_nxt;
9415505577SCheng Xu /* rsvd tcp seq for mpa-rsp in server. */
9515505577SCheng Xu if (qp->attrs.qp_type == ERDMA_QP_PASSIVE)
9615505577SCheng Xu req.send_nxt += MPA_DEFAULT_HDR_LEN + qp->attrs.pd_len;
9715505577SCheng Xu req.recv_nxt = tp->rcv_nxt;
9815505577SCheng Xu
9995f911d9SCheng Xu return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
10015505577SCheng Xu }
10115505577SCheng Xu
erdma_modify_qp_state_to_stop(struct erdma_qp * qp,struct erdma_qp_attrs * attrs,enum erdma_qp_attr_mask mask)10215505577SCheng Xu static int erdma_modify_qp_state_to_stop(struct erdma_qp *qp,
10315505577SCheng Xu struct erdma_qp_attrs *attrs,
10415505577SCheng Xu enum erdma_qp_attr_mask mask)
10515505577SCheng Xu {
10615505577SCheng Xu struct erdma_dev *dev = qp->dev;
10715505577SCheng Xu struct erdma_cmdq_modify_qp_req req;
10815505577SCheng Xu
10915505577SCheng Xu qp->attrs.state = attrs->state;
11015505577SCheng Xu
11115505577SCheng Xu erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
11215505577SCheng Xu CMDQ_OPCODE_MODIFY_QP);
11315505577SCheng Xu
11415505577SCheng Xu req.cfg = FIELD_PREP(ERDMA_CMD_MODIFY_QP_STATE_MASK, attrs->state) |
11515505577SCheng Xu FIELD_PREP(ERDMA_CMD_MODIFY_QP_QPN_MASK, QP_ID(qp));
11615505577SCheng Xu
11795f911d9SCheng Xu return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);
11815505577SCheng Xu }
11915505577SCheng Xu
erdma_modify_qp_internal(struct erdma_qp * qp,struct erdma_qp_attrs * attrs,enum erdma_qp_attr_mask mask)12015505577SCheng Xu int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
12115505577SCheng Xu enum erdma_qp_attr_mask mask)
12215505577SCheng Xu {
1230edf42cbSCheng Xu bool need_reflush = false;
12415505577SCheng Xu int drop_conn, ret = 0;
12515505577SCheng Xu
12615505577SCheng Xu if (!mask)
12715505577SCheng Xu return 0;
12815505577SCheng Xu
12915505577SCheng Xu if (!(mask & ERDMA_QP_ATTR_STATE))
13015505577SCheng Xu return 0;
13115505577SCheng Xu
13215505577SCheng Xu switch (qp->attrs.state) {
13315505577SCheng Xu case ERDMA_QP_STATE_IDLE:
13415505577SCheng Xu case ERDMA_QP_STATE_RTR:
13515505577SCheng Xu if (attrs->state == ERDMA_QP_STATE_RTS) {
13615505577SCheng Xu ret = erdma_modify_qp_state_to_rts(qp, attrs, mask);
13715505577SCheng Xu } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
13815505577SCheng Xu qp->attrs.state = ERDMA_QP_STATE_ERROR;
1390edf42cbSCheng Xu need_reflush = true;
14015505577SCheng Xu if (qp->cep) {
14115505577SCheng Xu erdma_cep_put(qp->cep);
14215505577SCheng Xu qp->cep = NULL;
14315505577SCheng Xu }
14415505577SCheng Xu ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
14515505577SCheng Xu }
14615505577SCheng Xu break;
14715505577SCheng Xu case ERDMA_QP_STATE_RTS:
14815505577SCheng Xu drop_conn = 0;
14915505577SCheng Xu
1500edf42cbSCheng Xu if (attrs->state == ERDMA_QP_STATE_CLOSING ||
1510edf42cbSCheng Xu attrs->state == ERDMA_QP_STATE_TERMINATE ||
1520edf42cbSCheng Xu attrs->state == ERDMA_QP_STATE_ERROR) {
15315505577SCheng Xu ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
15415505577SCheng Xu drop_conn = 1;
1550edf42cbSCheng Xu need_reflush = true;
15615505577SCheng Xu }
15715505577SCheng Xu
15815505577SCheng Xu if (drop_conn)
15915505577SCheng Xu erdma_qp_cm_drop(qp);
16015505577SCheng Xu
16115505577SCheng Xu break;
16215505577SCheng Xu case ERDMA_QP_STATE_TERMINATE:
16315505577SCheng Xu if (attrs->state == ERDMA_QP_STATE_ERROR)
16415505577SCheng Xu qp->attrs.state = ERDMA_QP_STATE_ERROR;
16515505577SCheng Xu break;
16615505577SCheng Xu case ERDMA_QP_STATE_CLOSING:
16715505577SCheng Xu if (attrs->state == ERDMA_QP_STATE_IDLE) {
16815505577SCheng Xu qp->attrs.state = ERDMA_QP_STATE_IDLE;
16915505577SCheng Xu } else if (attrs->state == ERDMA_QP_STATE_ERROR) {
17015505577SCheng Xu ret = erdma_modify_qp_state_to_stop(qp, attrs, mask);
17115505577SCheng Xu qp->attrs.state = ERDMA_QP_STATE_ERROR;
17215505577SCheng Xu } else if (attrs->state != ERDMA_QP_STATE_CLOSING) {
17315505577SCheng Xu return -ECONNABORTED;
17415505577SCheng Xu }
17515505577SCheng Xu break;
17615505577SCheng Xu default:
17715505577SCheng Xu break;
17815505577SCheng Xu }
17915505577SCheng Xu
1800edf42cbSCheng Xu if (need_reflush && !ret && rdma_is_kernel_res(&qp->ibqp.res)) {
1810edf42cbSCheng Xu qp->flags |= ERDMA_QP_IN_FLUSHING;
1820edf42cbSCheng Xu mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
1830edf42cbSCheng Xu usecs_to_jiffies(100));
1840edf42cbSCheng Xu }
1850edf42cbSCheng Xu
18615505577SCheng Xu return ret;
18715505577SCheng Xu }
18815505577SCheng Xu
erdma_qp_safe_free(struct kref * ref)18915505577SCheng Xu static void erdma_qp_safe_free(struct kref *ref)
19015505577SCheng Xu {
19115505577SCheng Xu struct erdma_qp *qp = container_of(ref, struct erdma_qp, ref);
19215505577SCheng Xu
19315505577SCheng Xu complete(&qp->safe_free);
19415505577SCheng Xu }
19515505577SCheng Xu
erdma_qp_put(struct erdma_qp * qp)19615505577SCheng Xu void erdma_qp_put(struct erdma_qp *qp)
19715505577SCheng Xu {
19815505577SCheng Xu WARN_ON(kref_read(&qp->ref) < 1);
19915505577SCheng Xu kref_put(&qp->ref, erdma_qp_safe_free);
20015505577SCheng Xu }
20115505577SCheng Xu
erdma_qp_get(struct erdma_qp * qp)20215505577SCheng Xu void erdma_qp_get(struct erdma_qp *qp)
20315505577SCheng Xu {
20415505577SCheng Xu kref_get(&qp->ref);
20515505577SCheng Xu }
20615505577SCheng Xu
fill_inline_data(struct erdma_qp * qp,const struct ib_send_wr * send_wr,u16 wqe_idx,u32 sgl_offset,__le32 * length_field)20715505577SCheng Xu static int fill_inline_data(struct erdma_qp *qp,
20815505577SCheng Xu const struct ib_send_wr *send_wr, u16 wqe_idx,
20915505577SCheng Xu u32 sgl_offset, __le32 *length_field)
21015505577SCheng Xu {
21115505577SCheng Xu u32 remain_size, copy_size, data_off, bytes = 0;
21215505577SCheng Xu char *data;
21315505577SCheng Xu int i = 0;
21415505577SCheng Xu
21515505577SCheng Xu wqe_idx += (sgl_offset >> SQEBB_SHIFT);
21615505577SCheng Xu sgl_offset &= (SQEBB_SIZE - 1);
21715505577SCheng Xu data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, qp->attrs.sq_size,
21815505577SCheng Xu SQEBB_SHIFT);
21915505577SCheng Xu
22015505577SCheng Xu while (i < send_wr->num_sge) {
22115505577SCheng Xu bytes += send_wr->sg_list[i].length;
22215505577SCheng Xu if (bytes > (int)ERDMA_MAX_INLINE)
22315505577SCheng Xu return -EINVAL;
22415505577SCheng Xu
22515505577SCheng Xu remain_size = send_wr->sg_list[i].length;
22615505577SCheng Xu data_off = 0;
22715505577SCheng Xu
22815505577SCheng Xu while (1) {
22915505577SCheng Xu copy_size = min(remain_size, SQEBB_SIZE - sgl_offset);
23015505577SCheng Xu
23115505577SCheng Xu memcpy(data + sgl_offset,
23215505577SCheng Xu (void *)(uintptr_t)send_wr->sg_list[i].addr +
23315505577SCheng Xu data_off,
23415505577SCheng Xu copy_size);
23515505577SCheng Xu remain_size -= copy_size;
23615505577SCheng Xu data_off += copy_size;
23715505577SCheng Xu sgl_offset += copy_size;
23815505577SCheng Xu wqe_idx += (sgl_offset >> SQEBB_SHIFT);
23915505577SCheng Xu sgl_offset &= (SQEBB_SIZE - 1);
24015505577SCheng Xu
24115505577SCheng Xu data = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
24215505577SCheng Xu qp->attrs.sq_size, SQEBB_SHIFT);
24315505577SCheng Xu if (!remain_size)
24415505577SCheng Xu break;
24515505577SCheng Xu }
24615505577SCheng Xu
24715505577SCheng Xu i++;
24815505577SCheng Xu }
24915505577SCheng Xu *length_field = cpu_to_le32(bytes);
25015505577SCheng Xu
25115505577SCheng Xu return bytes;
25215505577SCheng Xu }
25315505577SCheng Xu
fill_sgl(struct erdma_qp * qp,const struct ib_send_wr * send_wr,u16 wqe_idx,u32 sgl_offset,__le32 * length_field)25415505577SCheng Xu static int fill_sgl(struct erdma_qp *qp, const struct ib_send_wr *send_wr,
25515505577SCheng Xu u16 wqe_idx, u32 sgl_offset, __le32 *length_field)
25615505577SCheng Xu {
25715505577SCheng Xu int i = 0;
25815505577SCheng Xu u32 bytes = 0;
25915505577SCheng Xu char *sgl;
26015505577SCheng Xu
26115505577SCheng Xu if (send_wr->num_sge > qp->dev->attrs.max_send_sge)
26215505577SCheng Xu return -EINVAL;
26315505577SCheng Xu
26415505577SCheng Xu if (sgl_offset & 0xF)
26515505577SCheng Xu return -EINVAL;
26615505577SCheng Xu
26715505577SCheng Xu while (i < send_wr->num_sge) {
26815505577SCheng Xu wqe_idx += (sgl_offset >> SQEBB_SHIFT);
26915505577SCheng Xu sgl_offset &= (SQEBB_SIZE - 1);
27015505577SCheng Xu sgl = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx,
27115505577SCheng Xu qp->attrs.sq_size, SQEBB_SHIFT);
27215505577SCheng Xu
27315505577SCheng Xu bytes += send_wr->sg_list[i].length;
27415505577SCheng Xu memcpy(sgl + sgl_offset, &send_wr->sg_list[i],
27515505577SCheng Xu sizeof(struct ib_sge));
27615505577SCheng Xu
27715505577SCheng Xu sgl_offset += sizeof(struct ib_sge);
27815505577SCheng Xu i++;
27915505577SCheng Xu }
28015505577SCheng Xu
28115505577SCheng Xu *length_field = cpu_to_le32(bytes);
28215505577SCheng Xu return 0;
28315505577SCheng Xu }
28415505577SCheng Xu
erdma_push_one_sqe(struct erdma_qp * qp,u16 * pi,const struct ib_send_wr * send_wr)28515505577SCheng Xu static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
28615505577SCheng Xu const struct ib_send_wr *send_wr)
28715505577SCheng Xu {
28815505577SCheng Xu u32 wqe_size, wqebb_cnt, hw_op, flags, sgl_offset;
28915505577SCheng Xu u32 idx = *pi & (qp->attrs.sq_size - 1);
29015505577SCheng Xu enum ib_wr_opcode op = send_wr->opcode;
2910ca9c2e2SCheng Xu struct erdma_atomic_sqe *atomic_sqe;
29215505577SCheng Xu struct erdma_readreq_sqe *read_sqe;
29315505577SCheng Xu struct erdma_reg_mr_sqe *regmr_sge;
29415505577SCheng Xu struct erdma_write_sqe *write_sqe;
29515505577SCheng Xu struct erdma_send_sqe *send_sqe;
29615505577SCheng Xu struct ib_rdma_wr *rdma_wr;
2970ca9c2e2SCheng Xu struct erdma_sge *sge;
29815505577SCheng Xu __le32 *length_field;
2990ca9c2e2SCheng Xu struct erdma_mr *mr;
30015505577SCheng Xu u64 wqe_hdr, *entry;
30115505577SCheng Xu u32 attrs;
30215505577SCheng Xu int ret;
30315505577SCheng Xu
30415505577SCheng Xu entry = get_queue_entry(qp->kern_qp.sq_buf, idx, qp->attrs.sq_size,
30515505577SCheng Xu SQEBB_SHIFT);
30615505577SCheng Xu
30715505577SCheng Xu /* Clear the SQE header section. */
30815505577SCheng Xu *entry = 0;
30915505577SCheng Xu
31015505577SCheng Xu qp->kern_qp.swr_tbl[idx] = send_wr->wr_id;
31115505577SCheng Xu flags = send_wr->send_flags;
31215505577SCheng Xu wqe_hdr = FIELD_PREP(
31315505577SCheng Xu ERDMA_SQE_HDR_CE_MASK,
31415505577SCheng Xu ((flags & IB_SEND_SIGNALED) || qp->kern_qp.sig_all) ? 1 : 0);
31515505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SE_MASK,
31615505577SCheng Xu flags & IB_SEND_SOLICITED ? 1 : 0);
31715505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_FENCE_MASK,
31815505577SCheng Xu flags & IB_SEND_FENCE ? 1 : 0);
31915505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_INLINE_MASK,
32015505577SCheng Xu flags & IB_SEND_INLINE ? 1 : 0);
32115505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp));
32215505577SCheng Xu
32315505577SCheng Xu switch (op) {
32415505577SCheng Xu case IB_WR_RDMA_WRITE:
32515505577SCheng Xu case IB_WR_RDMA_WRITE_WITH_IMM:
32615505577SCheng Xu hw_op = ERDMA_OP_WRITE;
32715505577SCheng Xu if (op == IB_WR_RDMA_WRITE_WITH_IMM)
32815505577SCheng Xu hw_op = ERDMA_OP_WRITE_WITH_IMM;
32915505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
33015505577SCheng Xu rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);
33115505577SCheng Xu write_sqe = (struct erdma_write_sqe *)entry;
33215505577SCheng Xu
33315505577SCheng Xu write_sqe->imm_data = send_wr->ex.imm_data;
33415505577SCheng Xu write_sqe->sink_stag = cpu_to_le32(rdma_wr->rkey);
33515505577SCheng Xu write_sqe->sink_to_h =
33615505577SCheng Xu cpu_to_le32(upper_32_bits(rdma_wr->remote_addr));
33715505577SCheng Xu write_sqe->sink_to_l =
33815505577SCheng Xu cpu_to_le32(lower_32_bits(rdma_wr->remote_addr));
33915505577SCheng Xu
34015505577SCheng Xu length_field = &write_sqe->length;
34115505577SCheng Xu wqe_size = sizeof(struct erdma_write_sqe);
34215505577SCheng Xu sgl_offset = wqe_size;
34315505577SCheng Xu break;
34415505577SCheng Xu case IB_WR_RDMA_READ:
34515505577SCheng Xu case IB_WR_RDMA_READ_WITH_INV:
34615505577SCheng Xu read_sqe = (struct erdma_readreq_sqe *)entry;
34715505577SCheng Xu if (unlikely(send_wr->num_sge != 1))
34815505577SCheng Xu return -EINVAL;
34915505577SCheng Xu hw_op = ERDMA_OP_READ;
35015505577SCheng Xu if (op == IB_WR_RDMA_READ_WITH_INV) {
35115505577SCheng Xu hw_op = ERDMA_OP_READ_WITH_INV;
35215505577SCheng Xu read_sqe->invalid_stag =
35315505577SCheng Xu cpu_to_le32(send_wr->ex.invalidate_rkey);
35415505577SCheng Xu }
35515505577SCheng Xu
35615505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
35715505577SCheng Xu rdma_wr = container_of(send_wr, struct ib_rdma_wr, wr);
35815505577SCheng Xu read_sqe->length = cpu_to_le32(send_wr->sg_list[0].length);
35915505577SCheng Xu read_sqe->sink_stag = cpu_to_le32(send_wr->sg_list[0].lkey);
36015505577SCheng Xu read_sqe->sink_to_l =
36115505577SCheng Xu cpu_to_le32(lower_32_bits(send_wr->sg_list[0].addr));
36215505577SCheng Xu read_sqe->sink_to_h =
36315505577SCheng Xu cpu_to_le32(upper_32_bits(send_wr->sg_list[0].addr));
36415505577SCheng Xu
36515505577SCheng Xu sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
36615505577SCheng Xu qp->attrs.sq_size, SQEBB_SHIFT);
3670ca9c2e2SCheng Xu sge->addr = cpu_to_le64(rdma_wr->remote_addr);
3680ca9c2e2SCheng Xu sge->key = cpu_to_le32(rdma_wr->rkey);
3690ca9c2e2SCheng Xu sge->length = cpu_to_le32(send_wr->sg_list[0].length);
37015505577SCheng Xu wqe_size = sizeof(struct erdma_readreq_sqe) +
37115505577SCheng Xu send_wr->num_sge * sizeof(struct ib_sge);
37215505577SCheng Xu
37315505577SCheng Xu goto out;
37415505577SCheng Xu case IB_WR_SEND:
37515505577SCheng Xu case IB_WR_SEND_WITH_IMM:
37615505577SCheng Xu case IB_WR_SEND_WITH_INV:
37715505577SCheng Xu send_sqe = (struct erdma_send_sqe *)entry;
37815505577SCheng Xu hw_op = ERDMA_OP_SEND;
37915505577SCheng Xu if (op == IB_WR_SEND_WITH_IMM) {
38015505577SCheng Xu hw_op = ERDMA_OP_SEND_WITH_IMM;
38115505577SCheng Xu send_sqe->imm_data = send_wr->ex.imm_data;
38215505577SCheng Xu } else if (op == IB_WR_SEND_WITH_INV) {
38315505577SCheng Xu hw_op = ERDMA_OP_SEND_WITH_INV;
38415505577SCheng Xu send_sqe->invalid_stag =
38515505577SCheng Xu cpu_to_le32(send_wr->ex.invalidate_rkey);
38615505577SCheng Xu }
38715505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, hw_op);
38815505577SCheng Xu length_field = &send_sqe->length;
38915505577SCheng Xu wqe_size = sizeof(struct erdma_send_sqe);
39015505577SCheng Xu sgl_offset = wqe_size;
39115505577SCheng Xu
39215505577SCheng Xu break;
39315505577SCheng Xu case IB_WR_REG_MR:
39415505577SCheng Xu wqe_hdr |=
39515505577SCheng Xu FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK, ERDMA_OP_REG_MR);
39615505577SCheng Xu regmr_sge = (struct erdma_reg_mr_sqe *)entry;
39715505577SCheng Xu mr = to_emr(reg_wr(send_wr)->mr);
39815505577SCheng Xu
39915505577SCheng Xu mr->access = ERDMA_MR_ACC_LR |
40015505577SCheng Xu to_erdma_access_flags(reg_wr(send_wr)->access);
40115505577SCheng Xu regmr_sge->addr = cpu_to_le64(mr->ibmr.iova);
40215505577SCheng Xu regmr_sge->length = cpu_to_le32(mr->ibmr.length);
4031374901bSCheng Xu regmr_sge->stag = cpu_to_le32(reg_wr(send_wr)->key);
404ece43fadSCheng Xu attrs = FIELD_PREP(ERDMA_SQE_MR_ACCESS_MASK, mr->access) |
40515505577SCheng Xu FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,
40615505577SCheng Xu mr->mem.mtt_nents);
40715505577SCheng Xu
4080dd83a4dSCheng Xu if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
40915505577SCheng Xu attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
41015505577SCheng Xu /* Copy SGLs to SQE content to accelerate */
41115505577SCheng Xu memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
41215505577SCheng Xu qp->attrs.sq_size, SQEBB_SHIFT),
413*7244b4aaSCheng Xu mr->mem.mtt->buf, MTT_SIZE(mr->mem.mtt_nents));
41415505577SCheng Xu wqe_size = sizeof(struct erdma_reg_mr_sqe) +
41515505577SCheng Xu MTT_SIZE(mr->mem.mtt_nents);
41615505577SCheng Xu } else {
41715505577SCheng Xu attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 1);
41815505577SCheng Xu wqe_size = sizeof(struct erdma_reg_mr_sqe);
41915505577SCheng Xu }
42015505577SCheng Xu
42115505577SCheng Xu regmr_sge->attrs = cpu_to_le32(attrs);
42215505577SCheng Xu goto out;
42315505577SCheng Xu case IB_WR_LOCAL_INV:
42415505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
42515505577SCheng Xu ERDMA_OP_LOCAL_INV);
42615505577SCheng Xu regmr_sge = (struct erdma_reg_mr_sqe *)entry;
42715505577SCheng Xu regmr_sge->stag = cpu_to_le32(send_wr->ex.invalidate_rkey);
42815505577SCheng Xu wqe_size = sizeof(struct erdma_reg_mr_sqe);
42915505577SCheng Xu goto out;
4300ca9c2e2SCheng Xu case IB_WR_ATOMIC_CMP_AND_SWP:
4310ca9c2e2SCheng Xu case IB_WR_ATOMIC_FETCH_AND_ADD:
4320ca9c2e2SCheng Xu atomic_sqe = (struct erdma_atomic_sqe *)entry;
4330ca9c2e2SCheng Xu if (op == IB_WR_ATOMIC_CMP_AND_SWP) {
4340ca9c2e2SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
4350ca9c2e2SCheng Xu ERDMA_OP_ATOMIC_CAS);
4360ca9c2e2SCheng Xu atomic_sqe->fetchadd_swap_data =
4370ca9c2e2SCheng Xu cpu_to_le64(atomic_wr(send_wr)->swap);
4380ca9c2e2SCheng Xu atomic_sqe->cmp_data =
4390ca9c2e2SCheng Xu cpu_to_le64(atomic_wr(send_wr)->compare_add);
4400ca9c2e2SCheng Xu } else {
4410ca9c2e2SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
4423fe26c04SCheng Xu ERDMA_OP_ATOMIC_FAA);
4430ca9c2e2SCheng Xu atomic_sqe->fetchadd_swap_data =
4440ca9c2e2SCheng Xu cpu_to_le64(atomic_wr(send_wr)->compare_add);
4450ca9c2e2SCheng Xu }
4460ca9c2e2SCheng Xu
4470ca9c2e2SCheng Xu sge = get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
4480ca9c2e2SCheng Xu qp->attrs.sq_size, SQEBB_SHIFT);
4490ca9c2e2SCheng Xu sge->addr = cpu_to_le64(atomic_wr(send_wr)->remote_addr);
4500ca9c2e2SCheng Xu sge->key = cpu_to_le32(atomic_wr(send_wr)->rkey);
4510ca9c2e2SCheng Xu sge++;
4520ca9c2e2SCheng Xu
4530ca9c2e2SCheng Xu sge->addr = cpu_to_le64(send_wr->sg_list[0].addr);
4540ca9c2e2SCheng Xu sge->key = cpu_to_le32(send_wr->sg_list[0].lkey);
4550ca9c2e2SCheng Xu sge->length = cpu_to_le32(send_wr->sg_list[0].length);
4560ca9c2e2SCheng Xu
4570ca9c2e2SCheng Xu wqe_size = sizeof(*atomic_sqe);
4580ca9c2e2SCheng Xu goto out;
45915505577SCheng Xu default:
46015505577SCheng Xu return -EOPNOTSUPP;
46115505577SCheng Xu }
46215505577SCheng Xu
46315505577SCheng Xu if (flags & IB_SEND_INLINE) {
46415505577SCheng Xu ret = fill_inline_data(qp, send_wr, idx, sgl_offset,
46515505577SCheng Xu length_field);
46615505577SCheng Xu if (ret < 0)
46715505577SCheng Xu return -EINVAL;
46815505577SCheng Xu wqe_size += ret;
46915505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK, ret);
47015505577SCheng Xu } else {
47115505577SCheng Xu ret = fill_sgl(qp, send_wr, idx, sgl_offset, length_field);
47215505577SCheng Xu if (ret)
47315505577SCheng Xu return -EINVAL;
47415505577SCheng Xu wqe_size += send_wr->num_sge * sizeof(struct ib_sge);
47515505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_SGL_LEN_MASK,
47615505577SCheng Xu send_wr->num_sge);
47715505577SCheng Xu }
47815505577SCheng Xu
47915505577SCheng Xu out:
48015505577SCheng Xu wqebb_cnt = SQEBB_COUNT(wqe_size);
48115505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_CNT_MASK, wqebb_cnt - 1);
48215505577SCheng Xu *pi += wqebb_cnt;
48315505577SCheng Xu wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, *pi);
48415505577SCheng Xu
48515505577SCheng Xu *entry = wqe_hdr;
48615505577SCheng Xu
48715505577SCheng Xu return 0;
48815505577SCheng Xu }
48915505577SCheng Xu
kick_sq_db(struct erdma_qp * qp,u16 pi)49015505577SCheng Xu static void kick_sq_db(struct erdma_qp *qp, u16 pi)
49115505577SCheng Xu {
49215505577SCheng Xu u64 db_data = FIELD_PREP(ERDMA_SQE_HDR_QPN_MASK, QP_ID(qp)) |
49315505577SCheng Xu FIELD_PREP(ERDMA_SQE_HDR_WQEBB_INDEX_MASK, pi);
49415505577SCheng Xu
49515505577SCheng Xu *(u64 *)qp->kern_qp.sq_db_info = db_data;
49615505577SCheng Xu writeq(db_data, qp->kern_qp.hw_sq_db);
49715505577SCheng Xu }
49815505577SCheng Xu
erdma_post_send(struct ib_qp * ibqp,const struct ib_send_wr * send_wr,const struct ib_send_wr ** bad_send_wr)49915505577SCheng Xu int erdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *send_wr,
50015505577SCheng Xu const struct ib_send_wr **bad_send_wr)
50115505577SCheng Xu {
50215505577SCheng Xu struct erdma_qp *qp = to_eqp(ibqp);
50315505577SCheng Xu int ret = 0;
50415505577SCheng Xu const struct ib_send_wr *wr = send_wr;
50515505577SCheng Xu unsigned long flags;
50615505577SCheng Xu u16 sq_pi;
50715505577SCheng Xu
50815505577SCheng Xu if (!send_wr)
50915505577SCheng Xu return -EINVAL;
51015505577SCheng Xu
51115505577SCheng Xu spin_lock_irqsave(&qp->lock, flags);
51215505577SCheng Xu sq_pi = qp->kern_qp.sq_pi;
51315505577SCheng Xu
51415505577SCheng Xu while (wr) {
51515505577SCheng Xu if ((u16)(sq_pi - qp->kern_qp.sq_ci) >= qp->attrs.sq_size) {
51615505577SCheng Xu ret = -ENOMEM;
51715505577SCheng Xu *bad_send_wr = send_wr;
51815505577SCheng Xu break;
51915505577SCheng Xu }
52015505577SCheng Xu
52115505577SCheng Xu ret = erdma_push_one_sqe(qp, &sq_pi, wr);
52215505577SCheng Xu if (ret) {
52315505577SCheng Xu *bad_send_wr = wr;
52415505577SCheng Xu break;
52515505577SCheng Xu }
52615505577SCheng Xu qp->kern_qp.sq_pi = sq_pi;
52715505577SCheng Xu kick_sq_db(qp, sq_pi);
52815505577SCheng Xu
52915505577SCheng Xu wr = wr->next;
53015505577SCheng Xu }
53115505577SCheng Xu spin_unlock_irqrestore(&qp->lock, flags);
53215505577SCheng Xu
5330edf42cbSCheng Xu if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
5340edf42cbSCheng Xu mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
5350edf42cbSCheng Xu usecs_to_jiffies(100));
5360edf42cbSCheng Xu
53715505577SCheng Xu return ret;
53815505577SCheng Xu }
53915505577SCheng Xu
erdma_post_recv_one(struct erdma_qp * qp,const struct ib_recv_wr * recv_wr)54015505577SCheng Xu static int erdma_post_recv_one(struct erdma_qp *qp,
54115505577SCheng Xu const struct ib_recv_wr *recv_wr)
54215505577SCheng Xu {
54315505577SCheng Xu struct erdma_rqe *rqe =
54415505577SCheng Xu get_queue_entry(qp->kern_qp.rq_buf, qp->kern_qp.rq_pi,
54515505577SCheng Xu qp->attrs.rq_size, RQE_SHIFT);
54615505577SCheng Xu
54715505577SCheng Xu rqe->qe_idx = cpu_to_le16(qp->kern_qp.rq_pi + 1);
54815505577SCheng Xu rqe->qpn = cpu_to_le32(QP_ID(qp));
54915505577SCheng Xu
55015505577SCheng Xu if (recv_wr->num_sge == 0) {
55115505577SCheng Xu rqe->length = 0;
55215505577SCheng Xu } else if (recv_wr->num_sge == 1) {
55315505577SCheng Xu rqe->stag = cpu_to_le32(recv_wr->sg_list[0].lkey);
55415505577SCheng Xu rqe->to = cpu_to_le64(recv_wr->sg_list[0].addr);
55515505577SCheng Xu rqe->length = cpu_to_le32(recv_wr->sg_list[0].length);
55615505577SCheng Xu } else {
55715505577SCheng Xu return -EINVAL;
55815505577SCheng Xu }
55915505577SCheng Xu
56015505577SCheng Xu *(u64 *)qp->kern_qp.rq_db_info = *(u64 *)rqe;
56115505577SCheng Xu writeq(*(u64 *)rqe, qp->kern_qp.hw_rq_db);
56215505577SCheng Xu
56315505577SCheng Xu qp->kern_qp.rwr_tbl[qp->kern_qp.rq_pi & (qp->attrs.rq_size - 1)] =
56415505577SCheng Xu recv_wr->wr_id;
56515505577SCheng Xu qp->kern_qp.rq_pi++;
56615505577SCheng Xu
56715505577SCheng Xu return 0;
56815505577SCheng Xu }
56915505577SCheng Xu
erdma_post_recv(struct ib_qp * ibqp,const struct ib_recv_wr * recv_wr,const struct ib_recv_wr ** bad_recv_wr)57015505577SCheng Xu int erdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *recv_wr,
57115505577SCheng Xu const struct ib_recv_wr **bad_recv_wr)
57215505577SCheng Xu {
57315505577SCheng Xu const struct ib_recv_wr *wr = recv_wr;
57415505577SCheng Xu struct erdma_qp *qp = to_eqp(ibqp);
57515505577SCheng Xu unsigned long flags;
57615505577SCheng Xu int ret;
57715505577SCheng Xu
57815505577SCheng Xu spin_lock_irqsave(&qp->lock, flags);
57915505577SCheng Xu
58015505577SCheng Xu while (wr) {
58115505577SCheng Xu ret = erdma_post_recv_one(qp, wr);
58215505577SCheng Xu if (ret) {
58315505577SCheng Xu *bad_recv_wr = wr;
58415505577SCheng Xu break;
58515505577SCheng Xu }
58615505577SCheng Xu wr = wr->next;
58715505577SCheng Xu }
58815505577SCheng Xu
58915505577SCheng Xu spin_unlock_irqrestore(&qp->lock, flags);
5900edf42cbSCheng Xu
5910edf42cbSCheng Xu if (unlikely(qp->flags & ERDMA_QP_IN_FLUSHING))
5920edf42cbSCheng Xu mod_delayed_work(qp->dev->reflush_wq, &qp->reflush_dwork,
5930edf42cbSCheng Xu usecs_to_jiffies(100));
5940edf42cbSCheng Xu
59515505577SCheng Xu return ret;
59615505577SCheng Xu }
597