1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #include <rdma/ib_verbs.h> 8 9 #include "erdma_hw.h" 10 #include "erdma_verbs.h" 11 12 static void *get_next_valid_cqe(struct erdma_cq *cq) 13 { 14 __be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci, 15 cq->depth, CQE_SHIFT); 16 u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK, 17 __be32_to_cpu(READ_ONCE(*cqe))); 18 19 return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe : NULL; 20 } 21 22 static void notify_cq(struct erdma_cq *cq, u8 solcitied) 23 { 24 u64 db_data = 25 FIELD_PREP(ERDMA_CQDB_IDX_MASK, (cq->kern_cq.notify_cnt)) | 26 FIELD_PREP(ERDMA_CQDB_CQN_MASK, cq->cqn) | 27 FIELD_PREP(ERDMA_CQDB_ARM_MASK, 1) | 28 FIELD_PREP(ERDMA_CQDB_SOL_MASK, solcitied) | 29 FIELD_PREP(ERDMA_CQDB_CMDSN_MASK, cq->kern_cq.cmdsn) | 30 FIELD_PREP(ERDMA_CQDB_CI_MASK, cq->kern_cq.ci); 31 32 *cq->kern_cq.db_record = db_data; 33 writeq(db_data, cq->kern_cq.db); 34 } 35 36 int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 37 { 38 struct erdma_cq *cq = to_ecq(ibcq); 39 unsigned long irq_flags; 40 int ret = 0; 41 42 spin_lock_irqsave(&cq->kern_cq.lock, irq_flags); 43 44 notify_cq(cq, (flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED); 45 46 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && get_next_valid_cqe(cq)) 47 ret = 1; 48 49 cq->kern_cq.notify_cnt++; 50 51 spin_unlock_irqrestore(&cq->kern_cq.lock, irq_flags); 52 53 return ret; 54 } 55 56 static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = { 57 [ERDMA_OP_WRITE] = IB_WC_RDMA_WRITE, 58 [ERDMA_OP_READ] = IB_WC_RDMA_READ, 59 [ERDMA_OP_SEND] = IB_WC_SEND, 60 [ERDMA_OP_SEND_WITH_IMM] = IB_WC_SEND, 61 [ERDMA_OP_RECEIVE] = IB_WC_RECV, 62 [ERDMA_OP_RECV_IMM] = IB_WC_RECV_RDMA_WITH_IMM, 63 [ERDMA_OP_RECV_INV] = IB_WC_RECV, 64 [ERDMA_OP_WRITE_WITH_IMM] = IB_WC_RDMA_WRITE, 65 [ERDMA_OP_INVALIDATE] = IB_WC_LOCAL_INV, 66 [ERDMA_OP_RSP_SEND_IMM] = IB_WC_RECV, 67 [ERDMA_OP_SEND_WITH_INV] = IB_WC_SEND, 68 [ERDMA_OP_REG_MR] = IB_WC_REG_MR, 69 [ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV, 70 [ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ, 71 }; 72 73 static const struct { 74 enum erdma_wc_status erdma; 75 enum ib_wc_status base; 76 enum erdma_vendor_err vendor; 77 } map_cqe_status[ERDMA_NUM_WC_STATUS] = { 78 { ERDMA_WC_SUCCESS, IB_WC_SUCCESS, ERDMA_WC_VENDOR_NO_ERR }, 79 { ERDMA_WC_GENERAL_ERR, IB_WC_GENERAL_ERR, ERDMA_WC_VENDOR_NO_ERR }, 80 { ERDMA_WC_RECV_WQE_FORMAT_ERR, IB_WC_GENERAL_ERR, 81 ERDMA_WC_VENDOR_INVALID_RQE }, 82 { ERDMA_WC_RECV_STAG_INVALID_ERR, IB_WC_REM_ACCESS_ERR, 83 ERDMA_WC_VENDOR_RQE_INVALID_STAG }, 84 { ERDMA_WC_RECV_ADDR_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR, 85 ERDMA_WC_VENDOR_RQE_ADDR_VIOLATION }, 86 { ERDMA_WC_RECV_RIGHT_VIOLATION_ERR, IB_WC_REM_ACCESS_ERR, 87 ERDMA_WC_VENDOR_RQE_ACCESS_RIGHT_ERR }, 88 { ERDMA_WC_RECV_PDID_ERR, IB_WC_REM_ACCESS_ERR, 89 ERDMA_WC_VENDOR_RQE_INVALID_PD }, 90 { ERDMA_WC_RECV_WARRPING_ERR, IB_WC_REM_ACCESS_ERR, 91 ERDMA_WC_VENDOR_RQE_WRAP_ERR }, 92 { ERDMA_WC_SEND_WQE_FORMAT_ERR, IB_WC_LOC_QP_OP_ERR, 93 ERDMA_WC_VENDOR_INVALID_SQE }, 94 { ERDMA_WC_SEND_WQE_ORD_EXCEED, IB_WC_GENERAL_ERR, 95 ERDMA_WC_VENDOR_ZERO_ORD }, 96 { ERDMA_WC_SEND_STAG_INVALID_ERR, IB_WC_LOC_ACCESS_ERR, 97 ERDMA_WC_VENDOR_SQE_INVALID_STAG }, 98 { ERDMA_WC_SEND_ADDR_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR, 99 ERDMA_WC_VENDOR_SQE_ADDR_VIOLATION }, 100 { ERDMA_WC_SEND_RIGHT_VIOLATION_ERR, IB_WC_LOC_ACCESS_ERR, 101 ERDMA_WC_VENDOR_SQE_ACCESS_ERR }, 102 { ERDMA_WC_SEND_PDID_ERR, IB_WC_LOC_ACCESS_ERR, 103 ERDMA_WC_VENDOR_SQE_INVALID_PD }, 104 { ERDMA_WC_SEND_WARRPING_ERR, IB_WC_LOC_ACCESS_ERR, 105 ERDMA_WC_VENDOR_SQE_WARP_ERR }, 106 { ERDMA_WC_FLUSH_ERR, IB_WC_WR_FLUSH_ERR, ERDMA_WC_VENDOR_NO_ERR }, 107 { ERDMA_WC_RETRY_EXC_ERR, IB_WC_RETRY_EXC_ERR, ERDMA_WC_VENDOR_NO_ERR }, 108 }; 109 110 #define ERDMA_POLLCQ_NO_QP 1 111 112 static int erdma_poll_one_cqe(struct erdma_cq *cq, struct ib_wc *wc) 113 { 114 struct erdma_dev *dev = to_edev(cq->ibcq.device); 115 u8 opcode, syndrome, qtype; 116 struct erdma_kqp *kern_qp; 117 struct erdma_cqe *cqe; 118 struct erdma_qp *qp; 119 u16 wqe_idx, depth; 120 u32 qpn, cqe_hdr; 121 u64 *id_table; 122 u64 *wqe_hdr; 123 124 cqe = get_next_valid_cqe(cq); 125 if (!cqe) 126 return -EAGAIN; 127 128 cq->kern_cq.ci++; 129 130 /* cqbuf should be ready when we poll */ 131 dma_rmb(); 132 133 qpn = be32_to_cpu(cqe->qpn); 134 wqe_idx = be32_to_cpu(cqe->qe_idx); 135 cqe_hdr = be32_to_cpu(cqe->hdr); 136 137 qp = find_qp_by_qpn(dev, qpn); 138 if (!qp) 139 return ERDMA_POLLCQ_NO_QP; 140 141 kern_qp = &qp->kern_qp; 142 143 qtype = FIELD_GET(ERDMA_CQE_HDR_QTYPE_MASK, cqe_hdr); 144 syndrome = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, cqe_hdr); 145 opcode = FIELD_GET(ERDMA_CQE_HDR_OPCODE_MASK, cqe_hdr); 146 147 if (qtype == ERDMA_CQE_QTYPE_SQ) { 148 id_table = kern_qp->swr_tbl; 149 depth = qp->attrs.sq_size; 150 wqe_hdr = get_queue_entry(qp->kern_qp.sq_buf, wqe_idx, 151 qp->attrs.sq_size, SQEBB_SHIFT); 152 kern_qp->sq_ci = 153 FIELD_GET(ERDMA_SQE_HDR_WQEBB_CNT_MASK, *wqe_hdr) + 154 wqe_idx + 1; 155 } else { 156 id_table = kern_qp->rwr_tbl; 157 depth = qp->attrs.rq_size; 158 } 159 wc->wr_id = id_table[wqe_idx & (depth - 1)]; 160 wc->byte_len = be32_to_cpu(cqe->size); 161 162 wc->wc_flags = 0; 163 164 wc->opcode = wc_mapping_table[opcode]; 165 if (opcode == ERDMA_OP_RECV_IMM || opcode == ERDMA_OP_RSP_SEND_IMM) { 166 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(cqe->imm_data)); 167 wc->wc_flags |= IB_WC_WITH_IMM; 168 } else if (opcode == ERDMA_OP_RECV_INV) { 169 wc->ex.invalidate_rkey = be32_to_cpu(cqe->inv_rkey); 170 wc->wc_flags |= IB_WC_WITH_INVALIDATE; 171 } 172 173 if (syndrome >= ERDMA_NUM_WC_STATUS) 174 syndrome = ERDMA_WC_GENERAL_ERR; 175 176 wc->status = map_cqe_status[syndrome].base; 177 wc->vendor_err = map_cqe_status[syndrome].vendor; 178 wc->qp = &qp->ibqp; 179 180 return 0; 181 } 182 183 int erdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 184 { 185 struct erdma_cq *cq = to_ecq(ibcq); 186 unsigned long flags; 187 int npolled, ret; 188 189 spin_lock_irqsave(&cq->kern_cq.lock, flags); 190 191 for (npolled = 0; npolled < num_entries;) { 192 ret = erdma_poll_one_cqe(cq, wc + npolled); 193 194 if (ret == -EAGAIN) /* no received new CQEs. */ 195 break; 196 else if (ret) /* ignore invalid CQEs. */ 197 continue; 198 199 npolled++; 200 } 201 202 spin_unlock_irqrestore(&cq->kern_cq.lock, flags); 203 204 return npolled; 205 } 206