1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 */ 6 #include "hfi.h" 7 #include "trace.h" 8 #include "qp.h" 9 #include "opfn.h" 10 11 #define IB_BTHE_E BIT(IB_BTHE_E_SHIFT) 12 13 #define OPFN_CODE(code) BIT((code) - 1) 14 #define OPFN_MASK(code) OPFN_CODE(STL_VERBS_EXTD_##code) 15 16 struct hfi1_opfn_type { 17 bool (*request)(struct rvt_qp *qp, u64 *data); 18 bool (*response)(struct rvt_qp *qp, u64 *data); 19 bool (*reply)(struct rvt_qp *qp, u64 data); 20 void (*error)(struct rvt_qp *qp); 21 }; 22 23 static struct hfi1_opfn_type hfi1_opfn_handlers[STL_VERBS_EXTD_MAX] = { 24 [STL_VERBS_EXTD_TID_RDMA] = { 25 .request = tid_rdma_conn_req, 26 .response = tid_rdma_conn_resp, 27 .reply = tid_rdma_conn_reply, 28 .error = tid_rdma_conn_error, 29 }, 30 }; 31 32 static struct workqueue_struct *opfn_wq; 33 34 static void opfn_schedule_conn_request(struct rvt_qp *qp); 35 36 static bool hfi1_opfn_extended(u32 bth1) 37 { 38 return !!(bth1 & IB_BTHE_E); 39 } 40 41 static void opfn_conn_request(struct rvt_qp *qp) 42 { 43 struct hfi1_qp_priv *priv = qp->priv; 44 struct ib_atomic_wr wr; 45 u16 mask, capcode; 46 struct hfi1_opfn_type *extd; 47 u64 data; 48 unsigned long flags; 49 int ret = 0; 50 51 spin_lock_irqsave(&priv->opfn.lock, flags); 52 /* 53 * Exit if the extended bit is not set, or if nothing is requested, or 54 * if we have completed all requests, or if a previous request is in 55 * progress 56 */ 57 if (!priv->opfn.extended || !priv->opfn.requested || 58 priv->opfn.requested == priv->opfn.completed || priv->opfn.curr) 59 goto done; 60 61 mask = priv->opfn.requested & ~priv->opfn.completed; 62 capcode = ilog2(mask & ~(mask - 1)) + 1; 63 if (capcode >= STL_VERBS_EXTD_MAX) { 64 priv->opfn.completed |= OPFN_CODE(capcode); 65 goto done; 66 } 67 68 extd = &hfi1_opfn_handlers[capcode]; 69 if (!extd || !extd->request || !extd->request(qp, &data)) { 70 /* 71 * Either there is no handler for this capability or the request 72 * packet could not be generated. Either way, mark it as done so 73 * we don't keep attempting to complete it. 74 */ 75 priv->opfn.completed |= OPFN_CODE(capcode); 76 goto done; 77 } 78 79 data = (data & ~0xf) | capcode; 80 81 memset(&wr, 0, sizeof(wr)); 82 wr.wr.opcode = IB_WR_OPFN; 83 wr.remote_addr = HFI1_VERBS_E_ATOMIC_VADDR; 84 wr.compare_add = data; 85 86 priv->opfn.curr = capcode; /* A new request is now in progress */ 87 /* Drop opfn.lock before calling ib_post_send() */ 88 spin_unlock_irqrestore(&priv->opfn.lock, flags); 89 90 ret = ib_post_send(&qp->ibqp, &wr.wr, NULL); 91 if (ret) 92 goto err; 93 return; 94 err: 95 spin_lock_irqsave(&priv->opfn.lock, flags); 96 /* 97 * In case of an unexpected error return from ib_post_send 98 * clear opfn.curr and reschedule to try again 99 */ 100 priv->opfn.curr = STL_VERBS_EXTD_NONE; 101 opfn_schedule_conn_request(qp); 102 done: 103 spin_unlock_irqrestore(&priv->opfn.lock, flags); 104 } 105 106 void opfn_send_conn_request(struct work_struct *work) 107 { 108 struct hfi1_opfn_data *od; 109 struct hfi1_qp_priv *qpriv; 110 111 od = container_of(work, struct hfi1_opfn_data, opfn_work); 112 qpriv = container_of(od, struct hfi1_qp_priv, opfn); 113 114 opfn_conn_request(qpriv->owner); 115 } 116 117 /* 118 * When QP s_lock is held in the caller, the OPFN request must be scheduled 119 * to a different workqueue to avoid double locking QP s_lock in call to 120 * ib_post_send in opfn_conn_request 121 */ 122 static void opfn_schedule_conn_request(struct rvt_qp *qp) 123 { 124 struct hfi1_qp_priv *priv = qp->priv; 125 126 queue_work(opfn_wq, &priv->opfn.opfn_work); 127 } 128 129 void opfn_conn_response(struct rvt_qp *qp, struct rvt_ack_entry *e, 130 struct ib_atomic_eth *ateth) 131 { 132 struct hfi1_qp_priv *priv = qp->priv; 133 u64 data = be64_to_cpu(ateth->compare_data); 134 struct hfi1_opfn_type *extd; 135 u8 capcode; 136 unsigned long flags; 137 138 capcode = data & 0xf; 139 if (!capcode || capcode >= STL_VERBS_EXTD_MAX) 140 return; 141 142 extd = &hfi1_opfn_handlers[capcode]; 143 144 if (!extd || !extd->response) { 145 e->atomic_data = capcode; 146 return; 147 } 148 149 spin_lock_irqsave(&priv->opfn.lock, flags); 150 if (priv->opfn.completed & OPFN_CODE(capcode)) { 151 /* 152 * We are receiving a request for a feature that has already 153 * been negotiated. This may mean that the other side has reset 154 */ 155 priv->opfn.completed &= ~OPFN_CODE(capcode); 156 if (extd->error) 157 extd->error(qp); 158 } 159 160 if (extd->response(qp, &data)) 161 priv->opfn.completed |= OPFN_CODE(capcode); 162 e->atomic_data = (data & ~0xf) | capcode; 163 spin_unlock_irqrestore(&priv->opfn.lock, flags); 164 } 165 166 void opfn_conn_reply(struct rvt_qp *qp, u64 data) 167 { 168 struct hfi1_qp_priv *priv = qp->priv; 169 struct hfi1_opfn_type *extd; 170 u8 capcode; 171 unsigned long flags; 172 173 capcode = data & 0xf; 174 if (!capcode || capcode >= STL_VERBS_EXTD_MAX) 175 return; 176 177 spin_lock_irqsave(&priv->opfn.lock, flags); 178 /* 179 * Either there is no previous request or the reply is not for the 180 * current request 181 */ 182 if (!priv->opfn.curr || capcode != priv->opfn.curr) 183 goto done; 184 185 extd = &hfi1_opfn_handlers[capcode]; 186 187 if (!extd || !extd->reply) 188 goto clear; 189 190 if (extd->reply(qp, data)) 191 priv->opfn.completed |= OPFN_CODE(capcode); 192 clear: 193 /* 194 * Clear opfn.curr to indicate that the previous request is no longer in 195 * progress 196 */ 197 priv->opfn.curr = STL_VERBS_EXTD_NONE; 198 done: 199 spin_unlock_irqrestore(&priv->opfn.lock, flags); 200 } 201 202 void opfn_conn_error(struct rvt_qp *qp) 203 { 204 struct hfi1_qp_priv *priv = qp->priv; 205 struct hfi1_opfn_type *extd = NULL; 206 unsigned long flags; 207 u16 capcode; 208 209 /* 210 * The QP has gone into the Error state. We have to invalidate all 211 * negotiated feature, including the one in progress (if any). The RC 212 * QP handling will clean the WQE for the connection request. 213 */ 214 spin_lock_irqsave(&priv->opfn.lock, flags); 215 while (priv->opfn.completed) { 216 capcode = priv->opfn.completed & ~(priv->opfn.completed - 1); 217 extd = &hfi1_opfn_handlers[ilog2(capcode) + 1]; 218 if (extd->error) 219 extd->error(qp); 220 priv->opfn.completed &= ~OPFN_CODE(capcode); 221 } 222 priv->opfn.extended = 0; 223 priv->opfn.requested = 0; 224 priv->opfn.curr = STL_VERBS_EXTD_NONE; 225 spin_unlock_irqrestore(&priv->opfn.lock, flags); 226 } 227 228 void opfn_qp_init(struct rvt_qp *qp, struct ib_qp_attr *attr, int attr_mask) 229 { 230 struct ib_qp *ibqp = &qp->ibqp; 231 struct hfi1_qp_priv *priv = qp->priv; 232 unsigned long flags; 233 234 spin_lock_irqsave(&priv->opfn.lock, flags); 235 if (ibqp->qp_type == IB_QPT_RC && HFI1_CAP_IS_KSET(TID_RDMA)) { 236 struct tid_rdma_params *local = &priv->tid_rdma.local; 237 238 if (qp->pmtu == enum_to_mtu(OPA_MTU_4096) || 239 qp->pmtu == enum_to_mtu(OPA_MTU_8192)) { 240 tid_rdma_opfn_init(qp, local); 241 /* 242 * We only want to set the OPFN requested bit when the 243 * QP transitions to RTS. 244 */ 245 if (attr_mask & IB_QP_STATE && 246 attr->qp_state == IB_QPS_RTS) { 247 priv->opfn.requested |= OPFN_MASK(TID_RDMA); 248 /* 249 * If the QP is transitioning to RTS and the 250 * opfn.completed for TID RDMA has already been 251 * set, the QP is being moved *back* into RTS. 252 * We can now renegotiate the TID RDMA 253 * parameters. 254 */ 255 if (priv->opfn.completed & 256 OPFN_MASK(TID_RDMA)) { 257 priv->opfn.completed &= 258 ~OPFN_MASK(TID_RDMA); 259 /* 260 * Since the opfn.completed bit was 261 * already set, it is safe to assume 262 * that the opfn.extended is also set. 263 */ 264 opfn_schedule_conn_request(qp); 265 } 266 } 267 } else { 268 memset(local, 0, sizeof(*local)); 269 } 270 } 271 spin_unlock_irqrestore(&priv->opfn.lock, flags); 272 } 273 274 void opfn_trigger_conn_request(struct rvt_qp *qp, u32 bth1) 275 { 276 struct hfi1_qp_priv *priv = qp->priv; 277 278 if (!priv->opfn.extended && hfi1_opfn_extended(bth1) && 279 HFI1_CAP_IS_KSET(OPFN)) { 280 priv->opfn.extended = 1; 281 if (qp->state == IB_QPS_RTS) 282 opfn_conn_request(qp); 283 } 284 } 285 286 int opfn_init(void) 287 { 288 opfn_wq = alloc_workqueue("hfi_opfn", 289 WQ_SYSFS | WQ_HIGHPRI | WQ_CPU_INTENSIVE | 290 WQ_MEM_RECLAIM, 291 HFI1_MAX_ACTIVE_WORKQUEUE_ENTRIES); 292 if (!opfn_wq) 293 return -ENOMEM; 294 295 return 0; 296 } 297 298 void opfn_exit(void) 299 { 300 if (opfn_wq) { 301 destroy_workqueue(opfn_wq); 302 opfn_wq = NULL; 303 } 304 } 305