1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 */ 6 #ifndef HFI1_TID_RDMA_H 7 #define HFI1_TID_RDMA_H 8 9 #include <linux/circ_buf.h> 10 #include "common.h" 11 12 /* Add a convenience helper */ 13 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 14 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 15 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 16 17 #define TID_RDMA_MIN_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 18 #define TID_RDMA_MAX_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 19 #define TID_RDMA_MAX_PAGES (BIT(18) >> PAGE_SHIFT) 20 21 /* 22 * Bit definitions for priv->s_flags. 23 * These bit flags overload the bit flags defined for the QP's s_flags. 24 * Due to the fact that these bit fields are used only for the QP priv 25 * s_flags, there are no collisions. 26 * 27 * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock 28 * HFI1_R_TID_WAIT_INTERLCK - QP is waiting for responder interlock 29 */ 30 #define HFI1_S_TID_BUSY_SET BIT(0) 31 /* BIT(1) reserved for RVT_S_BUSY. */ 32 #define HFI1_R_TID_RSC_TIMER BIT(2) 33 /* BIT(3) reserved for RVT_S_RESP_PENDING. */ 34 /* BIT(4) reserved for RVT_S_ACK_PENDING. */ 35 #define HFI1_S_TID_WAIT_INTERLCK BIT(5) 36 #define HFI1_R_TID_WAIT_INTERLCK BIT(6) 37 /* BIT(7) - BIT(15) reserved for RVT_S_WAIT_*. */ 38 /* BIT(16) reserved for RVT_S_SEND_ONE */ 39 #define HFI1_S_TID_RETRY_TIMER BIT(17) 40 /* BIT(18) reserved for RVT_S_ECN. */ 41 #define HFI1_R_TID_SW_PSN BIT(19) 42 /* BIT(26) reserved for HFI1_S_WAIT_HALT */ 43 /* BIT(27) reserved for HFI1_S_WAIT_TID_RESP */ 44 /* BIT(28) reserved for HFI1_S_WAIT_TID_SPACE */ 45 46 /* 47 * Unlike regular IB RDMA VERBS, which do not require an entry 48 * in the s_ack_queue, TID RDMA WRITE requests do because they 49 * generate responses. 50 * Therefore, the s_ack_queue needs to be extended by a certain 51 * amount. The key point is that the queue needs to be extended 52 * without letting the "user" know so they user doesn't end up 53 * using these extra entries. 54 */ 55 #define HFI1_TID_RDMA_WRITE_CNT 8 56 57 struct tid_rdma_params { 58 struct rcu_head rcu_head; 59 u32 qp; 60 u32 max_len; 61 u16 jkey; 62 u8 max_read; 63 u8 max_write; 64 u8 timeout; 65 u8 urg; 66 u8 version; 67 }; 68 69 struct tid_rdma_qp_params { 70 struct work_struct trigger_work; 71 struct tid_rdma_params local; 72 struct tid_rdma_params __rcu *remote; 73 }; 74 75 /* Track state for each hardware flow */ 76 struct tid_flow_state { 77 u32 generation; 78 u32 psn; 79 u32 r_next_psn; /* next PSN to be received (in TID space) */ 80 u8 index; 81 u8 last_index; 82 u8 flags; 83 }; 84 85 enum tid_rdma_req_state { 86 TID_REQUEST_INACTIVE = 0, 87 TID_REQUEST_INIT, 88 TID_REQUEST_INIT_RESEND, 89 TID_REQUEST_ACTIVE, 90 TID_REQUEST_RESEND, 91 TID_REQUEST_RESEND_ACTIVE, 92 TID_REQUEST_QUEUED, 93 TID_REQUEST_SYNC, 94 TID_REQUEST_RNR_NAK, 95 TID_REQUEST_COMPLETE, 96 }; 97 98 struct tid_rdma_request { 99 struct rvt_qp *qp; 100 struct hfi1_ctxtdata *rcd; 101 union { 102 struct rvt_swqe *swqe; 103 struct rvt_ack_entry *ack; 104 } e; 105 106 struct tid_rdma_flow *flows; /* array of tid flows */ 107 struct rvt_sge_state ss; /* SGE state for TID RDMA requests */ 108 u16 n_flows; /* size of the flow buffer window */ 109 u16 setup_head; /* flow index we are setting up */ 110 u16 clear_tail; /* flow index we are clearing */ 111 u16 flow_idx; /* flow index most recently set up */ 112 u16 acked_tail; 113 114 u32 seg_len; 115 u32 total_len; 116 u32 r_ack_psn; /* next expected ack PSN */ 117 u32 r_flow_psn; /* IB PSN of next segment start */ 118 u32 r_last_acked; /* IB PSN of last ACK'ed packet */ 119 u32 s_next_psn; /* IB PSN of next segment start for read */ 120 121 u32 total_segs; /* segments required to complete a request */ 122 u32 cur_seg; /* index of current segment */ 123 u32 comp_seg; /* index of last completed segment */ 124 u32 ack_seg; /* index of last ack'ed segment */ 125 u32 alloc_seg; /* index of next segment to be allocated */ 126 u32 isge; /* index of "current" sge */ 127 u32 ack_pending; /* num acks pending for this request */ 128 129 enum tid_rdma_req_state state; 130 }; 131 132 /* 133 * When header suppression is used, PSNs associated with a "flow" are 134 * relevant (and not the PSNs maintained by verbs). Track per-flow 135 * PSNs here for a TID RDMA segment. 136 * 137 */ 138 struct flow_state { 139 u32 flags; 140 u32 resp_ib_psn; /* The IB PSN of the response for this flow */ 141 u32 generation; /* generation of flow */ 142 u32 spsn; /* starting PSN in TID space */ 143 u32 lpsn; /* last PSN in TID space */ 144 u32 r_next_psn; /* next PSN to be received (in TID space) */ 145 146 /* For tid rdma read */ 147 u32 ib_spsn; /* starting PSN in Verbs space */ 148 u32 ib_lpsn; /* last PSn in Verbs space */ 149 }; 150 151 struct tid_rdma_pageset { 152 dma_addr_t addr : 48; /* Only needed for the first page */ 153 u8 idx: 8; 154 u8 count : 7; 155 u8 mapped: 1; 156 }; 157 158 /** 159 * kern_tid_node - used for managing TID's in TID groups 160 * 161 * @grp_idx: rcd relative index to tid_group 162 * @map: grp->map captured prior to programming this TID group in HW 163 * @cnt: Only @cnt of available group entries are actually programmed 164 */ 165 struct kern_tid_node { 166 struct tid_group *grp; 167 u8 map; 168 u8 cnt; 169 }; 170 171 /* Overall info for a TID RDMA segment */ 172 struct tid_rdma_flow { 173 /* 174 * While a TID RDMA segment is being transferred, it uses a QP number 175 * from the "KDETH section of QP numbers" (which is different from the 176 * QP number that originated the request). Bits 11-15 of these QP 177 * numbers identify the "TID flow" for the segment. 178 */ 179 struct flow_state flow_state; 180 struct tid_rdma_request *req; 181 u32 tid_qpn; 182 u32 tid_offset; 183 u32 length; 184 u32 sent; 185 u8 tnode_cnt; 186 u8 tidcnt; 187 u8 tid_idx; 188 u8 idx; 189 u8 npagesets; 190 u8 npkts; 191 u8 pkt; 192 u8 resync_npkts; 193 struct kern_tid_node tnode[TID_RDMA_MAX_PAGES]; 194 struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES]; 195 u32 tid_entry[TID_RDMA_MAX_PAGES]; 196 }; 197 198 enum tid_rnr_nak_state { 199 TID_RNR_NAK_INIT = 0, 200 TID_RNR_NAK_SEND, 201 TID_RNR_NAK_SENT, 202 }; 203 204 bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data); 205 bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data); 206 bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data); 207 void tid_rdma_conn_error(struct rvt_qp *qp); 208 void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p); 209 210 int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit); 211 int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req, 212 struct rvt_sge_state *ss, bool *last); 213 int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req); 214 void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req); 215 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 216 217 /** 218 * trdma_clean_swqe - clean flows for swqe if large send queue 219 * @qp: the qp 220 * @wqe: the send wqe 221 */ 222 static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) 223 { 224 if (!wqe->priv) 225 return; 226 __trdma_clean_swqe(qp, wqe); 227 } 228 229 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp); 230 231 int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, 232 struct ib_qp_init_attr *init_attr); 233 void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); 234 235 void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp); 236 237 int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 238 void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 239 void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd); 240 241 struct cntr_entry; 242 u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry, 243 void *context, int vl, int mode, u64 data); 244 245 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, 246 struct ib_other_headers *ohdr, 247 u32 *bth1, u32 *bth2, u32 *len); 248 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 249 struct ib_other_headers *ohdr, u32 *bth1, 250 u32 *bth2, u32 *len); 251 void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet); 252 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 253 struct ib_other_headers *ohdr, u32 *bth0, 254 u32 *bth1, u32 *bth2, u32 *len, bool *last); 255 void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet); 256 bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, 257 struct hfi1_pportdata *ppd, 258 struct hfi1_packet *packet); 259 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 260 u32 *bth2); 261 void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp); 262 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); 263 264 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 265 static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp, 266 struct rvt_swqe *wqe) 267 { 268 if (wqe->priv && 269 (wqe->wr.opcode == IB_WR_RDMA_READ || 270 wqe->wr.opcode == IB_WR_RDMA_WRITE) && 271 wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE) 272 setup_tid_rdma_wqe(qp, wqe); 273 } 274 275 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 276 struct ib_other_headers *ohdr, 277 u32 *bth1, u32 *bth2, u32 *len); 278 279 void hfi1_compute_tid_rdma_flow_wt(void); 280 281 void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet); 282 283 u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 284 struct ib_other_headers *ohdr, u32 *bth1, 285 u32 bth2, u32 *len, 286 struct rvt_sge_state **ss); 287 288 void hfi1_del_tid_reap_timer(struct rvt_qp *qp); 289 290 void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet); 291 292 bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, 293 struct ib_other_headers *ohdr, 294 u32 *bth1, u32 *bth2, u32 *len); 295 296 void hfi1_rc_rcv_tid_rdma_write_data(struct hfi1_packet *packet); 297 298 u32 hfi1_build_tid_rdma_write_ack(struct rvt_qp *qp, struct rvt_ack_entry *e, 299 struct ib_other_headers *ohdr, u16 iflow, 300 u32 *bth1, u32 *bth2); 301 302 void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet); 303 304 void hfi1_add_tid_retry_timer(struct rvt_qp *qp); 305 void hfi1_del_tid_retry_timer(struct rvt_qp *qp); 306 307 u32 hfi1_build_tid_rdma_resync(struct rvt_qp *qp, struct rvt_swqe *wqe, 308 struct ib_other_headers *ohdr, u32 *bth1, 309 u32 *bth2, u16 fidx); 310 311 void hfi1_rc_rcv_tid_rdma_resync(struct hfi1_packet *packet); 312 313 struct hfi1_pkt_state; 314 int hfi1_make_tid_rdma_pkt(struct rvt_qp *qp, struct hfi1_pkt_state *ps); 315 316 void _hfi1_do_tid_send(struct work_struct *work); 317 318 bool hfi1_schedule_tid_send(struct rvt_qp *qp); 319 320 bool hfi1_tid_rdma_ack_interlock(struct rvt_qp *qp, struct rvt_ack_entry *e); 321 322 #endif /* HFI1_TID_RDMA_H */ 323