1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 */ 6 #ifndef HFI1_TID_RDMA_H 7 #define HFI1_TID_RDMA_H 8 9 #include <linux/circ_buf.h> 10 #include "common.h" 11 12 /* Add a convenience helper */ 13 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 14 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 15 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 16 17 #define TID_RDMA_MIN_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 18 #define TID_RDMA_MAX_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 19 #define TID_RDMA_MAX_PAGES (BIT(18) >> PAGE_SHIFT) 20 21 /* 22 * Bit definitions for priv->s_flags. 23 * These bit flags overload the bit flags defined for the QP's s_flags. 24 * Due to the fact that these bit fields are used only for the QP priv 25 * s_flags, there are no collisions. 26 * 27 * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock 28 */ 29 #define HFI1_R_TID_RSC_TIMER BIT(2) 30 /* BIT(4) reserved for RVT_S_ACK_PENDING. */ 31 #define HFI1_S_TID_WAIT_INTERLCK BIT(5) 32 #define HFI1_R_TID_SW_PSN BIT(19) 33 34 /* 35 * Unlike regular IB RDMA VERBS, which do not require an entry 36 * in the s_ack_queue, TID RDMA WRITE requests do because they 37 * generate responses. 38 * Therefore, the s_ack_queue needs to be extended by a certain 39 * amount. The key point is that the queue needs to be extended 40 * without letting the "user" know so they user doesn't end up 41 * using these extra entries. 42 */ 43 #define HFI1_TID_RDMA_WRITE_CNT 8 44 45 struct tid_rdma_params { 46 struct rcu_head rcu_head; 47 u32 qp; 48 u32 max_len; 49 u16 jkey; 50 u8 max_read; 51 u8 max_write; 52 u8 timeout; 53 u8 urg; 54 u8 version; 55 }; 56 57 struct tid_rdma_qp_params { 58 struct work_struct trigger_work; 59 struct tid_rdma_params local; 60 struct tid_rdma_params __rcu *remote; 61 }; 62 63 /* Track state for each hardware flow */ 64 struct tid_flow_state { 65 u32 generation; 66 u32 psn; 67 u32 r_next_psn; /* next PSN to be received (in TID space) */ 68 u8 index; 69 u8 last_index; 70 u8 flags; 71 }; 72 73 enum tid_rdma_req_state { 74 TID_REQUEST_INACTIVE = 0, 75 TID_REQUEST_INIT, 76 TID_REQUEST_INIT_RESEND, 77 TID_REQUEST_ACTIVE, 78 TID_REQUEST_RESEND, 79 TID_REQUEST_RESEND_ACTIVE, 80 TID_REQUEST_QUEUED, 81 TID_REQUEST_SYNC, 82 TID_REQUEST_RNR_NAK, 83 TID_REQUEST_COMPLETE, 84 }; 85 86 struct tid_rdma_request { 87 struct rvt_qp *qp; 88 struct hfi1_ctxtdata *rcd; 89 union { 90 struct rvt_swqe *swqe; 91 struct rvt_ack_entry *ack; 92 } e; 93 94 struct tid_rdma_flow *flows; /* array of tid flows */ 95 struct rvt_sge_state ss; /* SGE state for TID RDMA requests */ 96 u16 n_flows; /* size of the flow buffer window */ 97 u16 setup_head; /* flow index we are setting up */ 98 u16 clear_tail; /* flow index we are clearing */ 99 u16 flow_idx; /* flow index most recently set up */ 100 u16 acked_tail; 101 102 u32 seg_len; 103 u32 total_len; 104 u32 r_flow_psn; /* IB PSN of next segment start */ 105 u32 r_last_acked; /* IB PSN of last ACK'ed packet */ 106 u32 s_next_psn; /* IB PSN of next segment start for read */ 107 108 u32 total_segs; /* segments required to complete a request */ 109 u32 cur_seg; /* index of current segment */ 110 u32 comp_seg; /* index of last completed segment */ 111 u32 ack_seg; /* index of last ack'ed segment */ 112 u32 alloc_seg; /* index of next segment to be allocated */ 113 u32 isge; /* index of "current" sge */ 114 u32 ack_pending; /* num acks pending for this request */ 115 116 enum tid_rdma_req_state state; 117 }; 118 119 /* 120 * When header suppression is used, PSNs associated with a "flow" are 121 * relevant (and not the PSNs maintained by verbs). Track per-flow 122 * PSNs here for a TID RDMA segment. 123 * 124 */ 125 struct flow_state { 126 u32 flags; 127 u32 resp_ib_psn; /* The IB PSN of the response for this flow */ 128 u32 generation; /* generation of flow */ 129 u32 spsn; /* starting PSN in TID space */ 130 u32 lpsn; /* last PSN in TID space */ 131 u32 r_next_psn; /* next PSN to be received (in TID space) */ 132 133 /* For tid rdma read */ 134 u32 ib_spsn; /* starting PSN in Verbs space */ 135 u32 ib_lpsn; /* last PSn in Verbs space */ 136 }; 137 138 struct tid_rdma_pageset { 139 dma_addr_t addr : 48; /* Only needed for the first page */ 140 u8 idx: 8; 141 u8 count : 7; 142 u8 mapped: 1; 143 }; 144 145 /** 146 * kern_tid_node - used for managing TID's in TID groups 147 * 148 * @grp_idx: rcd relative index to tid_group 149 * @map: grp->map captured prior to programming this TID group in HW 150 * @cnt: Only @cnt of available group entries are actually programmed 151 */ 152 struct kern_tid_node { 153 struct tid_group *grp; 154 u8 map; 155 u8 cnt; 156 }; 157 158 /* Overall info for a TID RDMA segment */ 159 struct tid_rdma_flow { 160 /* 161 * While a TID RDMA segment is being transferred, it uses a QP number 162 * from the "KDETH section of QP numbers" (which is different from the 163 * QP number that originated the request). Bits 11-15 of these QP 164 * numbers identify the "TID flow" for the segment. 165 */ 166 struct flow_state flow_state; 167 struct tid_rdma_request *req; 168 u32 tid_qpn; 169 u32 tid_offset; 170 u32 length; 171 u32 sent; 172 u8 tnode_cnt; 173 u8 tidcnt; 174 u8 tid_idx; 175 u8 idx; 176 u8 npagesets; 177 u8 npkts; 178 u8 pkt; 179 u8 resync_npkts; 180 struct kern_tid_node tnode[TID_RDMA_MAX_PAGES]; 181 struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES]; 182 u32 tid_entry[TID_RDMA_MAX_PAGES]; 183 }; 184 185 enum tid_rnr_nak_state { 186 TID_RNR_NAK_INIT = 0, 187 TID_RNR_NAK_SEND, 188 TID_RNR_NAK_SENT, 189 }; 190 191 bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data); 192 bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data); 193 bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data); 194 void tid_rdma_conn_error(struct rvt_qp *qp); 195 void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p); 196 197 int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit); 198 int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req, 199 struct rvt_sge_state *ss, bool *last); 200 int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req); 201 void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req); 202 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 203 204 /** 205 * trdma_clean_swqe - clean flows for swqe if large send queue 206 * @qp: the qp 207 * @wqe: the send wqe 208 */ 209 static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) 210 { 211 if (!wqe->priv) 212 return; 213 __trdma_clean_swqe(qp, wqe); 214 } 215 216 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp); 217 218 int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, 219 struct ib_qp_init_attr *init_attr); 220 void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); 221 222 void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp); 223 224 int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 225 void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 226 void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd); 227 228 struct cntr_entry; 229 u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry, 230 void *context, int vl, int mode, u64 data); 231 232 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, 233 struct ib_other_headers *ohdr, 234 u32 *bth1, u32 *bth2, u32 *len); 235 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 236 struct ib_other_headers *ohdr, u32 *bth1, 237 u32 *bth2, u32 *len); 238 void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet); 239 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 240 struct ib_other_headers *ohdr, u32 *bth0, 241 u32 *bth1, u32 *bth2, u32 *len, bool *last); 242 void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet); 243 bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, 244 struct hfi1_pportdata *ppd, 245 struct hfi1_packet *packet); 246 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 247 u32 *bth2); 248 void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp); 249 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); 250 251 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 252 static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp, 253 struct rvt_swqe *wqe) 254 { 255 if (wqe->priv && 256 wqe->wr.opcode == IB_WR_RDMA_READ && 257 wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE) 258 setup_tid_rdma_wqe(qp, wqe); 259 } 260 261 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 262 struct ib_other_headers *ohdr, 263 u32 *bth1, u32 *bth2, u32 *len); 264 265 void hfi1_compute_tid_rdma_flow_wt(void); 266 267 void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet); 268 269 u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 270 struct ib_other_headers *ohdr, u32 *bth1, 271 u32 bth2, u32 *len, 272 struct rvt_sge_state **ss); 273 274 void hfi1_del_tid_reap_timer(struct rvt_qp *qp); 275 276 void hfi1_rc_rcv_tid_rdma_write_resp(struct hfi1_packet *packet); 277 278 bool hfi1_build_tid_rdma_packet(struct rvt_swqe *wqe, 279 struct ib_other_headers *ohdr, 280 u32 *bth1, u32 *bth2, u32 *len); 281 282 #endif /* HFI1_TID_RDMA_H */ 283