1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 */ 6 #ifndef HFI1_TID_RDMA_H 7 #define HFI1_TID_RDMA_H 8 9 #include <linux/circ_buf.h> 10 #include "common.h" 11 12 /* Add a convenience helper */ 13 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 14 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 15 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 16 17 #define TID_RDMA_MIN_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 18 #define TID_RDMA_MAX_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 19 #define TID_RDMA_MAX_PAGES (BIT(18) >> PAGE_SHIFT) 20 21 /* 22 * Bit definitions for priv->s_flags. 23 * These bit flags overload the bit flags defined for the QP's s_flags. 24 * Due to the fact that these bit fields are used only for the QP priv 25 * s_flags, there are no collisions. 26 * 27 * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock 28 */ 29 /* BIT(4) reserved for RVT_S_ACK_PENDING. */ 30 #define HFI1_S_TID_WAIT_INTERLCK BIT(5) 31 #define HFI1_R_TID_SW_PSN BIT(19) 32 33 /* 34 * Unlike regular IB RDMA VERBS, which do not require an entry 35 * in the s_ack_queue, TID RDMA WRITE requests do because they 36 * generate responses. 37 * Therefore, the s_ack_queue needs to be extended by a certain 38 * amount. The key point is that the queue needs to be extended 39 * without letting the "user" know so they user doesn't end up 40 * using these extra entries. 41 */ 42 #define HFI1_TID_RDMA_WRITE_CNT 8 43 44 struct tid_rdma_params { 45 struct rcu_head rcu_head; 46 u32 qp; 47 u32 max_len; 48 u16 jkey; 49 u8 max_read; 50 u8 max_write; 51 u8 timeout; 52 u8 urg; 53 u8 version; 54 }; 55 56 struct tid_rdma_qp_params { 57 struct work_struct trigger_work; 58 struct tid_rdma_params local; 59 struct tid_rdma_params __rcu *remote; 60 }; 61 62 /* Track state for each hardware flow */ 63 struct tid_flow_state { 64 u32 generation; 65 u32 psn; 66 u32 r_next_psn; /* next PSN to be received (in TID space) */ 67 u8 index; 68 u8 last_index; 69 u8 flags; 70 }; 71 72 enum tid_rdma_req_state { 73 TID_REQUEST_INACTIVE = 0, 74 TID_REQUEST_INIT, 75 TID_REQUEST_INIT_RESEND, 76 TID_REQUEST_ACTIVE, 77 TID_REQUEST_RESEND, 78 TID_REQUEST_RESEND_ACTIVE, 79 TID_REQUEST_QUEUED, 80 TID_REQUEST_SYNC, 81 TID_REQUEST_RNR_NAK, 82 TID_REQUEST_COMPLETE, 83 }; 84 85 struct tid_rdma_request { 86 struct rvt_qp *qp; 87 struct hfi1_ctxtdata *rcd; 88 union { 89 struct rvt_swqe *swqe; 90 struct rvt_ack_entry *ack; 91 } e; 92 93 struct tid_rdma_flow *flows; /* array of tid flows */ 94 struct rvt_sge_state ss; /* SGE state for TID RDMA requests */ 95 u16 n_flows; /* size of the flow buffer window */ 96 u16 setup_head; /* flow index we are setting up */ 97 u16 clear_tail; /* flow index we are clearing */ 98 u16 flow_idx; /* flow index most recently set up */ 99 u16 acked_tail; 100 101 u32 seg_len; 102 u32 total_len; 103 u32 r_flow_psn; /* IB PSN of next segment start */ 104 u32 s_next_psn; /* IB PSN of next segment start for read */ 105 106 u32 total_segs; /* segments required to complete a request */ 107 u32 cur_seg; /* index of current segment */ 108 u32 comp_seg; /* index of last completed segment */ 109 u32 ack_seg; /* index of last ack'ed segment */ 110 u32 alloc_seg; /* index of next segment to be allocated */ 111 u32 isge; /* index of "current" sge */ 112 u32 ack_pending; /* num acks pending for this request */ 113 114 enum tid_rdma_req_state state; 115 }; 116 117 /* 118 * When header suppression is used, PSNs associated with a "flow" are 119 * relevant (and not the PSNs maintained by verbs). Track per-flow 120 * PSNs here for a TID RDMA segment. 121 * 122 */ 123 struct flow_state { 124 u32 flags; 125 u32 resp_ib_psn; /* The IB PSN of the response for this flow */ 126 u32 generation; /* generation of flow */ 127 u32 spsn; /* starting PSN in TID space */ 128 u32 lpsn; /* last PSN in TID space */ 129 u32 r_next_psn; /* next PSN to be received (in TID space) */ 130 131 /* For tid rdma read */ 132 u32 ib_spsn; /* starting PSN in Verbs space */ 133 u32 ib_lpsn; /* last PSn in Verbs space */ 134 }; 135 136 struct tid_rdma_pageset { 137 dma_addr_t addr : 48; /* Only needed for the first page */ 138 u8 idx: 8; 139 u8 count : 7; 140 u8 mapped: 1; 141 }; 142 143 /** 144 * kern_tid_node - used for managing TID's in TID groups 145 * 146 * @grp_idx: rcd relative index to tid_group 147 * @map: grp->map captured prior to programming this TID group in HW 148 * @cnt: Only @cnt of available group entries are actually programmed 149 */ 150 struct kern_tid_node { 151 struct tid_group *grp; 152 u8 map; 153 u8 cnt; 154 }; 155 156 /* Overall info for a TID RDMA segment */ 157 struct tid_rdma_flow { 158 /* 159 * While a TID RDMA segment is being transferred, it uses a QP number 160 * from the "KDETH section of QP numbers" (which is different from the 161 * QP number that originated the request). Bits 11-15 of these QP 162 * numbers identify the "TID flow" for the segment. 163 */ 164 struct flow_state flow_state; 165 struct tid_rdma_request *req; 166 u32 tid_qpn; 167 u32 tid_offset; 168 u32 length; 169 u32 sent; 170 u8 tnode_cnt; 171 u8 tidcnt; 172 u8 tid_idx; 173 u8 idx; 174 u8 npagesets; 175 u8 npkts; 176 u8 pkt; 177 struct kern_tid_node tnode[TID_RDMA_MAX_PAGES]; 178 struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES]; 179 u32 tid_entry[TID_RDMA_MAX_PAGES]; 180 }; 181 182 enum tid_rnr_nak_state { 183 TID_RNR_NAK_INIT = 0, 184 TID_RNR_NAK_SEND, 185 TID_RNR_NAK_SENT, 186 }; 187 188 bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data); 189 bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data); 190 bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data); 191 void tid_rdma_conn_error(struct rvt_qp *qp); 192 void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p); 193 194 int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit); 195 int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req, 196 struct rvt_sge_state *ss, bool *last); 197 int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req); 198 void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req); 199 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 200 201 /** 202 * trdma_clean_swqe - clean flows for swqe if large send queue 203 * @qp: the qp 204 * @wqe: the send wqe 205 */ 206 static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) 207 { 208 if (!wqe->priv) 209 return; 210 __trdma_clean_swqe(qp, wqe); 211 } 212 213 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp); 214 215 int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, 216 struct ib_qp_init_attr *init_attr); 217 void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); 218 219 void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp); 220 221 int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 222 void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 223 void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd); 224 225 struct cntr_entry; 226 u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry, 227 void *context, int vl, int mode, u64 data); 228 229 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, 230 struct ib_other_headers *ohdr, 231 u32 *bth1, u32 *bth2, u32 *len); 232 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 233 struct ib_other_headers *ohdr, u32 *bth1, 234 u32 *bth2, u32 *len); 235 void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet); 236 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 237 struct ib_other_headers *ohdr, u32 *bth0, 238 u32 *bth1, u32 *bth2, u32 *len, bool *last); 239 void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet); 240 bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, 241 struct hfi1_pportdata *ppd, 242 struct hfi1_packet *packet); 243 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 244 u32 *bth2); 245 void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp); 246 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); 247 248 void setup_tid_rdma_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 249 static inline void hfi1_setup_tid_rdma_wqe(struct rvt_qp *qp, 250 struct rvt_swqe *wqe) 251 { 252 if (wqe->priv && 253 wqe->wr.opcode == IB_WR_RDMA_READ && 254 wqe->length >= TID_RDMA_MIN_SEGMENT_SIZE) 255 setup_tid_rdma_wqe(qp, wqe); 256 } 257 258 u32 hfi1_build_tid_rdma_write_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 259 struct ib_other_headers *ohdr, 260 u32 *bth1, u32 *bth2, u32 *len); 261 262 void hfi1_compute_tid_rdma_flow_wt(void); 263 264 void hfi1_rc_rcv_tid_rdma_write_req(struct hfi1_packet *packet); 265 266 u32 hfi1_build_tid_rdma_write_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 267 struct ib_other_headers *ohdr, u32 *bth1, 268 u32 bth2, u32 *len, 269 struct rvt_sge_state **ss); 270 271 #endif /* HFI1_TID_RDMA_H */ 272