1 /* SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) */ 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 */ 6 #ifndef HFI1_TID_RDMA_H 7 #define HFI1_TID_RDMA_H 8 9 #include <linux/circ_buf.h> 10 #include "common.h" 11 12 /* Add a convenience helper */ 13 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1)) 14 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size) 15 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size) 16 17 #define TID_RDMA_MAX_SEGMENT_SIZE BIT(18) /* 256 KiB (for now) */ 18 #define TID_RDMA_MAX_PAGES (BIT(18) >> PAGE_SHIFT) 19 20 /* 21 * Bit definitions for priv->s_flags. 22 * These bit flags overload the bit flags defined for the QP's s_flags. 23 * Due to the fact that these bit fields are used only for the QP priv 24 * s_flags, there are no collisions. 25 * 26 * HFI1_S_TID_WAIT_INTERLCK - QP is waiting for requester interlock 27 */ 28 #define HFI1_S_TID_WAIT_INTERLCK BIT(5) 29 30 struct tid_rdma_params { 31 struct rcu_head rcu_head; 32 u32 qp; 33 u32 max_len; 34 u16 jkey; 35 u8 max_read; 36 u8 max_write; 37 u8 timeout; 38 u8 urg; 39 u8 version; 40 }; 41 42 struct tid_rdma_qp_params { 43 struct work_struct trigger_work; 44 struct tid_rdma_params local; 45 struct tid_rdma_params __rcu *remote; 46 }; 47 48 /* Track state for each hardware flow */ 49 struct tid_flow_state { 50 u32 generation; 51 u32 psn; 52 u32 r_next_psn; /* next PSN to be received (in TID space) */ 53 u8 index; 54 u8 last_index; 55 u8 flags; 56 }; 57 58 enum tid_rdma_req_state { 59 TID_REQUEST_INACTIVE = 0, 60 TID_REQUEST_INIT, 61 TID_REQUEST_INIT_RESEND, 62 TID_REQUEST_ACTIVE, 63 TID_REQUEST_RESEND, 64 TID_REQUEST_RESEND_ACTIVE, 65 TID_REQUEST_QUEUED, 66 TID_REQUEST_SYNC, 67 TID_REQUEST_RNR_NAK, 68 TID_REQUEST_COMPLETE, 69 }; 70 71 struct tid_rdma_request { 72 struct rvt_qp *qp; 73 struct hfi1_ctxtdata *rcd; 74 union { 75 struct rvt_swqe *swqe; 76 struct rvt_ack_entry *ack; 77 } e; 78 79 struct tid_rdma_flow *flows; /* array of tid flows */ 80 u16 n_flows; /* size of the flow buffer window */ 81 u16 setup_head; /* flow index we are setting up */ 82 u16 clear_tail; /* flow index we are clearing */ 83 u16 flow_idx; /* flow index most recently set up */ 84 85 u32 seg_len; 86 u32 total_len; 87 u32 r_flow_psn; /* IB PSN of next segment start */ 88 u32 s_next_psn; /* IB PSN of next segment start for read */ 89 90 u32 total_segs; /* segments required to complete a request */ 91 u32 cur_seg; /* index of current segment */ 92 u32 comp_seg; /* index of last completed segment */ 93 u32 ack_seg; /* index of last ack'ed segment */ 94 u32 isge; /* index of "current" sge */ 95 u32 ack_pending; /* num acks pending for this request */ 96 97 enum tid_rdma_req_state state; 98 }; 99 100 /* 101 * When header suppression is used, PSNs associated with a "flow" are 102 * relevant (and not the PSNs maintained by verbs). Track per-flow 103 * PSNs here for a TID RDMA segment. 104 * 105 */ 106 struct flow_state { 107 u32 flags; 108 u32 resp_ib_psn; /* The IB PSN of the response for this flow */ 109 u32 generation; /* generation of flow */ 110 u32 spsn; /* starting PSN in TID space */ 111 u32 lpsn; /* last PSN in TID space */ 112 u32 r_next_psn; /* next PSN to be received (in TID space) */ 113 114 /* For tid rdma read */ 115 u32 ib_spsn; /* starting PSN in Verbs space */ 116 u32 ib_lpsn; /* last PSn in Verbs space */ 117 }; 118 119 struct tid_rdma_pageset { 120 dma_addr_t addr : 48; /* Only needed for the first page */ 121 u8 idx: 8; 122 u8 count : 7; 123 u8 mapped: 1; 124 }; 125 126 /** 127 * kern_tid_node - used for managing TID's in TID groups 128 * 129 * @grp_idx: rcd relative index to tid_group 130 * @map: grp->map captured prior to programming this TID group in HW 131 * @cnt: Only @cnt of available group entries are actually programmed 132 */ 133 struct kern_tid_node { 134 struct tid_group *grp; 135 u8 map; 136 u8 cnt; 137 }; 138 139 /* Overall info for a TID RDMA segment */ 140 struct tid_rdma_flow { 141 /* 142 * While a TID RDMA segment is being transferred, it uses a QP number 143 * from the "KDETH section of QP numbers" (which is different from the 144 * QP number that originated the request). Bits 11-15 of these QP 145 * numbers identify the "TID flow" for the segment. 146 */ 147 struct flow_state flow_state; 148 struct tid_rdma_request *req; 149 u32 tid_qpn; 150 u32 tid_offset; 151 u32 length; 152 u32 sent; 153 u8 tnode_cnt; 154 u8 tidcnt; 155 u8 tid_idx; 156 u8 idx; 157 u8 npagesets; 158 u8 npkts; 159 u8 pkt; 160 struct kern_tid_node tnode[TID_RDMA_MAX_PAGES]; 161 struct tid_rdma_pageset pagesets[TID_RDMA_MAX_PAGES]; 162 u32 tid_entry[TID_RDMA_MAX_PAGES]; 163 }; 164 165 bool tid_rdma_conn_req(struct rvt_qp *qp, u64 *data); 166 bool tid_rdma_conn_reply(struct rvt_qp *qp, u64 data); 167 bool tid_rdma_conn_resp(struct rvt_qp *qp, u64 *data); 168 void tid_rdma_conn_error(struct rvt_qp *qp); 169 void tid_rdma_opfn_init(struct rvt_qp *qp, struct tid_rdma_params *p); 170 171 int hfi1_kern_exp_rcv_init(struct hfi1_ctxtdata *rcd, int reinit); 172 int hfi1_kern_exp_rcv_setup(struct tid_rdma_request *req, 173 struct rvt_sge_state *ss, bool *last); 174 int hfi1_kern_exp_rcv_clear(struct tid_rdma_request *req); 175 void hfi1_kern_exp_rcv_clear_all(struct tid_rdma_request *req); 176 void __trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe); 177 178 /** 179 * trdma_clean_swqe - clean flows for swqe if large send queue 180 * @qp: the qp 181 * @wqe: the send wqe 182 */ 183 static inline void trdma_clean_swqe(struct rvt_qp *qp, struct rvt_swqe *wqe) 184 { 185 if (!wqe->priv) 186 return; 187 __trdma_clean_swqe(qp, wqe); 188 } 189 190 void hfi1_kern_read_tid_flow_free(struct rvt_qp *qp); 191 192 int hfi1_qp_priv_init(struct rvt_dev_info *rdi, struct rvt_qp *qp, 193 struct ib_qp_init_attr *init_attr); 194 void hfi1_qp_priv_tid_free(struct rvt_dev_info *rdi, struct rvt_qp *qp); 195 196 void hfi1_tid_rdma_flush_wait(struct rvt_qp *qp); 197 198 int hfi1_kern_setup_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 199 void hfi1_kern_clear_hw_flow(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp); 200 void hfi1_kern_init_ctxt_generations(struct hfi1_ctxtdata *rcd); 201 202 struct cntr_entry; 203 u64 hfi1_access_sw_tid_wait(const struct cntr_entry *entry, 204 void *context, int vl, int mode, u64 data); 205 206 u32 hfi1_build_tid_rdma_read_packet(struct rvt_swqe *wqe, 207 struct ib_other_headers *ohdr, 208 u32 *bth1, u32 *bth2, u32 *len); 209 u32 hfi1_build_tid_rdma_read_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 210 struct ib_other_headers *ohdr, u32 *bth1, 211 u32 *bth2, u32 *len); 212 void hfi1_rc_rcv_tid_rdma_read_req(struct hfi1_packet *packet); 213 u32 hfi1_build_tid_rdma_read_resp(struct rvt_qp *qp, struct rvt_ack_entry *e, 214 struct ib_other_headers *ohdr, u32 *bth0, 215 u32 *bth1, u32 *bth2, u32 *len, bool *last); 216 void hfi1_rc_rcv_tid_rdma_read_resp(struct hfi1_packet *packet); 217 bool hfi1_handle_kdeth_eflags(struct hfi1_ctxtdata *rcd, 218 struct hfi1_pportdata *ppd, 219 struct hfi1_packet *packet); 220 void hfi1_tid_rdma_restart_req(struct rvt_qp *qp, struct rvt_swqe *wqe, 221 u32 *bth2); 222 void hfi1_qp_kern_exp_rcv_clear_all(struct rvt_qp *qp); 223 bool hfi1_tid_rdma_wqe_interlock(struct rvt_qp *qp, struct rvt_swqe *wqe); 224 225 #endif /* HFI1_TID_RDMA_H */ 226