1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
2 /*
3  * Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #ifndef _EFA_IO_H_
7 #define _EFA_IO_H_
8 
9 #define EFA_IO_TX_DESC_NUM_BUFS              2
10 #define EFA_IO_TX_DESC_NUM_RDMA_BUFS         1
11 #define EFA_IO_TX_DESC_INLINE_MAX_SIZE       32
12 #define EFA_IO_TX_DESC_IMM_DATA_SIZE         4
13 
14 enum efa_io_queue_type {
15 	/* send queue (of a QP) */
16 	EFA_IO_SEND_QUEUE                           = 1,
17 	/* recv queue (of a QP) */
18 	EFA_IO_RECV_QUEUE                           = 2,
19 };
20 
21 enum efa_io_send_op_type {
22 	/* send message */
23 	EFA_IO_SEND                                 = 0,
24 	/* RDMA read */
25 	EFA_IO_RDMA_READ                            = 1,
26 	/* RDMA write */
27 	EFA_IO_RDMA_WRITE                           = 2,
28 };
29 
30 enum efa_io_comp_status {
31 	/* Successful completion */
32 	EFA_IO_COMP_STATUS_OK                       = 0,
33 	/* Flushed during QP destroy */
34 	EFA_IO_COMP_STATUS_FLUSHED                  = 1,
35 	/* Internal QP error */
36 	EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2,
37 	/* Bad operation type */
38 	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_OP_TYPE = 3,
39 	/* Bad AH */
40 	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH   = 4,
41 	/* LKEY not registered or does not match IOVA */
42 	EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5,
43 	/* Message too long */
44 	EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH   = 6,
45 	/* Destination ENI is down or does not run EFA */
46 	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7,
47 	/* Connection was reset by remote side */
48 	EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT       = 8,
49 	/* Bad dest QP number (QP does not exist or is in error state) */
50 	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9,
51 	/* Destination resource not ready (no WQEs posted on RQ) */
52 	EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR         = 10,
53 	/* Receiver SGL too short */
54 	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH  = 11,
55 	/* Unexpected status returned by responder */
56 	EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS  = 12,
57 	/* Unresponsive remote - detected locally */
58 	EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13,
59 };
60 
61 struct efa_io_tx_meta_desc {
62 	/* Verbs-generated Request ID */
63 	u16 req_id;
64 
65 	/*
66 	 * control flags
67 	 * 3:0 : op_type - enum efa_io_send_op_type
68 	 * 4 : has_imm - immediate_data field carries valid
69 	 *    data.
70 	 * 5 : inline_msg - inline mode - inline message data
71 	 *    follows this descriptor (no buffer descriptors).
72 	 *    Note that it is different from immediate data
73 	 * 6 : meta_extension - Extended metadata. MBZ
74 	 * 7 : meta_desc - Indicates metadata descriptor.
75 	 *    Must be set.
76 	 */
77 	u8 ctrl1;
78 
79 	/*
80 	 * control flags
81 	 * 0 : phase
82 	 * 1 : reserved25 - MBZ
83 	 * 2 : first - Indicates first descriptor in
84 	 *    transaction. Must be set.
85 	 * 3 : last - Indicates last descriptor in
86 	 *    transaction. Must be set.
87 	 * 4 : comp_req - Indicates whether completion should
88 	 *    be posted, after packet is transmitted. Valid only
89 	 *    for the first descriptor
90 	 * 7:5 : reserved29 - MBZ
91 	 */
92 	u8 ctrl2;
93 
94 	u16 dest_qp_num;
95 
96 	/*
97 	 * If inline_msg bit is set, length of inline message in bytes,
98 	 *    otherwise length of SGL (number of buffers).
99 	 */
100 	u16 length;
101 
102 	/*
103 	 * immediate data: if has_imm is set, then this field is included
104 	 *    within Tx message and reported in remote Rx completion.
105 	 */
106 	u32 immediate_data;
107 
108 	u16 ah;
109 
110 	u16 reserved;
111 
112 	/* Queue key */
113 	u32 qkey;
114 
115 	u8 reserved2[12];
116 };
117 
118 /*
119  * Tx queue buffer descriptor, for any transport type. Preceded by metadata
120  * descriptor.
121  */
122 struct efa_io_tx_buf_desc {
123 	/* length in bytes */
124 	u32 length;
125 
126 	/*
127 	 * 23:0 : lkey - local memory translation key
128 	 * 31:24 : reserved - MBZ
129 	 */
130 	u32 lkey;
131 
132 	/* Buffer address bits[31:0] */
133 	u32 buf_addr_lo;
134 
135 	/* Buffer address bits[63:32] */
136 	u32 buf_addr_hi;
137 };
138 
139 struct efa_io_remote_mem_addr {
140 	/* length in bytes */
141 	u32 length;
142 
143 	/* remote memory translation key */
144 	u32 rkey;
145 
146 	/* Buffer address bits[31:0] */
147 	u32 buf_addr_lo;
148 
149 	/* Buffer address bits[63:32] */
150 	u32 buf_addr_hi;
151 };
152 
153 struct efa_io_rdma_req {
154 	/* Remote memory address */
155 	struct efa_io_remote_mem_addr remote_mem;
156 
157 	/* Local memory address */
158 	struct efa_io_tx_buf_desc local_mem[1];
159 };
160 
161 /*
162  * Tx WQE, composed of tx meta descriptors followed by either tx buffer
163  * descriptors or inline data
164  */
165 struct efa_io_tx_wqe {
166 	/* TX meta */
167 	struct efa_io_tx_meta_desc meta;
168 
169 	union {
170 		/* Send buffer descriptors */
171 		struct efa_io_tx_buf_desc sgl[2];
172 
173 		u8 inline_data[32];
174 
175 		/* RDMA local and remote memory addresses */
176 		struct efa_io_rdma_req rdma_req;
177 	} data;
178 };
179 
180 /*
181  * Rx buffer descriptor; RX WQE is composed of one or more RX buffer
182  * descriptors.
183  */
184 struct efa_io_rx_desc {
185 	/* Buffer address bits[31:0] */
186 	u32 buf_addr_lo;
187 
188 	/* Buffer Pointer[63:32] */
189 	u32 buf_addr_hi;
190 
191 	/* Verbs-generated request id. */
192 	u16 req_id;
193 
194 	/* Length in bytes. */
195 	u16 length;
196 
197 	/*
198 	 * LKey and control flags
199 	 * 23:0 : lkey
200 	 * 29:24 : reserved - MBZ
201 	 * 30 : first - Indicates first descriptor in WQE
202 	 * 31 : last - Indicates last descriptor in WQE
203 	 */
204 	u32 lkey_ctrl;
205 };
206 
207 /* Common IO completion descriptor */
208 struct efa_io_cdesc_common {
209 	/*
210 	 * verbs-generated request ID, as provided in the completed tx or rx
211 	 *    descriptor.
212 	 */
213 	u16 req_id;
214 
215 	u8 status;
216 
217 	/*
218 	 * flags
219 	 * 0 : phase - Phase bit
220 	 * 2:1 : q_type - enum efa_io_queue_type: send/recv
221 	 * 3 : has_imm - indicates that immediate data is
222 	 *    present - for RX completions only
223 	 * 6:4 : op_type - enum efa_io_send_op_type
224 	 * 7 : reserved31 - MBZ
225 	 */
226 	u8 flags;
227 
228 	/* local QP number */
229 	u16 qp_num;
230 };
231 
232 /* Tx completion descriptor */
233 struct efa_io_tx_cdesc {
234 	/* Common completion info */
235 	struct efa_io_cdesc_common common;
236 
237 	/* MBZ */
238 	u16 reserved16;
239 };
240 
241 /* Rx Completion Descriptor */
242 struct efa_io_rx_cdesc {
243 	/* Common completion info */
244 	struct efa_io_cdesc_common common;
245 
246 	/* Transferred length bits[15:0] */
247 	u16 length;
248 
249 	/* Remote Address Handle FW index, 0xFFFF indicates invalid ah */
250 	u16 ah;
251 
252 	u16 src_qp_num;
253 
254 	/* Immediate data */
255 	u32 imm;
256 };
257 
258 /* Rx Completion Descriptor RDMA write info */
259 struct efa_io_rx_cdesc_rdma_write {
260 	/* Transferred length bits[31:16] */
261 	u16 length_hi;
262 };
263 
264 /* Extended Rx Completion Descriptor */
265 struct efa_io_rx_cdesc_ex {
266 	/* Base RX completion info */
267 	struct efa_io_rx_cdesc base;
268 
269 	union {
270 		struct efa_io_rx_cdesc_rdma_write rdma_write;
271 
272 		/*
273 		 * Valid only in case of unknown AH (0xFFFF) and CQ
274 		 * set_src_addr is enabled.
275 		 */
276 		u8 src_addr[16];
277 	} u;
278 };
279 
280 /* tx_meta_desc */
281 #define EFA_IO_TX_META_DESC_OP_TYPE_MASK                    GENMASK(3, 0)
282 #define EFA_IO_TX_META_DESC_HAS_IMM_MASK                    BIT(4)
283 #define EFA_IO_TX_META_DESC_INLINE_MSG_MASK                 BIT(5)
284 #define EFA_IO_TX_META_DESC_META_EXTENSION_MASK             BIT(6)
285 #define EFA_IO_TX_META_DESC_META_DESC_MASK                  BIT(7)
286 #define EFA_IO_TX_META_DESC_PHASE_MASK                      BIT(0)
287 #define EFA_IO_TX_META_DESC_FIRST_MASK                      BIT(2)
288 #define EFA_IO_TX_META_DESC_LAST_MASK                       BIT(3)
289 #define EFA_IO_TX_META_DESC_COMP_REQ_MASK                   BIT(4)
290 
291 /* tx_buf_desc */
292 #define EFA_IO_TX_BUF_DESC_LKEY_MASK                        GENMASK(23, 0)
293 
294 /* rx_desc */
295 #define EFA_IO_RX_DESC_LKEY_MASK                            GENMASK(23, 0)
296 #define EFA_IO_RX_DESC_FIRST_MASK                           BIT(30)
297 #define EFA_IO_RX_DESC_LAST_MASK                            BIT(31)
298 
299 /* cdesc_common */
300 #define EFA_IO_CDESC_COMMON_PHASE_MASK                      BIT(0)
301 #define EFA_IO_CDESC_COMMON_Q_TYPE_MASK                     GENMASK(2, 1)
302 #define EFA_IO_CDESC_COMMON_HAS_IMM_MASK                    BIT(3)
303 #define EFA_IO_CDESC_COMMON_OP_TYPE_MASK                    GENMASK(6, 4)
304 
305 #endif /* _EFA_IO_H_ */
306