1 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ 2 /* 3 * Copyright 2018-2022 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #ifndef _EFA_IO_H_ 7 #define _EFA_IO_H_ 8 9 #define EFA_IO_TX_DESC_NUM_BUFS 2 10 #define EFA_IO_TX_DESC_NUM_RDMA_BUFS 1 11 #define EFA_IO_TX_DESC_INLINE_MAX_SIZE 32 12 #define EFA_IO_TX_DESC_IMM_DATA_SIZE 4 13 14 enum efa_io_queue_type { 15 /* send queue (of a QP) */ 16 EFA_IO_SEND_QUEUE = 1, 17 /* recv queue (of a QP) */ 18 EFA_IO_RECV_QUEUE = 2, 19 }; 20 21 enum efa_io_send_op_type { 22 /* send message */ 23 EFA_IO_SEND = 0, 24 /* RDMA read */ 25 EFA_IO_RDMA_READ = 1, 26 }; 27 28 enum efa_io_comp_status { 29 /* Successful completion */ 30 EFA_IO_COMP_STATUS_OK = 0, 31 /* Flushed during QP destroy */ 32 EFA_IO_COMP_STATUS_FLUSHED = 1, 33 /* Internal QP error */ 34 EFA_IO_COMP_STATUS_LOCAL_ERROR_QP_INTERNAL_ERROR = 2, 35 /* Bad operation type */ 36 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_OP_TYPE = 3, 37 /* Bad AH */ 38 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_AH = 4, 39 /* LKEY not registered or does not match IOVA */ 40 EFA_IO_COMP_STATUS_LOCAL_ERROR_INVALID_LKEY = 5, 41 /* Message too long */ 42 EFA_IO_COMP_STATUS_LOCAL_ERROR_BAD_LENGTH = 6, 43 /* Destination ENI is down or does not run EFA */ 44 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_ADDRESS = 7, 45 /* Connection was reset by remote side */ 46 EFA_IO_COMP_STATUS_REMOTE_ERROR_ABORT = 8, 47 /* Bad dest QP number (QP does not exist or is in error state) */ 48 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_DEST_QPN = 9, 49 /* Destination resource not ready (no WQEs posted on RQ) */ 50 EFA_IO_COMP_STATUS_REMOTE_ERROR_RNR = 10, 51 /* Receiver SGL too short */ 52 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_LENGTH = 11, 53 /* Unexpected status returned by responder */ 54 EFA_IO_COMP_STATUS_REMOTE_ERROR_BAD_STATUS = 12, 55 /* Unresponsive remote - detected locally */ 56 EFA_IO_COMP_STATUS_LOCAL_ERROR_UNRESP_REMOTE = 13, 57 }; 58 59 struct efa_io_tx_meta_desc { 60 /* Verbs-generated Request ID */ 61 u16 req_id; 62 63 /* 64 * control flags 65 * 3:0 : op_type - operation type: send/rdma/fast mem 66 * ops/etc 67 * 4 : has_imm - immediate_data field carries valid 68 * data. 69 * 5 : inline_msg - inline mode - inline message data 70 * follows this descriptor (no buffer descriptors). 71 * Note that it is different from immediate data 72 * 6 : meta_extension - Extended metadata. MBZ 73 * 7 : meta_desc - Indicates metadata descriptor. 74 * Must be set. 75 */ 76 u8 ctrl1; 77 78 /* 79 * control flags 80 * 0 : phase 81 * 1 : reserved25 - MBZ 82 * 2 : first - Indicates first descriptor in 83 * transaction. Must be set. 84 * 3 : last - Indicates last descriptor in 85 * transaction. Must be set. 86 * 4 : comp_req - Indicates whether completion should 87 * be posted, after packet is transmitted. Valid only 88 * for the first descriptor 89 * 7:5 : reserved29 - MBZ 90 */ 91 u8 ctrl2; 92 93 u16 dest_qp_num; 94 95 /* 96 * If inline_msg bit is set, length of inline message in bytes, 97 * otherwise length of SGL (number of buffers). 98 */ 99 u16 length; 100 101 /* 102 * immediate data: if has_imm is set, then this field is included 103 * within Tx message and reported in remote Rx completion. 104 */ 105 u32 immediate_data; 106 107 u16 ah; 108 109 u16 reserved; 110 111 /* Queue key */ 112 u32 qkey; 113 114 u8 reserved2[12]; 115 }; 116 117 /* 118 * Tx queue buffer descriptor, for any transport type. Preceded by metadata 119 * descriptor. 120 */ 121 struct efa_io_tx_buf_desc { 122 /* length in bytes */ 123 u32 length; 124 125 /* 126 * 23:0 : lkey - local memory translation key 127 * 31:24 : reserved - MBZ 128 */ 129 u32 lkey; 130 131 /* Buffer address bits[31:0] */ 132 u32 buf_addr_lo; 133 134 /* Buffer address bits[63:32] */ 135 u32 buf_addr_hi; 136 }; 137 138 struct efa_io_remote_mem_addr { 139 /* length in bytes */ 140 u32 length; 141 142 /* remote memory translation key */ 143 u32 rkey; 144 145 /* Buffer address bits[31:0] */ 146 u32 buf_addr_lo; 147 148 /* Buffer address bits[63:32] */ 149 u32 buf_addr_hi; 150 }; 151 152 struct efa_io_rdma_req { 153 /* Remote memory address */ 154 struct efa_io_remote_mem_addr remote_mem; 155 156 /* Local memory address */ 157 struct efa_io_tx_buf_desc local_mem[1]; 158 }; 159 160 /* 161 * Tx WQE, composed of tx meta descriptors followed by either tx buffer 162 * descriptors or inline data 163 */ 164 struct efa_io_tx_wqe { 165 /* TX meta */ 166 struct efa_io_tx_meta_desc meta; 167 168 union { 169 /* Send buffer descriptors */ 170 struct efa_io_tx_buf_desc sgl[2]; 171 172 u8 inline_data[32]; 173 174 /* RDMA local and remote memory addresses */ 175 struct efa_io_rdma_req rdma_req; 176 } data; 177 }; 178 179 /* 180 * Rx buffer descriptor; RX WQE is composed of one or more RX buffer 181 * descriptors. 182 */ 183 struct efa_io_rx_desc { 184 /* Buffer address bits[31:0] */ 185 u32 buf_addr_lo; 186 187 /* Buffer Pointer[63:32] */ 188 u32 buf_addr_hi; 189 190 /* Verbs-generated request id. */ 191 u16 req_id; 192 193 /* Length in bytes. */ 194 u16 length; 195 196 /* 197 * LKey and control flags 198 * 23:0 : lkey 199 * 29:24 : reserved - MBZ 200 * 30 : first - Indicates first descriptor in WQE 201 * 31 : last - Indicates last descriptor in WQE 202 */ 203 u32 lkey_ctrl; 204 }; 205 206 /* Common IO completion descriptor */ 207 struct efa_io_cdesc_common { 208 /* 209 * verbs-generated request ID, as provided in the completed tx or rx 210 * descriptor. 211 */ 212 u16 req_id; 213 214 u8 status; 215 216 /* 217 * flags 218 * 0 : phase - Phase bit 219 * 2:1 : q_type - enum efa_io_queue_type: send/recv 220 * 3 : has_imm - indicates that immediate data is 221 * present - for RX completions only 222 * 7:4 : reserved28 - MBZ 223 */ 224 u8 flags; 225 226 /* local QP number */ 227 u16 qp_num; 228 229 /* Transferred length */ 230 u16 length; 231 }; 232 233 /* Tx completion descriptor */ 234 struct efa_io_tx_cdesc { 235 /* Common completion info */ 236 struct efa_io_cdesc_common common; 237 }; 238 239 /* Rx Completion Descriptor */ 240 struct efa_io_rx_cdesc { 241 /* Common completion info */ 242 struct efa_io_cdesc_common common; 243 244 /* Remote Address Handle FW index, 0xFFFF indicates invalid ah */ 245 u16 ah; 246 247 u16 src_qp_num; 248 249 /* Immediate data */ 250 u32 imm; 251 }; 252 253 /* Extended Rx Completion Descriptor */ 254 struct efa_io_rx_cdesc_ex { 255 /* Base RX completion info */ 256 struct efa_io_rx_cdesc rx_cdesc_base; 257 258 /* 259 * Valid only in case of unknown AH (0xFFFF) and CQ set_src_addr is 260 * enabled. 261 */ 262 u8 src_addr[16]; 263 }; 264 265 /* tx_meta_desc */ 266 #define EFA_IO_TX_META_DESC_OP_TYPE_MASK GENMASK(3, 0) 267 #define EFA_IO_TX_META_DESC_HAS_IMM_MASK BIT(4) 268 #define EFA_IO_TX_META_DESC_INLINE_MSG_MASK BIT(5) 269 #define EFA_IO_TX_META_DESC_META_EXTENSION_MASK BIT(6) 270 #define EFA_IO_TX_META_DESC_META_DESC_MASK BIT(7) 271 #define EFA_IO_TX_META_DESC_PHASE_MASK BIT(0) 272 #define EFA_IO_TX_META_DESC_FIRST_MASK BIT(2) 273 #define EFA_IO_TX_META_DESC_LAST_MASK BIT(3) 274 #define EFA_IO_TX_META_DESC_COMP_REQ_MASK BIT(4) 275 276 /* tx_buf_desc */ 277 #define EFA_IO_TX_BUF_DESC_LKEY_MASK GENMASK(23, 0) 278 279 /* rx_desc */ 280 #define EFA_IO_RX_DESC_LKEY_MASK GENMASK(23, 0) 281 #define EFA_IO_RX_DESC_FIRST_MASK BIT(30) 282 #define EFA_IO_RX_DESC_LAST_MASK BIT(31) 283 284 /* cdesc_common */ 285 #define EFA_IO_CDESC_COMMON_PHASE_MASK BIT(0) 286 #define EFA_IO_CDESC_COMMON_Q_TYPE_MASK GENMASK(2, 1) 287 #define EFA_IO_CDESC_COMMON_HAS_IMM_MASK BIT(3) 288 289 #endif /* _EFA_IO_H_ */ 290