1 #include <linux/socket.h> 2 #include <linux/in.h> 3 #include <linux/in6.h> 4 #include <rdma/ib_verbs.h> 5 #include <rdma/rdma_cm.h> 6 #include <scsi/iser.h> 7 8 9 #define DRV_NAME "isert" 10 #define PFX DRV_NAME ": " 11 12 #define isert_dbg(fmt, arg...) \ 13 do { \ 14 if (unlikely(isert_debug_level > 2)) \ 15 printk(KERN_DEBUG PFX "%s: " fmt,\ 16 __func__ , ## arg); \ 17 } while (0) 18 19 #define isert_warn(fmt, arg...) \ 20 do { \ 21 if (unlikely(isert_debug_level > 0)) \ 22 pr_warn(PFX "%s: " fmt, \ 23 __func__ , ## arg); \ 24 } while (0) 25 26 #define isert_info(fmt, arg...) \ 27 do { \ 28 if (unlikely(isert_debug_level > 1)) \ 29 pr_info(PFX "%s: " fmt, \ 30 __func__ , ## arg); \ 31 } while (0) 32 33 #define isert_err(fmt, arg...) \ 34 pr_err(PFX "%s: " fmt, __func__ , ## arg) 35 36 /* Constant PDU lengths calculations */ 37 #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + \ 38 sizeof(struct iscsi_hdr)) 39 #define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) 40 41 /* QP settings */ 42 /* Maximal bounds on received asynchronous PDUs */ 43 #define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */ 44 45 #define ISERT_MAX_RX_MISC_PDUS 6 /* 46 * NOOP_OUT(2), TEXT(1), 47 * SCSI_TMFUNC(2), LOGOUT(1) 48 */ 49 50 #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */ 51 52 #define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) 53 54 #define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) 55 56 #define ISERT_INFLIGHT_DATAOUTS 8 57 58 #define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX * \ 59 (1 + ISERT_INFLIGHT_DATAOUTS) + \ 60 ISERT_MAX_TX_MISC_PDUS + \ 61 ISERT_MAX_RX_MISC_PDUS) 62 63 #define ISER_RX_PAD_SIZE (ISCSI_DEF_MAX_RECV_SEG_LEN + 4096 - \ 64 (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge) + \ 65 sizeof(struct ib_cqe))) 66 67 #define ISCSI_ISER_SG_TABLESIZE 256 68 69 enum isert_desc_type { 70 ISCSI_TX_CONTROL, 71 ISCSI_TX_DATAIN 72 }; 73 74 enum iser_ib_op_code { 75 ISER_IB_RECV, 76 ISER_IB_SEND, 77 ISER_IB_RDMA_WRITE, 78 ISER_IB_RDMA_READ, 79 }; 80 81 enum iser_conn_state { 82 ISER_CONN_INIT, 83 ISER_CONN_UP, 84 ISER_CONN_BOUND, 85 ISER_CONN_FULL_FEATURE, 86 ISER_CONN_TERMINATING, 87 ISER_CONN_DOWN, 88 }; 89 90 struct iser_rx_desc { 91 struct iser_ctrl iser_header; 92 struct iscsi_hdr iscsi_header; 93 char data[ISCSI_DEF_MAX_RECV_SEG_LEN]; 94 u64 dma_addr; 95 struct ib_sge rx_sg; 96 struct ib_cqe rx_cqe; 97 char pad[ISER_RX_PAD_SIZE]; 98 } __packed; 99 100 static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe) 101 { 102 return container_of(cqe, struct iser_rx_desc, rx_cqe); 103 } 104 105 struct iser_tx_desc { 106 struct iser_ctrl iser_header; 107 struct iscsi_hdr iscsi_header; 108 enum isert_desc_type type; 109 u64 dma_addr; 110 struct ib_sge tx_sg[2]; 111 struct ib_cqe tx_cqe; 112 int num_sge; 113 struct ib_send_wr send_wr; 114 } __packed; 115 116 static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe) 117 { 118 return container_of(cqe, struct iser_tx_desc, tx_cqe); 119 } 120 121 122 enum isert_indicator { 123 ISERT_PROTECTED = 1 << 0, 124 ISERT_DATA_KEY_VALID = 1 << 1, 125 ISERT_PROT_KEY_VALID = 1 << 2, 126 ISERT_SIG_KEY_VALID = 1 << 3, 127 }; 128 129 struct pi_context { 130 struct ib_mr *prot_mr; 131 struct ib_mr *sig_mr; 132 }; 133 134 struct fast_reg_descriptor { 135 struct list_head list; 136 struct ib_mr *data_mr; 137 u8 ind; 138 struct pi_context *pi_ctx; 139 }; 140 141 struct isert_data_buf { 142 struct scatterlist *sg; 143 int nents; 144 u32 sg_off; 145 u32 len; /* cur_rdma_length */ 146 u32 offset; 147 unsigned int dma_nents; 148 enum dma_data_direction dma_dir; 149 }; 150 151 enum { 152 DATA = 0, 153 PROT = 1, 154 SIG = 2, 155 }; 156 157 struct isert_cmd { 158 uint32_t read_stag; 159 uint32_t write_stag; 160 uint64_t read_va; 161 uint64_t write_va; 162 uint32_t inv_rkey; 163 u64 pdu_buf_dma; 164 u32 pdu_buf_len; 165 struct isert_conn *conn; 166 struct iscsi_cmd *iscsi_cmd; 167 struct iser_tx_desc tx_desc; 168 struct iser_rx_desc *rx_desc; 169 enum iser_ib_op_code iser_ib_op; 170 struct ib_sge *ib_sge; 171 struct ib_sge s_ib_sge; 172 int rdma_wr_num; 173 struct ib_rdma_wr *rdma_wr; 174 struct ib_rdma_wr s_rdma_wr; 175 struct ib_sge ib_sg[3]; 176 struct isert_data_buf data; 177 struct isert_data_buf prot; 178 struct fast_reg_descriptor *fr_desc; 179 struct work_struct comp_work; 180 struct scatterlist sg; 181 }; 182 183 static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc) 184 { 185 return container_of(desc, struct isert_cmd, tx_desc); 186 } 187 188 struct isert_device; 189 190 struct isert_conn { 191 enum iser_conn_state state; 192 u32 responder_resources; 193 u32 initiator_depth; 194 bool pi_support; 195 u32 max_sge; 196 struct iser_rx_desc *login_req_buf; 197 char *login_rsp_buf; 198 u64 login_req_dma; 199 int login_req_len; 200 u64 login_rsp_dma; 201 struct iser_rx_desc *rx_descs; 202 struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS]; 203 struct iscsi_conn *conn; 204 struct list_head node; 205 struct completion login_comp; 206 struct completion login_req_comp; 207 struct iser_tx_desc login_tx_desc; 208 struct rdma_cm_id *cm_id; 209 struct ib_qp *qp; 210 struct isert_device *device; 211 struct mutex mutex; 212 struct completion wait_comp_err; 213 struct kref kref; 214 struct list_head fr_pool; 215 int fr_pool_size; 216 /* lock to protect fastreg pool */ 217 spinlock_t pool_lock; 218 struct work_struct release_work; 219 struct ib_recv_wr beacon; 220 bool logout_posted; 221 bool snd_w_inv; 222 }; 223 224 #define ISERT_MAX_CQ 64 225 226 /** 227 * struct isert_comp - iSER completion context 228 * 229 * @device: pointer to device handle 230 * @cq: completion queue 231 * @active_qps: Number of active QPs attached 232 * to completion context 233 */ 234 struct isert_comp { 235 struct isert_device *device; 236 struct ib_cq *cq; 237 int active_qps; 238 }; 239 240 struct isert_device { 241 int use_fastreg; 242 bool pi_capable; 243 int refcount; 244 struct ib_device *ib_device; 245 struct ib_pd *pd; 246 struct isert_comp *comps; 247 int comps_used; 248 struct list_head dev_node; 249 int (*reg_rdma_mem)(struct isert_cmd *isert_cmd, 250 struct iscsi_conn *conn); 251 void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, 252 struct isert_conn *isert_conn); 253 }; 254 255 struct isert_np { 256 struct iscsi_np *np; 257 struct semaphore sem; 258 struct rdma_cm_id *cm_id; 259 struct mutex mutex; 260 struct list_head accepted; 261 struct list_head pending; 262 }; 263