1 #include <linux/socket.h> 2 #include <linux/in.h> 3 #include <linux/in6.h> 4 #include <rdma/ib_verbs.h> 5 #include <rdma/rdma_cm.h> 6 #include <scsi/iser.h> 7 8 9 #define DRV_NAME "isert" 10 #define PFX DRV_NAME ": " 11 12 #define isert_dbg(fmt, arg...) \ 13 do { \ 14 if (unlikely(isert_debug_level > 2)) \ 15 printk(KERN_DEBUG PFX "%s: " fmt,\ 16 __func__ , ## arg); \ 17 } while (0) 18 19 #define isert_warn(fmt, arg...) \ 20 do { \ 21 if (unlikely(isert_debug_level > 0)) \ 22 pr_warn(PFX "%s: " fmt, \ 23 __func__ , ## arg); \ 24 } while (0) 25 26 #define isert_info(fmt, arg...) \ 27 do { \ 28 if (unlikely(isert_debug_level > 1)) \ 29 pr_info(PFX "%s: " fmt, \ 30 __func__ , ## arg); \ 31 } while (0) 32 33 #define isert_err(fmt, arg...) \ 34 pr_err(PFX "%s: " fmt, __func__ , ## arg) 35 36 /* Constant PDU lengths calculations */ 37 #define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + \ 38 sizeof(struct iscsi_hdr)) 39 #define ISER_RECV_DATA_SEG_LEN 8192 40 #define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN) 41 #define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) 42 43 /* QP settings */ 44 /* Maximal bounds on received asynchronous PDUs */ 45 #define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */ 46 47 #define ISERT_MAX_RX_MISC_PDUS 6 /* 48 * NOOP_OUT(2), TEXT(1), 49 * SCSI_TMFUNC(2), LOGOUT(1) 50 */ 51 52 #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */ 53 54 #define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) 55 56 #define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) 57 58 #define ISERT_INFLIGHT_DATAOUTS 8 59 60 #define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX * \ 61 (1 + ISERT_INFLIGHT_DATAOUTS) + \ 62 ISERT_MAX_TX_MISC_PDUS + \ 63 ISERT_MAX_RX_MISC_PDUS) 64 65 #define ISER_RX_PAD_SIZE (ISER_RECV_DATA_SEG_LEN + 4096 - \ 66 (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge))) 67 68 #define ISCSI_ISER_SG_TABLESIZE 256 69 #define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL 70 #define ISER_BEACON_WRID 0xfffffffffffffffeULL 71 72 enum isert_desc_type { 73 ISCSI_TX_CONTROL, 74 ISCSI_TX_DATAIN 75 }; 76 77 enum iser_ib_op_code { 78 ISER_IB_RECV, 79 ISER_IB_SEND, 80 ISER_IB_RDMA_WRITE, 81 ISER_IB_RDMA_READ, 82 }; 83 84 enum iser_conn_state { 85 ISER_CONN_INIT, 86 ISER_CONN_UP, 87 ISER_CONN_FULL_FEATURE, 88 ISER_CONN_TERMINATING, 89 ISER_CONN_DOWN, 90 }; 91 92 struct iser_rx_desc { 93 struct iser_ctrl iser_header; 94 struct iscsi_hdr iscsi_header; 95 char data[ISER_RECV_DATA_SEG_LEN]; 96 u64 dma_addr; 97 struct ib_sge rx_sg; 98 char pad[ISER_RX_PAD_SIZE]; 99 } __packed; 100 101 struct iser_tx_desc { 102 struct iser_ctrl iser_header; 103 struct iscsi_hdr iscsi_header; 104 enum isert_desc_type type; 105 u64 dma_addr; 106 struct ib_sge tx_sg[2]; 107 int num_sge; 108 struct isert_cmd *isert_cmd; 109 struct ib_send_wr send_wr; 110 } __packed; 111 112 enum isert_indicator { 113 ISERT_PROTECTED = 1 << 0, 114 ISERT_DATA_KEY_VALID = 1 << 1, 115 ISERT_PROT_KEY_VALID = 1 << 2, 116 ISERT_SIG_KEY_VALID = 1 << 3, 117 }; 118 119 struct pi_context { 120 struct ib_mr *prot_mr; 121 struct ib_mr *sig_mr; 122 }; 123 124 struct fast_reg_descriptor { 125 struct list_head list; 126 struct ib_mr *data_mr; 127 u8 ind; 128 struct pi_context *pi_ctx; 129 }; 130 131 struct isert_data_buf { 132 struct scatterlist *sg; 133 int nents; 134 u32 sg_off; 135 u32 len; /* cur_rdma_length */ 136 u32 offset; 137 unsigned int dma_nents; 138 enum dma_data_direction dma_dir; 139 }; 140 141 enum { 142 DATA = 0, 143 PROT = 1, 144 SIG = 2, 145 }; 146 147 struct isert_rdma_wr { 148 struct isert_cmd *isert_cmd; 149 enum iser_ib_op_code iser_ib_op; 150 struct ib_sge *ib_sge; 151 struct ib_sge s_ib_sge; 152 int rdma_wr_num; 153 struct ib_rdma_wr *rdma_wr; 154 struct ib_rdma_wr s_rdma_wr; 155 struct ib_sge ib_sg[3]; 156 struct isert_data_buf data; 157 struct isert_data_buf prot; 158 struct fast_reg_descriptor *fr_desc; 159 }; 160 161 struct isert_cmd { 162 uint32_t read_stag; 163 uint32_t write_stag; 164 uint64_t read_va; 165 uint64_t write_va; 166 uint32_t inv_rkey; 167 u64 pdu_buf_dma; 168 u32 pdu_buf_len; 169 struct isert_conn *conn; 170 struct iscsi_cmd *iscsi_cmd; 171 struct iser_tx_desc tx_desc; 172 struct iser_rx_desc *rx_desc; 173 struct isert_rdma_wr rdma_wr; 174 struct work_struct comp_work; 175 struct scatterlist sg; 176 }; 177 178 struct isert_device; 179 180 struct isert_conn { 181 enum iser_conn_state state; 182 int post_recv_buf_count; 183 u32 responder_resources; 184 u32 initiator_depth; 185 bool pi_support; 186 u32 max_sge; 187 char *login_buf; 188 char *login_req_buf; 189 char *login_rsp_buf; 190 u64 login_req_dma; 191 int login_req_len; 192 u64 login_rsp_dma; 193 struct iser_rx_desc *rx_descs; 194 struct ib_recv_wr rx_wr[ISERT_QP_MAX_RECV_DTOS]; 195 struct iscsi_conn *conn; 196 struct list_head node; 197 struct completion login_comp; 198 struct completion login_req_comp; 199 struct iser_tx_desc login_tx_desc; 200 struct rdma_cm_id *cm_id; 201 struct ib_qp *qp; 202 struct isert_device *device; 203 struct mutex mutex; 204 struct completion wait; 205 struct completion wait_comp_err; 206 struct kref kref; 207 struct list_head fr_pool; 208 int fr_pool_size; 209 /* lock to protect fastreg pool */ 210 spinlock_t pool_lock; 211 struct work_struct release_work; 212 struct ib_recv_wr beacon; 213 bool logout_posted; 214 bool snd_w_inv; 215 }; 216 217 #define ISERT_MAX_CQ 64 218 219 /** 220 * struct isert_comp - iSER completion context 221 * 222 * @device: pointer to device handle 223 * @cq: completion queue 224 * @wcs: work completion array 225 * @active_qps: Number of active QPs attached 226 * to completion context 227 * @work: completion work handle 228 */ 229 struct isert_comp { 230 struct isert_device *device; 231 struct ib_cq *cq; 232 struct ib_wc wcs[16]; 233 int active_qps; 234 struct work_struct work; 235 }; 236 237 struct isert_device { 238 int use_fastreg; 239 bool pi_capable; 240 int refcount; 241 struct ib_device *ib_device; 242 struct ib_pd *pd; 243 struct isert_comp *comps; 244 int comps_used; 245 struct list_head dev_node; 246 int (*reg_rdma_mem)(struct iscsi_conn *conn, 247 struct iscsi_cmd *cmd, 248 struct isert_rdma_wr *wr); 249 void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, 250 struct isert_conn *isert_conn); 251 }; 252 253 struct isert_np { 254 struct iscsi_np *np; 255 struct semaphore sem; 256 struct rdma_cm_id *cm_id; 257 struct mutex mutex; 258 struct list_head accepted; 259 struct list_head pending; 260 }; 261