1 #include <linux/socket.h> 2 #include <linux/in.h> 3 #include <linux/in6.h> 4 #include <rdma/ib_verbs.h> 5 #include <rdma/rdma_cm.h> 6 7 #define DRV_NAME "isert" 8 #define PFX DRV_NAME ": " 9 10 #define isert_dbg(fmt, arg...) \ 11 do { \ 12 if (unlikely(isert_debug_level > 2)) \ 13 printk(KERN_DEBUG PFX "%s: " fmt,\ 14 __func__ , ## arg); \ 15 } while (0) 16 17 #define isert_warn(fmt, arg...) \ 18 do { \ 19 if (unlikely(isert_debug_level > 0)) \ 20 pr_warn(PFX "%s: " fmt, \ 21 __func__ , ## arg); \ 22 } while (0) 23 24 #define isert_info(fmt, arg...) \ 25 do { \ 26 if (unlikely(isert_debug_level > 1)) \ 27 pr_info(PFX "%s: " fmt, \ 28 __func__ , ## arg); \ 29 } while (0) 30 31 #define isert_err(fmt, arg...) \ 32 pr_err(PFX "%s: " fmt, __func__ , ## arg) 33 34 #define ISCSI_ISER_SG_TABLESIZE 256 35 #define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL 36 #define ISER_BEACON_WRID 0xfffffffffffffffeULL 37 38 enum isert_desc_type { 39 ISCSI_TX_CONTROL, 40 ISCSI_TX_DATAIN 41 }; 42 43 enum iser_ib_op_code { 44 ISER_IB_RECV, 45 ISER_IB_SEND, 46 ISER_IB_RDMA_WRITE, 47 ISER_IB_RDMA_READ, 48 }; 49 50 enum iser_conn_state { 51 ISER_CONN_INIT, 52 ISER_CONN_UP, 53 ISER_CONN_FULL_FEATURE, 54 ISER_CONN_TERMINATING, 55 ISER_CONN_DOWN, 56 }; 57 58 struct iser_rx_desc { 59 struct iser_hdr iser_header; 60 struct iscsi_hdr iscsi_header; 61 char data[ISER_RECV_DATA_SEG_LEN]; 62 u64 dma_addr; 63 struct ib_sge rx_sg; 64 char pad[ISER_RX_PAD_SIZE]; 65 } __packed; 66 67 struct iser_tx_desc { 68 struct iser_hdr iser_header; 69 struct iscsi_hdr iscsi_header; 70 enum isert_desc_type type; 71 u64 dma_addr; 72 struct ib_sge tx_sg[2]; 73 int num_sge; 74 struct isert_cmd *isert_cmd; 75 struct ib_send_wr send_wr; 76 } __packed; 77 78 enum isert_indicator { 79 ISERT_PROTECTED = 1 << 0, 80 ISERT_DATA_KEY_VALID = 1 << 1, 81 ISERT_PROT_KEY_VALID = 1 << 2, 82 ISERT_SIG_KEY_VALID = 1 << 3, 83 }; 84 85 struct pi_context { 86 struct ib_mr *prot_mr; 87 struct ib_fast_reg_page_list *prot_frpl; 88 struct ib_mr *sig_mr; 89 }; 90 91 struct fast_reg_descriptor { 92 struct list_head list; 93 struct ib_mr *data_mr; 94 struct ib_fast_reg_page_list *data_frpl; 95 u8 ind; 96 struct pi_context *pi_ctx; 97 }; 98 99 struct isert_data_buf { 100 struct scatterlist *sg; 101 int nents; 102 u32 sg_off; 103 u32 len; /* cur_rdma_length */ 104 u32 offset; 105 unsigned int dma_nents; 106 enum dma_data_direction dma_dir; 107 }; 108 109 enum { 110 DATA = 0, 111 PROT = 1, 112 SIG = 2, 113 }; 114 115 struct isert_rdma_wr { 116 struct list_head wr_list; 117 struct isert_cmd *isert_cmd; 118 enum iser_ib_op_code iser_ib_op; 119 struct ib_sge *ib_sge; 120 struct ib_sge s_ib_sge; 121 int send_wr_num; 122 struct ib_send_wr *send_wr; 123 struct ib_send_wr s_send_wr; 124 struct ib_sge ib_sg[3]; 125 struct isert_data_buf data; 126 struct isert_data_buf prot; 127 struct fast_reg_descriptor *fr_desc; 128 }; 129 130 struct isert_cmd { 131 uint32_t read_stag; 132 uint32_t write_stag; 133 uint64_t read_va; 134 uint64_t write_va; 135 u64 pdu_buf_dma; 136 u32 pdu_buf_len; 137 u32 read_va_off; 138 u32 write_va_off; 139 u32 rdma_wr_num; 140 struct isert_conn *conn; 141 struct iscsi_cmd *iscsi_cmd; 142 struct iser_tx_desc tx_desc; 143 struct isert_rdma_wr rdma_wr; 144 struct work_struct comp_work; 145 }; 146 147 struct isert_device; 148 149 struct isert_conn { 150 enum iser_conn_state state; 151 int post_recv_buf_count; 152 u32 responder_resources; 153 u32 initiator_depth; 154 bool pi_support; 155 u32 max_sge; 156 char *login_buf; 157 char *login_req_buf; 158 char *login_rsp_buf; 159 u64 login_req_dma; 160 int login_req_len; 161 u64 login_rsp_dma; 162 unsigned int rx_desc_head; 163 struct iser_rx_desc *rx_descs; 164 struct ib_recv_wr rx_wr[ISERT_MIN_POSTED_RX]; 165 struct iscsi_conn *conn; 166 struct list_head accept_node; 167 struct completion login_comp; 168 struct completion login_req_comp; 169 struct iser_tx_desc login_tx_desc; 170 struct rdma_cm_id *cm_id; 171 struct ib_qp *qp; 172 struct isert_device *device; 173 struct mutex mutex; 174 struct completion wait; 175 struct completion wait_comp_err; 176 struct kref kref; 177 struct list_head fr_pool; 178 int fr_pool_size; 179 /* lock to protect fastreg pool */ 180 spinlock_t pool_lock; 181 struct work_struct release_work; 182 struct ib_recv_wr beacon; 183 bool logout_posted; 184 }; 185 186 #define ISERT_MAX_CQ 64 187 188 /** 189 * struct isert_comp - iSER completion context 190 * 191 * @device: pointer to device handle 192 * @cq: completion queue 193 * @wcs: work completion array 194 * @active_qps: Number of active QPs attached 195 * to completion context 196 * @work: completion work handle 197 */ 198 struct isert_comp { 199 struct isert_device *device; 200 struct ib_cq *cq; 201 struct ib_wc wcs[16]; 202 int active_qps; 203 struct work_struct work; 204 }; 205 206 struct isert_device { 207 int use_fastreg; 208 bool pi_capable; 209 int refcount; 210 struct ib_device *ib_device; 211 struct ib_pd *pd; 212 struct ib_mr *mr; 213 struct isert_comp *comps; 214 int comps_used; 215 struct list_head dev_node; 216 struct ib_device_attr dev_attr; 217 int (*reg_rdma_mem)(struct iscsi_conn *conn, 218 struct iscsi_cmd *cmd, 219 struct isert_rdma_wr *wr); 220 void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, 221 struct isert_conn *isert_conn); 222 }; 223 224 struct isert_np { 225 struct iscsi_np *np; 226 struct semaphore np_sem; 227 struct rdma_cm_id *np_cm_id; 228 struct mutex np_accept_mutex; 229 struct list_head np_accept_list; 230 struct completion np_login_comp; 231 }; 232