1 #include <linux/socket.h> 2 #include <linux/in.h> 3 #include <linux/in6.h> 4 #include <rdma/ib_verbs.h> 5 #include <rdma/rdma_cm.h> 6 7 #define ISERT_RDMA_LISTEN_BACKLOG 10 8 #define ISCSI_ISER_SG_TABLESIZE 256 9 #define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL 10 11 enum isert_desc_type { 12 ISCSI_TX_CONTROL, 13 ISCSI_TX_DATAIN 14 }; 15 16 enum iser_ib_op_code { 17 ISER_IB_RECV, 18 ISER_IB_SEND, 19 ISER_IB_RDMA_WRITE, 20 ISER_IB_RDMA_READ, 21 }; 22 23 enum iser_conn_state { 24 ISER_CONN_INIT, 25 ISER_CONN_UP, 26 ISER_CONN_TERMINATING, 27 ISER_CONN_DOWN, 28 }; 29 30 struct iser_rx_desc { 31 struct iser_hdr iser_header; 32 struct iscsi_hdr iscsi_header; 33 char data[ISER_RECV_DATA_SEG_LEN]; 34 u64 dma_addr; 35 struct ib_sge rx_sg; 36 char pad[ISER_RX_PAD_SIZE]; 37 } __packed; 38 39 struct iser_tx_desc { 40 struct iser_hdr iser_header; 41 struct iscsi_hdr iscsi_header; 42 enum isert_desc_type type; 43 u64 dma_addr; 44 struct ib_sge tx_sg[2]; 45 int num_sge; 46 struct isert_cmd *isert_cmd; 47 struct llist_node *comp_llnode_batch; 48 struct llist_node comp_llnode; 49 bool llnode_active; 50 struct ib_send_wr send_wr; 51 } __packed; 52 53 enum isert_indicator { 54 ISERT_PROTECTED = 1 << 0, 55 ISERT_DATA_KEY_VALID = 1 << 1, 56 ISERT_PROT_KEY_VALID = 1 << 2, 57 ISERT_SIG_KEY_VALID = 1 << 3, 58 }; 59 60 struct pi_context { 61 struct ib_mr *prot_mr; 62 struct ib_fast_reg_page_list *prot_frpl; 63 struct ib_mr *sig_mr; 64 }; 65 66 struct fast_reg_descriptor { 67 struct list_head list; 68 struct ib_mr *data_mr; 69 struct ib_fast_reg_page_list *data_frpl; 70 u8 ind; 71 struct pi_context *pi_ctx; 72 }; 73 74 struct isert_data_buf { 75 struct scatterlist *sg; 76 int nents; 77 u32 sg_off; 78 u32 len; /* cur_rdma_length */ 79 u32 offset; 80 unsigned int dma_nents; 81 enum dma_data_direction dma_dir; 82 }; 83 84 struct isert_rdma_wr { 85 struct list_head wr_list; 86 struct isert_cmd *isert_cmd; 87 enum iser_ib_op_code iser_ib_op; 88 struct ib_sge *ib_sge; 89 struct ib_sge s_ib_sge; 90 int send_wr_num; 91 struct ib_send_wr *send_wr; 92 struct ib_send_wr s_send_wr; 93 struct isert_data_buf data; 94 struct isert_data_buf prot; 95 struct fast_reg_descriptor *fr_desc; 96 }; 97 98 struct isert_cmd { 99 uint32_t read_stag; 100 uint32_t write_stag; 101 uint64_t read_va; 102 uint64_t write_va; 103 u64 pdu_buf_dma; 104 u32 pdu_buf_len; 105 u32 read_va_off; 106 u32 write_va_off; 107 u32 rdma_wr_num; 108 struct isert_conn *conn; 109 struct iscsi_cmd *iscsi_cmd; 110 struct iser_tx_desc tx_desc; 111 struct isert_rdma_wr rdma_wr; 112 struct work_struct comp_work; 113 }; 114 115 struct isert_device; 116 117 struct isert_conn { 118 enum iser_conn_state state; 119 bool logout_posted; 120 int post_recv_buf_count; 121 atomic_t post_send_buf_count; 122 u32 responder_resources; 123 u32 initiator_depth; 124 u32 max_sge; 125 char *login_buf; 126 char *login_req_buf; 127 char *login_rsp_buf; 128 u64 login_req_dma; 129 u64 login_rsp_dma; 130 unsigned int conn_rx_desc_head; 131 struct iser_rx_desc *conn_rx_descs; 132 struct ib_recv_wr conn_rx_wr[ISERT_MIN_POSTED_RX]; 133 struct iscsi_conn *conn; 134 struct list_head conn_accept_node; 135 struct completion conn_login_comp; 136 struct iser_tx_desc conn_login_tx_desc; 137 struct rdma_cm_id *conn_cm_id; 138 struct ib_pd *conn_pd; 139 struct ib_mr *conn_mr; 140 struct ib_qp *conn_qp; 141 struct isert_device *conn_device; 142 struct work_struct conn_logout_work; 143 struct mutex conn_mutex; 144 struct completion conn_wait; 145 struct completion conn_wait_comp_err; 146 struct kref conn_kref; 147 struct list_head conn_fr_pool; 148 int conn_fr_pool_size; 149 /* lock to protect fastreg pool */ 150 spinlock_t conn_lock; 151 #define ISERT_COMP_BATCH_COUNT 8 152 int conn_comp_batch; 153 struct llist_head conn_comp_llist; 154 }; 155 156 #define ISERT_MAX_CQ 64 157 158 struct isert_cq_desc { 159 struct isert_device *device; 160 int cq_index; 161 struct work_struct cq_rx_work; 162 struct work_struct cq_tx_work; 163 }; 164 165 struct isert_device { 166 int use_fastreg; 167 bool pi_capable; 168 int cqs_used; 169 int refcount; 170 int cq_active_qps[ISERT_MAX_CQ]; 171 struct ib_device *ib_device; 172 struct ib_cq *dev_rx_cq[ISERT_MAX_CQ]; 173 struct ib_cq *dev_tx_cq[ISERT_MAX_CQ]; 174 struct isert_cq_desc *cq_desc; 175 struct list_head dev_node; 176 struct ib_device_attr dev_attr; 177 int (*reg_rdma_mem)(struct iscsi_conn *conn, 178 struct iscsi_cmd *cmd, 179 struct isert_rdma_wr *wr); 180 void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd, 181 struct isert_conn *isert_conn); 182 }; 183 184 struct isert_np { 185 struct semaphore np_sem; 186 struct rdma_cm_id *np_cm_id; 187 struct mutex np_accept_mutex; 188 struct list_head np_accept_list; 189 struct completion np_login_comp; 190 }; 191