1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include <linux/socket.h>
3 #include <linux/in.h>
4 #include <linux/in6.h>
5 #include <rdma/ib_verbs.h>
6 #include <rdma/rdma_cm.h>
7 #include <rdma/rw.h>
8 #include <scsi/iser.h>
9 
10 
11 #define DRV_NAME	"isert"
12 #define PFX		DRV_NAME ": "
13 
14 #define isert_dbg(fmt, arg...)				 \
15 	do {						 \
16 		if (unlikely(isert_debug_level > 2))	 \
17 			printk(KERN_DEBUG PFX "%s: " fmt,\
18 				__func__ , ## arg);	 \
19 	} while (0)
20 
21 #define isert_warn(fmt, arg...)				\
22 	do {						\
23 		if (unlikely(isert_debug_level > 0))	\
24 			pr_warn(PFX "%s: " fmt,         \
25 				__func__ , ## arg);	\
26 	} while (0)
27 
28 #define isert_info(fmt, arg...)				\
29 	do {						\
30 		if (unlikely(isert_debug_level > 1))	\
31 			pr_info(PFX "%s: " fmt,         \
32 				__func__ , ## arg);	\
33 	} while (0)
34 
35 #define isert_err(fmt, arg...) \
36 	pr_err(PFX "%s: " fmt, __func__ , ## arg)
37 
38 /* Constant PDU lengths calculations */
39 #define ISER_HEADERS_LEN	(sizeof(struct iser_ctrl) + \
40 				 sizeof(struct iscsi_hdr))
41 #define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
42 
43 /* QP settings */
44 /* Maximal bounds on received asynchronous PDUs */
45 #define ISERT_MAX_TX_MISC_PDUS	4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
46 
47 #define ISERT_MAX_RX_MISC_PDUS	6 /*
48 				   * NOOP_OUT(2), TEXT(1),
49 				   * SCSI_TMFUNC(2), LOGOUT(1)
50 				   */
51 
52 #define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */
53 
54 #define ISERT_QP_MAX_RECV_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX)
55 
56 #define ISERT_MIN_POSTED_RX	(ISCSI_DEF_XMIT_CMDS_MAX >> 2)
57 
58 #define ISERT_QP_MAX_REQ_DTOS	(ISCSI_DEF_XMIT_CMDS_MAX +    \
59 				ISERT_MAX_TX_MISC_PDUS	+ \
60 				ISERT_MAX_RX_MISC_PDUS)
61 
62 /*
63  * RX size is default of 8k plus headers, but data needs to align to
64  * 512 boundary, so use 1024 to have the extra space for alignment.
65  */
66 #define ISER_RX_SIZE		(ISCSI_DEF_MAX_RECV_SEG_LEN + 1024)
67 
68 /* Minimum I/O size is 512KB */
69 #define ISCSI_ISER_MIN_SG_TABLESIZE 128
70 
71 /* Maximum support is 16MB I/O size */
72 #define ISCSI_ISER_MAX_SG_TABLESIZE	4096
73 
74 enum isert_desc_type {
75 	ISCSI_TX_CONTROL,
76 	ISCSI_TX_DATAIN
77 };
78 
79 enum iser_conn_state {
80 	ISER_CONN_INIT,
81 	ISER_CONN_UP,
82 	ISER_CONN_BOUND,
83 	ISER_CONN_FULL_FEATURE,
84 	ISER_CONN_TERMINATING,
85 	ISER_CONN_DOWN,
86 };
87 
88 struct iser_rx_desc {
89 	char		buf[ISER_RX_SIZE];
90 	u64		dma_addr;
91 	struct ib_sge	rx_sg;
92 	struct ib_cqe	rx_cqe;
93 	bool		in_use;
94 };
95 
96 static inline struct iser_rx_desc *cqe_to_rx_desc(struct ib_cqe *cqe)
97 {
98 	return container_of(cqe, struct iser_rx_desc, rx_cqe);
99 }
100 
101 static void *isert_get_iser_hdr(struct iser_rx_desc *desc)
102 {
103 	return PTR_ALIGN(desc->buf + ISER_HEADERS_LEN, 512) - ISER_HEADERS_LEN;
104 }
105 
106 static size_t isert_get_hdr_offset(struct iser_rx_desc *desc)
107 {
108 	return isert_get_iser_hdr(desc) - (void *)desc->buf;
109 }
110 
111 static void *isert_get_iscsi_hdr(struct iser_rx_desc *desc)
112 {
113 	return isert_get_iser_hdr(desc) + sizeof(struct iser_ctrl);
114 }
115 
116 static void *isert_get_data(struct iser_rx_desc *desc)
117 {
118 	void *data = isert_get_iser_hdr(desc) + ISER_HEADERS_LEN;
119 
120 	WARN_ON((uintptr_t)data & 511);
121 	return data;
122 }
123 
124 struct iser_tx_desc {
125 	struct iser_ctrl iser_header;
126 	struct iscsi_hdr iscsi_header;
127 	enum isert_desc_type type;
128 	u64		dma_addr;
129 	struct ib_sge	tx_sg[2];
130 	struct ib_cqe	tx_cqe;
131 	int		num_sge;
132 	struct ib_send_wr send_wr;
133 } __packed;
134 
135 static inline struct iser_tx_desc *cqe_to_tx_desc(struct ib_cqe *cqe)
136 {
137 	return container_of(cqe, struct iser_tx_desc, tx_cqe);
138 }
139 
140 struct isert_cmd {
141 	uint32_t		read_stag;
142 	uint32_t		write_stag;
143 	uint64_t		read_va;
144 	uint64_t		write_va;
145 	uint32_t		inv_rkey;
146 	u64			pdu_buf_dma;
147 	u32			pdu_buf_len;
148 	struct isert_conn	*conn;
149 	struct iscsi_cmd	*iscsi_cmd;
150 	struct iser_tx_desc	tx_desc;
151 	struct iser_rx_desc	*rx_desc;
152 	struct rdma_rw_ctx	rw;
153 	struct work_struct	comp_work;
154 	struct scatterlist	sg;
155 	bool			ctx_init_done;
156 };
157 
158 static inline struct isert_cmd *tx_desc_to_cmd(struct iser_tx_desc *desc)
159 {
160 	return container_of(desc, struct isert_cmd, tx_desc);
161 }
162 
163 struct isert_device;
164 
165 struct isert_conn {
166 	enum iser_conn_state	state;
167 	u32			responder_resources;
168 	u32			initiator_depth;
169 	bool			pi_support;
170 	struct iser_rx_desc	*login_desc;
171 	char			*login_rsp_buf;
172 	int			login_req_len;
173 	u64			login_rsp_dma;
174 	struct iser_rx_desc	*rx_descs;
175 	struct ib_recv_wr	rx_wr[ISERT_QP_MAX_RECV_DTOS];
176 	struct iscsi_conn	*conn;
177 	struct list_head	node;
178 	struct completion	login_comp;
179 	struct completion	login_req_comp;
180 	struct iser_tx_desc	login_tx_desc;
181 	struct rdma_cm_id	*cm_id;
182 	struct ib_qp		*qp;
183 	struct ib_cq		*cq;
184 	u32			cq_size;
185 	struct isert_device	*device;
186 	struct mutex		mutex;
187 	struct kref		kref;
188 	struct work_struct	release_work;
189 	bool                    logout_posted;
190 	bool                    snd_w_inv;
191 	wait_queue_head_t	rem_wait;
192 	bool			dev_removed;
193 };
194 
195 struct isert_device {
196 	bool			pi_capable;
197 	int			refcount;
198 	struct ib_device	*ib_device;
199 	struct ib_pd		*pd;
200 	struct isert_comp	*comps;
201 	int                     comps_used;
202 	struct list_head	dev_node;
203 };
204 
205 struct isert_np {
206 	struct iscsi_np         *np;
207 	struct semaphore	sem;
208 	struct rdma_cm_id	*cm_id;
209 	struct mutex		mutex;
210 	struct list_head	accepted;
211 	struct list_head	pending;
212 };
213