1 /*
2  * iSER transport for the Open iSCSI Initiator & iSER transport internals
3  *
4  * Copyright (C) 2004 Dmitry Yusupov
5  * Copyright (C) 2004 Alex Aizman
6  * Copyright (C) 2005 Mike Christie
7  * based on code maintained by open-iscsi@googlegroups.com
8  *
9  * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
10  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
11  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
12  *
13  * This software is available to you under a choice of one of two
14  * licenses.  You may choose to be licensed under the terms of the GNU
15  * General Public License (GPL) Version 2, available from the file
16  * COPYING in the main directory of this source tree, or the
17  * OpenIB.org BSD license below:
18  *
19  *     Redistribution and use in source and binary forms, with or
20  *     without modification, are permitted provided that the following
21  *     conditions are met:
22  *
23  *	- Redistributions of source code must retain the above
24  *	  copyright notice, this list of conditions and the following
25  *	  disclaimer.
26  *
27  *	- Redistributions in binary form must reproduce the above
28  *	  copyright notice, this list of conditions and the following
29  *	  disclaimer in the documentation and/or other materials
30  *	  provided with the distribution.
31  *
32  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
33  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
34  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
35  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
36  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
37  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
38  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39  * SOFTWARE.
40  */
41 #ifndef __ISCSI_ISER_H__
42 #define __ISCSI_ISER_H__
43 
44 #include <linux/types.h>
45 #include <linux/net.h>
46 #include <linux/printk.h>
47 #include <scsi/libiscsi.h>
48 #include <scsi/scsi_transport_iscsi.h>
49 #include <scsi/scsi_cmnd.h>
50 #include <scsi/scsi_device.h>
51 
52 #include <linux/interrupt.h>
53 #include <linux/wait.h>
54 #include <linux/sched.h>
55 #include <linux/list.h>
56 #include <linux/slab.h>
57 #include <linux/dma-mapping.h>
58 #include <linux/mutex.h>
59 #include <linux/mempool.h>
60 #include <linux/uio.h>
61 
62 #include <linux/socket.h>
63 #include <linux/in.h>
64 #include <linux/in6.h>
65 
66 #include <rdma/ib_verbs.h>
67 #include <rdma/ib_fmr_pool.h>
68 #include <rdma/rdma_cm.h>
69 
70 #define DRV_NAME	"iser"
71 #define PFX		DRV_NAME ": "
72 #define DRV_VER		"1.3"
73 
74 #define iser_dbg(fmt, arg...)				\
75 	do {						\
76 		if (iser_debug_level > 2)		\
77 			printk(KERN_DEBUG PFX "%s:" fmt,\
78 				__func__ , ## arg);	\
79 	} while (0)
80 
81 #define iser_warn(fmt, arg...)				\
82 	do {						\
83 		if (iser_debug_level > 0)		\
84 			pr_warn(PFX "%s:" fmt,          \
85 				__func__ , ## arg);	\
86 	} while (0)
87 
88 #define iser_info(fmt, arg...)				\
89 	do {						\
90 		if (iser_debug_level > 1)		\
91 			pr_info(PFX "%s:" fmt,          \
92 				__func__ , ## arg);	\
93 	} while (0)
94 
95 #define iser_err(fmt, arg...)				\
96 	do {						\
97 		printk(KERN_ERR PFX "%s:" fmt,          \
98 		       __func__ , ## arg);		\
99 	} while (0)
100 
101 #define SHIFT_4K	12
102 #define SIZE_4K	(1ULL << SHIFT_4K)
103 #define MASK_4K	(~(SIZE_4K-1))
104 
105 					/* support up to 512KB in one RDMA */
106 #define ISCSI_ISER_SG_TABLESIZE         (0x80000 >> SHIFT_4K)
107 #define ISER_DEF_XMIT_CMDS_DEFAULT		512
108 #if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
109 	#define ISER_DEF_XMIT_CMDS_MAX		ISCSI_DEF_XMIT_CMDS_MAX
110 #else
111 	#define ISER_DEF_XMIT_CMDS_MAX		ISER_DEF_XMIT_CMDS_DEFAULT
112 #endif
113 #define ISER_DEF_CMD_PER_LUN		ISER_DEF_XMIT_CMDS_MAX
114 
115 /* QP settings */
116 /* Maximal bounds on received asynchronous PDUs */
117 #define ISER_MAX_RX_MISC_PDUS		4 /* NOOP_IN(2) , ASYNC_EVENT(2)   */
118 
119 #define ISER_MAX_TX_MISC_PDUS		6 /* NOOP_OUT(2), TEXT(1),         *
120 					   * SCSI_TMFUNC(2), LOGOUT(1) */
121 
122 #define ISER_QP_MAX_RECV_DTOS		(ISER_DEF_XMIT_CMDS_MAX)
123 
124 #define ISER_MIN_POSTED_RX		(ISER_DEF_XMIT_CMDS_MAX >> 2)
125 
126 /* the max TX (send) WR supported by the iSER QP is defined by                 *
127  * max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect   *
128  * to have at max for SCSI command. The tx posting & completion handling code  *
129  * supports -EAGAIN scheme where tx is suspended till the QP has room for more *
130  * send WR. D=8 comes from 64K/8K                                              */
131 
132 #define ISER_INFLIGHT_DATAOUTS		8
133 
134 #define ISER_QP_MAX_REQ_DTOS		(ISER_DEF_XMIT_CMDS_MAX *    \
135 					(1 + ISER_INFLIGHT_DATAOUTS) + \
136 					ISER_MAX_TX_MISC_PDUS        + \
137 					ISER_MAX_RX_MISC_PDUS)
138 
139 /* Max registration work requests per command */
140 #define ISER_MAX_REG_WR_PER_CMD		5
141 
142 /* For Signature we don't support DATAOUTs so no need to make room for them */
143 #define ISER_QP_SIG_MAX_REQ_DTOS	(ISER_DEF_XMIT_CMDS_MAX	*       \
144 					(1 + ISER_MAX_REG_WR_PER_CMD) + \
145 					ISER_MAX_TX_MISC_PDUS         + \
146 					ISER_MAX_RX_MISC_PDUS)
147 
148 #define ISER_VER			0x10
149 #define ISER_WSV			0x08
150 #define ISER_RSV			0x04
151 
152 #define ISER_FASTREG_LI_WRID		0xffffffffffffffffULL
153 
154 struct iser_hdr {
155 	u8      flags;
156 	u8      rsvd[3];
157 	__be32  write_stag; /* write rkey */
158 	__be64  write_va;
159 	__be32  read_stag;  /* read rkey */
160 	__be64  read_va;
161 } __attribute__((packed));
162 
163 
164 #define ISER_ZBVA_NOT_SUPPORTED		0x80
165 #define ISER_SEND_W_INV_NOT_SUPPORTED	0x40
166 
167 struct iser_cm_hdr {
168 	u8      flags;
169 	u8      rsvd[3];
170 } __packed;
171 
172 /* Constant PDU lengths calculations */
173 #define ISER_HEADERS_LEN  (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr))
174 
175 #define ISER_RECV_DATA_SEG_LEN	128
176 #define ISER_RX_PAYLOAD_SIZE	(ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
177 #define ISER_RX_LOGIN_SIZE	(ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
178 
179 /* Length of an object name string */
180 #define ISER_OBJECT_NAME_SIZE		    64
181 
182 enum iser_ib_conn_state {
183 	ISER_CONN_INIT,		   /* descriptor allocd, no conn          */
184 	ISER_CONN_PENDING,	   /* in the process of being established */
185 	ISER_CONN_UP,		   /* up and running                      */
186 	ISER_CONN_TERMINATING,	   /* in the process of being terminated  */
187 	ISER_CONN_DOWN,		   /* shut down                           */
188 	ISER_CONN_STATES_NUM
189 };
190 
191 enum iser_task_status {
192 	ISER_TASK_STATUS_INIT = 0,
193 	ISER_TASK_STATUS_STARTED,
194 	ISER_TASK_STATUS_COMPLETED
195 };
196 
197 enum iser_data_dir {
198 	ISER_DIR_IN = 0,	   /* to initiator */
199 	ISER_DIR_OUT,		   /* from initiator */
200 	ISER_DIRS_NUM
201 };
202 
203 struct iser_data_buf {
204 	void               *buf;      /* pointer to the sg list               */
205 	unsigned int       size;      /* num entries of this sg               */
206 	unsigned long      data_len;  /* total data len                       */
207 	unsigned int       dma_nents; /* returned by dma_map_sg               */
208 	char       	   *copy_buf; /* allocated copy buf for SGs unaligned *
209 	                               * for rdma which are copied            */
210 	struct scatterlist sg_single; /* SG-ified clone of a non SG SC or     *
211 				       * unaligned SG                         */
212   };
213 
214 /* fwd declarations */
215 struct iser_device;
216 struct iser_cq_desc;
217 struct iscsi_iser_task;
218 struct iscsi_endpoint;
219 
220 struct iser_mem_reg {
221 	u32  lkey;
222 	u32  rkey;
223 	u64  va;
224 	u64  len;
225 	void *mem_h;
226 	int  is_mr;
227 };
228 
229 struct iser_regd_buf {
230 	struct iser_mem_reg     reg;        /* memory registration info        */
231 	void                    *virt_addr;
232 	struct iser_device      *device;    /* device->device for dma_unmap    */
233 	enum dma_data_direction direction;  /* direction for dma_unmap	       */
234 	unsigned int            data_size;
235 };
236 
237 enum iser_desc_type {
238 	ISCSI_TX_CONTROL ,
239 	ISCSI_TX_SCSI_COMMAND,
240 	ISCSI_TX_DATAOUT
241 };
242 
243 struct iser_tx_desc {
244 	struct iser_hdr              iser_header;
245 	struct iscsi_hdr             iscsi_header;
246 	enum   iser_desc_type        type;
247 	u64		             dma_addr;
248 	/* sg[0] points to iser/iscsi headers, sg[1] optionally points to either
249 	of immediate data, unsolicited data-out or control (login,text) */
250 	struct ib_sge		     tx_sg[2];
251 	int                          num_sge;
252 };
253 
254 #define ISER_RX_PAD_SIZE	(256 - (ISER_RX_PAYLOAD_SIZE + \
255 					sizeof(u64) + sizeof(struct ib_sge)))
256 struct iser_rx_desc {
257 	struct iser_hdr              iser_header;
258 	struct iscsi_hdr             iscsi_header;
259 	char		             data[ISER_RECV_DATA_SEG_LEN];
260 	u64		             dma_addr;
261 	struct ib_sge		     rx_sg;
262 	char		             pad[ISER_RX_PAD_SIZE];
263 } __attribute__((packed));
264 
265 #define ISER_MAX_CQ 4
266 
267 struct iser_conn;
268 struct iscsi_iser_task;
269 
270 struct iser_device {
271 	struct ib_device             *ib_device;
272 	struct ib_pd	             *pd;
273 	struct ib_device_attr	     dev_attr;
274 	struct ib_cq	             *rx_cq[ISER_MAX_CQ];
275 	struct ib_cq	             *tx_cq[ISER_MAX_CQ];
276 	struct ib_mr	             *mr;
277 	struct tasklet_struct	     cq_tasklet[ISER_MAX_CQ];
278 	struct ib_event_handler      event_handler;
279 	struct list_head             ig_list; /* entry in ig devices list */
280 	int                          refcount;
281 	int                          cq_active_qps[ISER_MAX_CQ];
282 	int			     cqs_used;
283 	struct iser_cq_desc	     *cq_desc;
284 	int                          (*iser_alloc_rdma_reg_res)(struct iser_conn *ib_conn,
285 								unsigned cmds_max);
286 	void                         (*iser_free_rdma_reg_res)(struct iser_conn *ib_conn);
287 	int                          (*iser_reg_rdma_mem)(struct iscsi_iser_task *iser_task,
288 							  enum iser_data_dir cmd_dir);
289 	void                         (*iser_unreg_rdma_mem)(struct iscsi_iser_task *iser_task,
290 							    enum iser_data_dir cmd_dir);
291 };
292 
293 #define ISER_CHECK_GUARD	0xc0
294 #define ISER_CHECK_REFTAG	0x0f
295 #define ISER_CHECK_APPTAG	0x30
296 
297 enum iser_reg_indicator {
298 	ISER_DATA_KEY_VALID	= 1 << 0,
299 	ISER_PROT_KEY_VALID	= 1 << 1,
300 	ISER_SIG_KEY_VALID	= 1 << 2,
301 	ISER_FASTREG_PROTECTED	= 1 << 3,
302 };
303 
304 struct iser_pi_context {
305 	struct ib_mr                   *prot_mr;
306 	struct ib_fast_reg_page_list   *prot_frpl;
307 	struct ib_mr                   *sig_mr;
308 };
309 
310 struct fast_reg_descriptor {
311 	struct list_head		  list;
312 	/* For fast registration - FRWR */
313 	struct ib_mr			 *data_mr;
314 	struct ib_fast_reg_page_list     *data_frpl;
315 	struct iser_pi_context		 *pi_ctx;
316 	/* registration indicators container */
317 	u8				  reg_indicators;
318 };
319 
320 struct iser_conn {
321 	struct iscsi_conn	     *iscsi_conn;
322 	struct iscsi_endpoint	     *ep;
323 	enum iser_ib_conn_state	     state;	    /* rdma connection state   */
324 	atomic_t		     refcount;
325 	spinlock_t		     lock;	    /* used for state changes  */
326 	struct iser_device           *device;       /* device context          */
327 	struct rdma_cm_id            *cma_id;       /* CMA ID		       */
328 	struct ib_qp	             *qp;           /* QP 		       */
329 	wait_queue_head_t	     wait;          /* waitq for conn/disconn  */
330 	unsigned		     qp_max_recv_dtos; /* num of rx buffers */
331 	unsigned		     qp_max_recv_dtos_mask; /* above minus 1 */
332 	unsigned		     min_posted_rx; /* qp_max_recv_dtos >> 2 */
333 	int                          post_recv_buf_count; /* posted rx count  */
334 	atomic_t                     post_send_buf_count; /* posted tx count   */
335 	char 			     name[ISER_OBJECT_NAME_SIZE];
336 	struct list_head	     conn_list;       /* entry in ig conn list */
337 
338 	char  			     *login_buf;
339 	char			     *login_req_buf, *login_resp_buf;
340 	u64			     login_req_dma, login_resp_dma;
341 	unsigned int 		     rx_desc_head;
342 	struct iser_rx_desc	     *rx_descs;
343 	struct ib_recv_wr	     rx_wr[ISER_MIN_POSTED_RX];
344 	bool			     pi_support;
345 
346 	/* Connection memory registration pool */
347 	union {
348 		struct {
349 			struct ib_fmr_pool      *pool;	   /* pool of IB FMRs         */
350 			struct iser_page_vec	*page_vec; /* represents SG to fmr maps*
351 							    * maps serialized as tx is*/
352 		} fmr;
353 		struct {
354 			struct list_head	pool;
355 			int			pool_size;
356 		} fastreg;
357 	};
358 };
359 
360 struct iscsi_iser_task {
361 	struct iser_tx_desc          desc;
362 	struct iser_conn	     *ib_conn;
363 	enum iser_task_status 	     status;
364 	struct scsi_cmnd	     *sc;
365 	int                          command_sent;  /* set if command  sent  */
366 	int                          dir[ISER_DIRS_NUM];      /* set if dir use*/
367 	struct iser_regd_buf         rdma_regd[ISER_DIRS_NUM];/* regd rdma buf */
368 	struct iser_data_buf         data[ISER_DIRS_NUM];     /* orig. data des*/
369 	struct iser_data_buf         data_copy[ISER_DIRS_NUM];/* contig. copy  */
370 	struct iser_data_buf         prot[ISER_DIRS_NUM];     /* prot desc     */
371 	struct iser_data_buf         prot_copy[ISER_DIRS_NUM];/* prot copy     */
372 };
373 
374 struct iser_page_vec {
375 	u64 *pages;
376 	int length;
377 	int offset;
378 	int data_size;
379 };
380 
381 struct iser_cq_desc {
382 	struct iser_device           *device;
383 	int                          cq_index;
384 };
385 
386 struct iser_global {
387 	struct mutex      device_list_mutex;/*                   */
388 	struct list_head  device_list;	     /* all iSER devices */
389 	struct mutex      connlist_mutex;
390 	struct list_head  connlist;		/* all iSER IB connections */
391 
392 	struct kmem_cache *desc_cache;
393 };
394 
395 extern struct iser_global ig;
396 extern int iser_debug_level;
397 extern bool iser_pi_enable;
398 extern int iser_pi_guard;
399 
400 /* allocate connection resources needed for rdma functionality */
401 int iser_conn_set_full_featured_mode(struct iscsi_conn *conn);
402 
403 int iser_send_control(struct iscsi_conn *conn,
404 		      struct iscsi_task *task);
405 
406 int iser_send_command(struct iscsi_conn *conn,
407 		      struct iscsi_task *task);
408 
409 int iser_send_data_out(struct iscsi_conn *conn,
410 		       struct iscsi_task *task,
411 		       struct iscsi_data *hdr);
412 
413 void iscsi_iser_recv(struct iscsi_conn *conn,
414 		     struct iscsi_hdr       *hdr,
415 		     char                   *rx_data,
416 		     int                    rx_data_len);
417 
418 void iser_conn_init(struct iser_conn *ib_conn);
419 
420 void iser_conn_get(struct iser_conn *ib_conn);
421 
422 int iser_conn_put(struct iser_conn *ib_conn, int destroy_cma_id_allowed);
423 
424 void iser_conn_terminate(struct iser_conn *ib_conn);
425 
426 void iser_rcv_completion(struct iser_rx_desc *desc,
427 			 unsigned long    dto_xfer_len,
428 			struct iser_conn *ib_conn);
429 
430 void iser_snd_completion(struct iser_tx_desc *desc, struct iser_conn *ib_conn);
431 
432 void iser_task_rdma_init(struct iscsi_iser_task *task);
433 
434 void iser_task_rdma_finalize(struct iscsi_iser_task *task);
435 
436 void iser_free_rx_descriptors(struct iser_conn *ib_conn);
437 
438 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
439 				     struct iser_data_buf *mem,
440 				     struct iser_data_buf *mem_copy,
441 				     enum iser_data_dir cmd_dir);
442 
443 int  iser_reg_rdma_mem_fmr(struct iscsi_iser_task *task,
444 			   enum iser_data_dir cmd_dir);
445 int  iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *task,
446 			       enum iser_data_dir cmd_dir);
447 
448 int  iser_connect(struct iser_conn   *ib_conn,
449 		  struct sockaddr_in *src_addr,
450 		  struct sockaddr_in *dst_addr,
451 		  int                non_blocking);
452 
453 int  iser_reg_page_vec(struct iser_conn     *ib_conn,
454 		       struct iser_page_vec *page_vec,
455 		       struct iser_mem_reg  *mem_reg);
456 
457 void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
458 			enum iser_data_dir cmd_dir);
459 void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
460 			    enum iser_data_dir cmd_dir);
461 
462 int  iser_post_recvl(struct iser_conn *ib_conn);
463 int  iser_post_recvm(struct iser_conn *ib_conn, int count);
464 int  iser_post_send(struct iser_conn *ib_conn, struct iser_tx_desc *tx_desc);
465 
466 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
467 			    struct iser_data_buf       *data,
468 			    enum   iser_data_dir       iser_dir,
469 			    enum   dma_data_direction  dma_dir);
470 
471 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
472 			      struct iser_data_buf *data);
473 int  iser_initialize_task_headers(struct iscsi_task *task,
474 			struct iser_tx_desc *tx_desc);
475 int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session);
476 int iser_create_fmr_pool(struct iser_conn *ib_conn, unsigned cmds_max);
477 void iser_free_fmr_pool(struct iser_conn *ib_conn);
478 int iser_create_fastreg_pool(struct iser_conn *ib_conn, unsigned cmds_max);
479 void iser_free_fastreg_pool(struct iser_conn *ib_conn);
480 u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
481 			     enum iser_data_dir cmd_dir, sector_t *sector);
482 #endif
483