1 /*
2  * iSCSI Initiator over iSER Data-Path
3  *
4  * Copyright (C) 2004 Dmitry Yusupov
5  * Copyright (C) 2004 Alex Aizman
6  * Copyright (C) 2005 Mike Christie
7  * Copyright (c) 2005, 2006 Voltaire, Inc. All rights reserved.
8  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
9  * maintained by openib-general@openib.org
10  *
11  * This software is available to you under a choice of one of two
12  * licenses.  You may choose to be licensed under the terms of the GNU
13  * General Public License (GPL) Version 2, available from the file
14  * COPYING in the main directory of this source tree, or the
15  * OpenIB.org BSD license below:
16  *
17  *     Redistribution and use in source and binary forms, with or
18  *     without modification, are permitted provided that the following
19  *     conditions are met:
20  *
21  *	- Redistributions of source code must retain the above
22  *	  copyright notice, this list of conditions and the following
23  *	  disclaimer.
24  *
25  *	- Redistributions in binary form must reproduce the above
26  *	  copyright notice, this list of conditions and the following
27  *	  disclaimer in the documentation and/or other materials
28  *	  provided with the distribution.
29  *
30  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
31  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
33  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
34  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
35  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
36  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37  * SOFTWARE.
38  *
39  * Credits:
40  *	Christoph Hellwig
41  *	FUJITA Tomonori
42  *	Arne Redlich
43  *	Zhenyu Wang
44  * Modified by:
45  *      Erez Zilber
46  */
47 
48 #include <linux/types.h>
49 #include <linux/list.h>
50 #include <linux/hardirq.h>
51 #include <linux/kfifo.h>
52 #include <linux/blkdev.h>
53 #include <linux/init.h>
54 #include <linux/ioctl.h>
55 #include <linux/cdev.h>
56 #include <linux/in.h>
57 #include <linux/net.h>
58 #include <linux/scatterlist.h>
59 #include <linux/delay.h>
60 #include <linux/slab.h>
61 #include <linux/module.h>
62 
63 #include <net/sock.h>
64 
65 #include <asm/uaccess.h>
66 
67 #include <scsi/scsi_cmnd.h>
68 #include <scsi/scsi_device.h>
69 #include <scsi/scsi_eh.h>
70 #include <scsi/scsi_tcq.h>
71 #include <scsi/scsi_host.h>
72 #include <scsi/scsi.h>
73 #include <scsi/scsi_transport_iscsi.h>
74 
75 #include "iscsi_iser.h"
76 
77 MODULE_DESCRIPTION("iSER (iSCSI Extensions for RDMA) Datamover");
78 MODULE_LICENSE("Dual BSD/GPL");
79 MODULE_AUTHOR("Alex Nezhinsky, Dan Bar Dov, Or Gerlitz");
80 MODULE_VERSION(DRV_VER);
81 
82 static struct scsi_host_template iscsi_iser_sht;
83 static struct iscsi_transport iscsi_iser_transport;
84 static struct scsi_transport_template *iscsi_iser_scsi_transport;
85 static struct workqueue_struct *release_wq;
86 struct iser_global ig;
87 
88 int iser_debug_level = 0;
89 module_param_named(debug_level, iser_debug_level, int, S_IRUGO | S_IWUSR);
90 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:disabled)");
91 
92 static unsigned int iscsi_max_lun = 512;
93 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
94 MODULE_PARM_DESC(max_lun, "Max LUNs to allow per session (default:512");
95 
96 unsigned int iser_max_sectors = ISER_DEF_MAX_SECTORS;
97 module_param_named(max_sectors, iser_max_sectors, uint, S_IRUGO | S_IWUSR);
98 MODULE_PARM_DESC(max_sectors, "Max number of sectors in a single scsi command (default:1024");
99 
100 bool iser_always_reg = true;
101 module_param_named(always_register, iser_always_reg, bool, S_IRUGO);
102 MODULE_PARM_DESC(always_register,
103 		 "Always register memory, even for continuous memory regions (default:true)");
104 
105 bool iser_pi_enable = false;
106 module_param_named(pi_enable, iser_pi_enable, bool, S_IRUGO);
107 MODULE_PARM_DESC(pi_enable, "Enable T10-PI offload support (default:disabled)");
108 
109 int iser_pi_guard;
110 module_param_named(pi_guard, iser_pi_guard, int, S_IRUGO);
111 MODULE_PARM_DESC(pi_guard, "T10-PI guard_type [deprecated]");
112 
113 /*
114  * iscsi_iser_recv() - Process a successfull recv completion
115  * @conn:         iscsi connection
116  * @hdr:          iscsi header
117  * @rx_data:      buffer containing receive data payload
118  * @rx_data_len:  length of rx_data
119  *
120  * Notes: In case of data length errors or iscsi PDU completion failures
121  *        this routine will signal iscsi layer of connection failure.
122  */
123 void
124 iscsi_iser_recv(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
125 		char *rx_data, int rx_data_len)
126 {
127 	int rc = 0;
128 	int datalen;
129 	int ahslen;
130 
131 	/* verify PDU length */
132 	datalen = ntoh24(hdr->dlength);
133 	if (datalen > rx_data_len || (datalen + 4) < rx_data_len) {
134 		iser_err("wrong datalen %d (hdr), %d (IB)\n",
135 			datalen, rx_data_len);
136 		rc = ISCSI_ERR_DATALEN;
137 		goto error;
138 	}
139 
140 	if (datalen != rx_data_len)
141 		iser_dbg("aligned datalen (%d) hdr, %d (IB)\n",
142 			datalen, rx_data_len);
143 
144 	/* read AHS */
145 	ahslen = hdr->hlength * 4;
146 
147 	rc = iscsi_complete_pdu(conn, hdr, rx_data, rx_data_len);
148 	if (rc && rc != ISCSI_ERR_NO_SCSI_CMD)
149 		goto error;
150 
151 	return;
152 error:
153 	iscsi_conn_failure(conn, rc);
154 }
155 
156 /**
157  * iscsi_iser_pdu_alloc() - allocate an iscsi-iser PDU
158  * @task:     iscsi task
159  * @opcode:   iscsi command opcode
160  *
161  * Netes: This routine can't fail, just assign iscsi task
162  *        hdr and max hdr size.
163  */
164 static int
165 iscsi_iser_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
166 {
167 	struct iscsi_iser_task *iser_task = task->dd_data;
168 
169 	task->hdr = (struct iscsi_hdr *)&iser_task->desc.iscsi_header;
170 	task->hdr_max = sizeof(iser_task->desc.iscsi_header);
171 
172 	return 0;
173 }
174 
175 /**
176  * iser_initialize_task_headers() - Initialize task headers
177  * @task:       iscsi task
178  * @tx_desc:    iser tx descriptor
179  *
180  * Notes:
181  * This routine may race with iser teardown flow for scsi
182  * error handling TMFs. So for TMF we should acquire the
183  * state mutex to avoid dereferencing the IB device which
184  * may have already been terminated.
185  */
186 int
187 iser_initialize_task_headers(struct iscsi_task *task,
188 			     struct iser_tx_desc *tx_desc)
189 {
190 	struct iser_conn *iser_conn = task->conn->dd_data;
191 	struct iser_device *device = iser_conn->ib_conn.device;
192 	struct iscsi_iser_task *iser_task = task->dd_data;
193 	u64 dma_addr;
194 	const bool mgmt_task = !task->sc && !in_interrupt();
195 	int ret = 0;
196 
197 	if (unlikely(mgmt_task))
198 		mutex_lock(&iser_conn->state_mutex);
199 
200 	if (unlikely(iser_conn->state != ISER_CONN_UP)) {
201 		ret = -ENODEV;
202 		goto out;
203 	}
204 
205 	dma_addr = ib_dma_map_single(device->ib_device, (void *)tx_desc,
206 				ISER_HEADERS_LEN, DMA_TO_DEVICE);
207 	if (ib_dma_mapping_error(device->ib_device, dma_addr)) {
208 		ret = -ENOMEM;
209 		goto out;
210 	}
211 
212 	tx_desc->wr_idx = 0;
213 	tx_desc->mapped = true;
214 	tx_desc->dma_addr = dma_addr;
215 	tx_desc->tx_sg[0].addr   = tx_desc->dma_addr;
216 	tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
217 	tx_desc->tx_sg[0].lkey   = device->pd->local_dma_lkey;
218 
219 	iser_task->iser_conn = iser_conn;
220 out:
221 	if (unlikely(mgmt_task))
222 		mutex_unlock(&iser_conn->state_mutex);
223 
224 	return ret;
225 }
226 
227 /**
228  * iscsi_iser_task_init() - Initialize iscsi-iser task
229  * @task: iscsi task
230  *
231  * Initialize the task for the scsi command or mgmt command.
232  *
233  * Return: Returns zero on success or -ENOMEM when failing
234  *         to init task headers (dma mapping error).
235  */
236 static int
237 iscsi_iser_task_init(struct iscsi_task *task)
238 {
239 	struct iscsi_iser_task *iser_task = task->dd_data;
240 	int ret;
241 
242 	ret = iser_initialize_task_headers(task, &iser_task->desc);
243 	if (ret) {
244 		iser_err("Failed to init task %p, err = %d\n",
245 			 iser_task, ret);
246 		return ret;
247 	}
248 
249 	/* mgmt task */
250 	if (!task->sc)
251 		return 0;
252 
253 	iser_task->command_sent = 0;
254 	iser_task_rdma_init(iser_task);
255 	iser_task->sc = task->sc;
256 
257 	return 0;
258 }
259 
260 /**
261  * iscsi_iser_mtask_xmit() - xmit management (immediate) task
262  * @conn: iscsi connection
263  * @task: task management task
264  *
265  * Notes:
266  *	The function can return -EAGAIN in which case caller must
267  *	call it again later, or recover. '0' return code means successful
268  *	xmit.
269  *
270  **/
271 static int
272 iscsi_iser_mtask_xmit(struct iscsi_conn *conn, struct iscsi_task *task)
273 {
274 	int error = 0;
275 
276 	iser_dbg("mtask xmit [cid %d itt 0x%x]\n", conn->id, task->itt);
277 
278 	error = iser_send_control(conn, task);
279 
280 	/* since iser xmits control with zero copy, tasks can not be recycled
281 	 * right after sending them.
282 	 * The recycling scheme is based on whether a response is expected
283 	 * - if yes, the task is recycled at iscsi_complete_pdu
284 	 * - if no,  the task is recycled at iser_snd_completion
285 	 */
286 	return error;
287 }
288 
289 static int
290 iscsi_iser_task_xmit_unsol_data(struct iscsi_conn *conn,
291 				 struct iscsi_task *task)
292 {
293 	struct iscsi_r2t_info *r2t = &task->unsol_r2t;
294 	struct iscsi_data hdr;
295 	int error = 0;
296 
297 	/* Send data-out PDUs while there's still unsolicited data to send */
298 	while (iscsi_task_has_unsol_data(task)) {
299 		iscsi_prep_data_out_pdu(task, r2t, &hdr);
300 		iser_dbg("Sending data-out: itt 0x%x, data count %d\n",
301 			   hdr.itt, r2t->data_count);
302 
303 		/* the buffer description has been passed with the command */
304 		/* Send the command */
305 		error = iser_send_data_out(conn, task, &hdr);
306 		if (error) {
307 			r2t->datasn--;
308 			goto iscsi_iser_task_xmit_unsol_data_exit;
309 		}
310 		r2t->sent += r2t->data_count;
311 		iser_dbg("Need to send %d more as data-out PDUs\n",
312 			   r2t->data_length - r2t->sent);
313 	}
314 
315 iscsi_iser_task_xmit_unsol_data_exit:
316 	return error;
317 }
318 
319 /**
320  * iscsi_iser_task_xmit() - xmit iscsi-iser task
321  * @task: iscsi task
322  *
323  * Return: zero on success or escalates $error on failure.
324  */
325 static int
326 iscsi_iser_task_xmit(struct iscsi_task *task)
327 {
328 	struct iscsi_conn *conn = task->conn;
329 	struct iscsi_iser_task *iser_task = task->dd_data;
330 	int error = 0;
331 
332 	if (!task->sc)
333 		return iscsi_iser_mtask_xmit(conn, task);
334 
335 	if (task->sc->sc_data_direction == DMA_TO_DEVICE) {
336 		BUG_ON(scsi_bufflen(task->sc) == 0);
337 
338 		iser_dbg("cmd [itt %x total %d imm %d unsol_data %d\n",
339 			   task->itt, scsi_bufflen(task->sc),
340 			   task->imm_count, task->unsol_r2t.data_length);
341 	}
342 
343 	iser_dbg("ctask xmit [cid %d itt 0x%x]\n",
344 		   conn->id, task->itt);
345 
346 	/* Send the cmd PDU */
347 	if (!iser_task->command_sent) {
348 		error = iser_send_command(conn, task);
349 		if (error)
350 			goto iscsi_iser_task_xmit_exit;
351 		iser_task->command_sent = 1;
352 	}
353 
354 	/* Send unsolicited data-out PDU(s) if necessary */
355 	if (iscsi_task_has_unsol_data(task))
356 		error = iscsi_iser_task_xmit_unsol_data(conn, task);
357 
358  iscsi_iser_task_xmit_exit:
359 	return error;
360 }
361 
362 /**
363  * iscsi_iser_cleanup_task() - cleanup an iscsi-iser task
364  * @task: iscsi task
365  *
366  * Notes: In case the RDMA device is already NULL (might have
367  *        been removed in DEVICE_REMOVAL CM event it will bail-out
368  *        without doing dma unmapping.
369  */
370 static void iscsi_iser_cleanup_task(struct iscsi_task *task)
371 {
372 	struct iscsi_iser_task *iser_task = task->dd_data;
373 	struct iser_tx_desc *tx_desc = &iser_task->desc;
374 	struct iser_conn *iser_conn = task->conn->dd_data;
375 	struct iser_device *device = iser_conn->ib_conn.device;
376 
377 	/* DEVICE_REMOVAL event might have already released the device */
378 	if (!device)
379 		return;
380 
381 	if (likely(tx_desc->mapped)) {
382 		ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr,
383 				    ISER_HEADERS_LEN, DMA_TO_DEVICE);
384 		tx_desc->mapped = false;
385 	}
386 
387 	/* mgmt tasks do not need special cleanup */
388 	if (!task->sc)
389 		return;
390 
391 	if (iser_task->status == ISER_TASK_STATUS_STARTED) {
392 		iser_task->status = ISER_TASK_STATUS_COMPLETED;
393 		iser_task_rdma_finalize(iser_task);
394 	}
395 }
396 
397 /**
398  * iscsi_iser_check_protection() - check protection information status of task.
399  * @task:     iscsi task
400  * @sector:   error sector if exsists (output)
401  *
402  * Return: zero if no data-integrity errors have occured
403  *         0x1: data-integrity error occured in the guard-block
404  *         0x2: data-integrity error occured in the reference tag
405  *         0x3: data-integrity error occured in the application tag
406  *
407  *         In addition the error sector is marked.
408  */
409 static u8
410 iscsi_iser_check_protection(struct iscsi_task *task, sector_t *sector)
411 {
412 	struct iscsi_iser_task *iser_task = task->dd_data;
413 
414 	if (iser_task->dir[ISER_DIR_IN])
415 		return iser_check_task_pi_status(iser_task, ISER_DIR_IN,
416 						 sector);
417 	else
418 		return iser_check_task_pi_status(iser_task, ISER_DIR_OUT,
419 						 sector);
420 }
421 
422 /**
423  * iscsi_iser_conn_create() - create a new iscsi-iser connection
424  * @cls_session: iscsi class connection
425  * @conn_idx:    connection index within the session (for MCS)
426  *
427  * Return: iscsi_cls_conn when iscsi_conn_setup succeeds or NULL
428  *         otherwise.
429  */
430 static struct iscsi_cls_conn *
431 iscsi_iser_conn_create(struct iscsi_cls_session *cls_session,
432 		       uint32_t conn_idx)
433 {
434 	struct iscsi_conn *conn;
435 	struct iscsi_cls_conn *cls_conn;
436 
437 	cls_conn = iscsi_conn_setup(cls_session, 0, conn_idx);
438 	if (!cls_conn)
439 		return NULL;
440 	conn = cls_conn->dd_data;
441 
442 	/*
443 	 * due to issues with the login code re iser sematics
444 	 * this not set in iscsi_conn_setup - FIXME
445 	 */
446 	conn->max_recv_dlength = ISER_RECV_DATA_SEG_LEN;
447 
448 	return cls_conn;
449 }
450 
451 /**
452  * iscsi_iser_conn_bind() - bind iscsi and iser connection structures
453  * @cls_session:     iscsi class session
454  * @cls_conn:        iscsi class connection
455  * @transport_eph:   transport end-point handle
456  * @is_leading:      indicate if this is the session leading connection (MCS)
457  *
458  * Return: zero on success, $error if iscsi_conn_bind fails and
459  *         -EINVAL in case end-point doesn't exsits anymore or iser connection
460  *         state is not UP (teardown already started).
461  */
462 static int
463 iscsi_iser_conn_bind(struct iscsi_cls_session *cls_session,
464 		     struct iscsi_cls_conn *cls_conn,
465 		     uint64_t transport_eph,
466 		     int is_leading)
467 {
468 	struct iscsi_conn *conn = cls_conn->dd_data;
469 	struct iser_conn *iser_conn;
470 	struct iscsi_endpoint *ep;
471 	int error;
472 
473 	error = iscsi_conn_bind(cls_session, cls_conn, is_leading);
474 	if (error)
475 		return error;
476 
477 	/* the transport ep handle comes from user space so it must be
478 	 * verified against the global ib connections list */
479 	ep = iscsi_lookup_endpoint(transport_eph);
480 	if (!ep) {
481 		iser_err("can't bind eph %llx\n",
482 			 (unsigned long long)transport_eph);
483 		return -EINVAL;
484 	}
485 	iser_conn = ep->dd_data;
486 
487 	mutex_lock(&iser_conn->state_mutex);
488 	if (iser_conn->state != ISER_CONN_UP) {
489 		error = -EINVAL;
490 		iser_err("iser_conn %p state is %d, teardown started\n",
491 			 iser_conn, iser_conn->state);
492 		goto out;
493 	}
494 
495 	error = iser_alloc_rx_descriptors(iser_conn, conn->session);
496 	if (error)
497 		goto out;
498 
499 	/* binds the iSER connection retrieved from the previously
500 	 * connected ep_handle to the iSCSI layer connection. exchanges
501 	 * connection pointers */
502 	iser_info("binding iscsi conn %p to iser_conn %p\n", conn, iser_conn);
503 
504 	conn->dd_data = iser_conn;
505 	iser_conn->iscsi_conn = conn;
506 
507 out:
508 	mutex_unlock(&iser_conn->state_mutex);
509 	return error;
510 }
511 
512 /**
513  * iscsi_iser_conn_start() - start iscsi-iser connection
514  * @cls_conn: iscsi class connection
515  *
516  * Notes: Here iser intialize (or re-initialize) stop_completion as
517  *        from this point iscsi must call conn_stop in session/connection
518  *        teardown so iser transport must wait for it.
519  */
520 static int
521 iscsi_iser_conn_start(struct iscsi_cls_conn *cls_conn)
522 {
523 	struct iscsi_conn *iscsi_conn;
524 	struct iser_conn *iser_conn;
525 
526 	iscsi_conn = cls_conn->dd_data;
527 	iser_conn = iscsi_conn->dd_data;
528 	reinit_completion(&iser_conn->stop_completion);
529 
530 	return iscsi_conn_start(cls_conn);
531 }
532 
533 /**
534  * iscsi_iser_conn_stop() - stop iscsi-iser connection
535  * @cls_conn:  iscsi class connection
536  * @flag:      indicate if recover or terminate (passed as is)
537  *
538  * Notes: Calling iscsi_conn_stop might theoretically race with
539  *        DEVICE_REMOVAL event and dereference a previously freed RDMA device
540  *        handle, so we call it under iser the state lock to protect against
541  *        this kind of race.
542  */
543 static void
544 iscsi_iser_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
545 {
546 	struct iscsi_conn *conn = cls_conn->dd_data;
547 	struct iser_conn *iser_conn = conn->dd_data;
548 
549 	iser_info("stopping iscsi_conn: %p, iser_conn: %p\n", conn, iser_conn);
550 
551 	/*
552 	 * Userspace may have goofed up and not bound the connection or
553 	 * might have only partially setup the connection.
554 	 */
555 	if (iser_conn) {
556 		mutex_lock(&iser_conn->state_mutex);
557 		iser_conn_terminate(iser_conn);
558 		iscsi_conn_stop(cls_conn, flag);
559 
560 		/* unbind */
561 		iser_conn->iscsi_conn = NULL;
562 		conn->dd_data = NULL;
563 
564 		complete(&iser_conn->stop_completion);
565 		mutex_unlock(&iser_conn->state_mutex);
566 	} else {
567 		iscsi_conn_stop(cls_conn, flag);
568 	}
569 }
570 
571 /**
572  * iscsi_iser_session_destroy() - destroy iscsi-iser session
573  * @cls_session: iscsi class session
574  *
575  * Removes and free iscsi host.
576  */
577 static void
578 iscsi_iser_session_destroy(struct iscsi_cls_session *cls_session)
579 {
580 	struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
581 
582 	iscsi_session_teardown(cls_session);
583 	iscsi_host_remove(shost);
584 	iscsi_host_free(shost);
585 }
586 
587 static inline unsigned int
588 iser_dif_prot_caps(int prot_caps)
589 {
590 	return ((prot_caps & IB_PROT_T10DIF_TYPE_1) ?
591 		SHOST_DIF_TYPE1_PROTECTION | SHOST_DIX_TYPE0_PROTECTION |
592 		SHOST_DIX_TYPE1_PROTECTION : 0) |
593 	       ((prot_caps & IB_PROT_T10DIF_TYPE_2) ?
594 		SHOST_DIF_TYPE2_PROTECTION | SHOST_DIX_TYPE2_PROTECTION : 0) |
595 	       ((prot_caps & IB_PROT_T10DIF_TYPE_3) ?
596 		SHOST_DIF_TYPE3_PROTECTION | SHOST_DIX_TYPE3_PROTECTION : 0);
597 }
598 
599 /**
600  * iscsi_iser_session_create() - create an iscsi-iser session
601  * @ep:             iscsi end-point handle
602  * @cmds_max:       maximum commands in this session
603  * @qdepth:         session command queue depth
604  * @initial_cmdsn:  initiator command sequnce number
605  *
606  * Allocates and adds a scsi host, expose DIF supprot if
607  * exists, and sets up an iscsi session.
608  */
609 static struct iscsi_cls_session *
610 iscsi_iser_session_create(struct iscsi_endpoint *ep,
611 			  uint16_t cmds_max, uint16_t qdepth,
612 			  uint32_t initial_cmdsn)
613 {
614 	struct iscsi_cls_session *cls_session;
615 	struct iscsi_session *session;
616 	struct Scsi_Host *shost;
617 	struct iser_conn *iser_conn = NULL;
618 	struct ib_conn *ib_conn;
619 	u16 max_cmds;
620 
621 	shost = iscsi_host_alloc(&iscsi_iser_sht, 0, 0);
622 	if (!shost)
623 		return NULL;
624 	shost->transportt = iscsi_iser_scsi_transport;
625 	shost->cmd_per_lun = qdepth;
626 	shost->max_lun = iscsi_max_lun;
627 	shost->max_id = 0;
628 	shost->max_channel = 0;
629 	shost->max_cmd_len = 16;
630 
631 	/*
632 	 * older userspace tools (before 2.0-870) did not pass us
633 	 * the leading conn's ep so this will be NULL;
634 	 */
635 	if (ep) {
636 		iser_conn = ep->dd_data;
637 		max_cmds = iser_conn->max_cmds;
638 		shost->sg_tablesize = iser_conn->scsi_sg_tablesize;
639 		shost->max_sectors = iser_conn->scsi_max_sectors;
640 
641 		mutex_lock(&iser_conn->state_mutex);
642 		if (iser_conn->state != ISER_CONN_UP) {
643 			iser_err("iser conn %p already started teardown\n",
644 				 iser_conn);
645 			mutex_unlock(&iser_conn->state_mutex);
646 			goto free_host;
647 		}
648 
649 		ib_conn = &iser_conn->ib_conn;
650 		if (ib_conn->pi_support) {
651 			u32 sig_caps = ib_conn->device->dev_attr.sig_prot_cap;
652 
653 			scsi_host_set_prot(shost, iser_dif_prot_caps(sig_caps));
654 			scsi_host_set_guard(shost, SHOST_DIX_GUARD_IP |
655 						   SHOST_DIX_GUARD_CRC);
656 		}
657 
658 		/*
659 		 * Limit the sg_tablesize and max_sectors based on the device
660 		 * max fastreg page list length.
661 		 */
662 		shost->sg_tablesize = min_t(unsigned short, shost->sg_tablesize,
663 			ib_conn->device->dev_attr.max_fast_reg_page_list_len);
664 		shost->max_sectors = min_t(unsigned int,
665 			1024, (shost->sg_tablesize * PAGE_SIZE) >> 9);
666 
667 		if (iscsi_host_add(shost,
668 				   ib_conn->device->ib_device->dma_device)) {
669 			mutex_unlock(&iser_conn->state_mutex);
670 			goto free_host;
671 		}
672 		mutex_unlock(&iser_conn->state_mutex);
673 	} else {
674 		max_cmds = ISER_DEF_XMIT_CMDS_MAX;
675 		if (iscsi_host_add(shost, NULL))
676 			goto free_host;
677 	}
678 
679 	if (cmds_max > max_cmds) {
680 		iser_info("cmds_max changed from %u to %u\n",
681 			  cmds_max, max_cmds);
682 		cmds_max = max_cmds;
683 	}
684 
685 	cls_session = iscsi_session_setup(&iscsi_iser_transport, shost,
686 					  cmds_max, 0,
687 					  sizeof(struct iscsi_iser_task),
688 					  initial_cmdsn, 0);
689 	if (!cls_session)
690 		goto remove_host;
691 	session = cls_session->dd_data;
692 
693 	shost->can_queue = session->scsi_cmds_max;
694 	return cls_session;
695 
696 remove_host:
697 	iscsi_host_remove(shost);
698 free_host:
699 	iscsi_host_free(shost);
700 	return NULL;
701 }
702 
703 static int
704 iscsi_iser_set_param(struct iscsi_cls_conn *cls_conn,
705 		     enum iscsi_param param, char *buf, int buflen)
706 {
707 	int value;
708 
709 	switch (param) {
710 	case ISCSI_PARAM_MAX_RECV_DLENGTH:
711 		/* TBD */
712 		break;
713 	case ISCSI_PARAM_HDRDGST_EN:
714 		sscanf(buf, "%d", &value);
715 		if (value) {
716 			iser_err("DataDigest wasn't negotiated to None\n");
717 			return -EPROTO;
718 		}
719 		break;
720 	case ISCSI_PARAM_DATADGST_EN:
721 		sscanf(buf, "%d", &value);
722 		if (value) {
723 			iser_err("DataDigest wasn't negotiated to None\n");
724 			return -EPROTO;
725 		}
726 		break;
727 	case ISCSI_PARAM_IFMARKER_EN:
728 		sscanf(buf, "%d", &value);
729 		if (value) {
730 			iser_err("IFMarker wasn't negotiated to No\n");
731 			return -EPROTO;
732 		}
733 		break;
734 	case ISCSI_PARAM_OFMARKER_EN:
735 		sscanf(buf, "%d", &value);
736 		if (value) {
737 			iser_err("OFMarker wasn't negotiated to No\n");
738 			return -EPROTO;
739 		}
740 		break;
741 	default:
742 		return iscsi_set_param(cls_conn, param, buf, buflen);
743 	}
744 
745 	return 0;
746 }
747 
748 /**
749  * iscsi_iser_set_param() - set class connection parameter
750  * @cls_conn:    iscsi class connection
751  * @stats:       iscsi stats to output
752  *
753  * Output connection statistics.
754  */
755 static void
756 iscsi_iser_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
757 {
758 	struct iscsi_conn *conn = cls_conn->dd_data;
759 
760 	stats->txdata_octets = conn->txdata_octets;
761 	stats->rxdata_octets = conn->rxdata_octets;
762 	stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
763 	stats->dataout_pdus = conn->dataout_pdus_cnt;
764 	stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
765 	stats->datain_pdus = conn->datain_pdus_cnt; /* always 0 */
766 	stats->r2t_pdus = conn->r2t_pdus_cnt; /* always 0 */
767 	stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
768 	stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
769 	stats->custom_length = 1;
770 	strcpy(stats->custom[0].desc, "fmr_unalign_cnt");
771 	stats->custom[0].value = conn->fmr_unalign_cnt;
772 }
773 
774 static int iscsi_iser_get_ep_param(struct iscsi_endpoint *ep,
775 				   enum iscsi_param param, char *buf)
776 {
777 	struct iser_conn *iser_conn = ep->dd_data;
778 	int len;
779 
780 	switch (param) {
781 	case ISCSI_PARAM_CONN_PORT:
782 	case ISCSI_PARAM_CONN_ADDRESS:
783 		if (!iser_conn || !iser_conn->ib_conn.cma_id)
784 			return -ENOTCONN;
785 
786 		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
787 				&iser_conn->ib_conn.cma_id->route.addr.dst_addr,
788 				param, buf);
789 		break;
790 	default:
791 		return -ENOSYS;
792 	}
793 
794 	return len;
795 }
796 
797 /**
798  * iscsi_iser_ep_connect() - Initiate iSER connection establishment
799  * @shost:          scsi_host
800  * @dst_addr:       destination address
801  * @non-blocking:   indicate if routine can block
802  *
803  * Allocate an iscsi endpoint, an iser_conn structure and bind them.
804  * After that start RDMA connection establishment via rdma_cm. We
805  * don't allocate iser_conn embedded in iscsi_endpoint since in teardown
806  * the endpoint will be destroyed at ep_disconnect while iser_conn will
807  * cleanup its resources asynchronuously.
808  *
809  * Return: iscsi_endpoint created by iscsi layer or ERR_PTR(error)
810  *         if fails.
811  */
812 static struct iscsi_endpoint *
813 iscsi_iser_ep_connect(struct Scsi_Host *shost, struct sockaddr *dst_addr,
814 		      int non_blocking)
815 {
816 	int err;
817 	struct iser_conn *iser_conn;
818 	struct iscsi_endpoint *ep;
819 
820 	ep = iscsi_create_endpoint(0);
821 	if (!ep)
822 		return ERR_PTR(-ENOMEM);
823 
824 	iser_conn = kzalloc(sizeof(*iser_conn), GFP_KERNEL);
825 	if (!iser_conn) {
826 		err = -ENOMEM;
827 		goto failure;
828 	}
829 
830 	ep->dd_data = iser_conn;
831 	iser_conn->ep = ep;
832 	iser_conn_init(iser_conn);
833 
834 	err = iser_connect(iser_conn, NULL, dst_addr, non_blocking);
835 	if (err)
836 		goto failure;
837 
838 	return ep;
839 failure:
840 	iscsi_destroy_endpoint(ep);
841 	return ERR_PTR(err);
842 }
843 
844 /**
845  * iscsi_iser_ep_poll() - poll for iser connection establishment to complete
846  * @ep:            iscsi endpoint (created at ep_connect)
847  * @timeout_ms:    polling timeout allowed in ms.
848  *
849  * This routine boils down to waiting for up_completion signaling
850  * that cma_id got CONNECTED event.
851  *
852  * Return: 1 if succeeded in connection establishment, 0 if timeout expired
853  *         (libiscsi will retry will kick in) or -1 if interrupted by signal
854  *         or more likely iser connection state transitioned to TEMINATING or
855  *         DOWN during the wait period.
856  */
857 static int
858 iscsi_iser_ep_poll(struct iscsi_endpoint *ep, int timeout_ms)
859 {
860 	struct iser_conn *iser_conn = ep->dd_data;
861 	int rc;
862 
863 	rc = wait_for_completion_interruptible_timeout(&iser_conn->up_completion,
864 						       msecs_to_jiffies(timeout_ms));
865 	/* if conn establishment failed, return error code to iscsi */
866 	if (rc == 0) {
867 		mutex_lock(&iser_conn->state_mutex);
868 		if (iser_conn->state == ISER_CONN_TERMINATING ||
869 		    iser_conn->state == ISER_CONN_DOWN)
870 			rc = -1;
871 		mutex_unlock(&iser_conn->state_mutex);
872 	}
873 
874 	iser_info("iser conn %p rc = %d\n", iser_conn, rc);
875 
876 	if (rc > 0)
877 		return 1; /* success, this is the equivalent of POLLOUT */
878 	else if (!rc)
879 		return 0; /* timeout */
880 	else
881 		return rc; /* signal */
882 }
883 
884 /**
885  * iscsi_iser_ep_disconnect() - Initiate connection teardown process
886  * @ep:    iscsi endpoint handle
887  *
888  * This routine is not blocked by iser and RDMA termination process
889  * completion as we queue a deffered work for iser/RDMA destruction
890  * and cleanup or actually call it immediately in case we didn't pass
891  * iscsi conn bind/start stage, thus it is safe.
892  */
893 static void
894 iscsi_iser_ep_disconnect(struct iscsi_endpoint *ep)
895 {
896 	struct iser_conn *iser_conn = ep->dd_data;
897 
898 	iser_info("ep %p iser conn %p\n", ep, iser_conn);
899 
900 	mutex_lock(&iser_conn->state_mutex);
901 	iser_conn_terminate(iser_conn);
902 
903 	/*
904 	 * if iser_conn and iscsi_conn are bound, we must wait for
905 	 * iscsi_conn_stop and flush errors completion before freeing
906 	 * the iser resources. Otherwise we are safe to free resources
907 	 * immediately.
908 	 */
909 	if (iser_conn->iscsi_conn) {
910 		INIT_WORK(&iser_conn->release_work, iser_release_work);
911 		queue_work(release_wq, &iser_conn->release_work);
912 		mutex_unlock(&iser_conn->state_mutex);
913 	} else {
914 		iser_conn->state = ISER_CONN_DOWN;
915 		mutex_unlock(&iser_conn->state_mutex);
916 		iser_conn_release(iser_conn);
917 	}
918 
919 	iscsi_destroy_endpoint(ep);
920 }
921 
922 static umode_t iser_attr_is_visible(int param_type, int param)
923 {
924 	switch (param_type) {
925 	case ISCSI_HOST_PARAM:
926 		switch (param) {
927 		case ISCSI_HOST_PARAM_NETDEV_NAME:
928 		case ISCSI_HOST_PARAM_HWADDRESS:
929 		case ISCSI_HOST_PARAM_INITIATOR_NAME:
930 			return S_IRUGO;
931 		default:
932 			return 0;
933 		}
934 	case ISCSI_PARAM:
935 		switch (param) {
936 		case ISCSI_PARAM_MAX_RECV_DLENGTH:
937 		case ISCSI_PARAM_MAX_XMIT_DLENGTH:
938 		case ISCSI_PARAM_HDRDGST_EN:
939 		case ISCSI_PARAM_DATADGST_EN:
940 		case ISCSI_PARAM_CONN_ADDRESS:
941 		case ISCSI_PARAM_CONN_PORT:
942 		case ISCSI_PARAM_EXP_STATSN:
943 		case ISCSI_PARAM_PERSISTENT_ADDRESS:
944 		case ISCSI_PARAM_PERSISTENT_PORT:
945 		case ISCSI_PARAM_PING_TMO:
946 		case ISCSI_PARAM_RECV_TMO:
947 		case ISCSI_PARAM_INITIAL_R2T_EN:
948 		case ISCSI_PARAM_MAX_R2T:
949 		case ISCSI_PARAM_IMM_DATA_EN:
950 		case ISCSI_PARAM_FIRST_BURST:
951 		case ISCSI_PARAM_MAX_BURST:
952 		case ISCSI_PARAM_PDU_INORDER_EN:
953 		case ISCSI_PARAM_DATASEQ_INORDER_EN:
954 		case ISCSI_PARAM_TARGET_NAME:
955 		case ISCSI_PARAM_TPGT:
956 		case ISCSI_PARAM_USERNAME:
957 		case ISCSI_PARAM_PASSWORD:
958 		case ISCSI_PARAM_USERNAME_IN:
959 		case ISCSI_PARAM_PASSWORD_IN:
960 		case ISCSI_PARAM_FAST_ABORT:
961 		case ISCSI_PARAM_ABORT_TMO:
962 		case ISCSI_PARAM_LU_RESET_TMO:
963 		case ISCSI_PARAM_TGT_RESET_TMO:
964 		case ISCSI_PARAM_IFACE_NAME:
965 		case ISCSI_PARAM_INITIATOR_NAME:
966 		case ISCSI_PARAM_DISCOVERY_SESS:
967 			return S_IRUGO;
968 		default:
969 			return 0;
970 		}
971 	}
972 
973 	return 0;
974 }
975 
976 static struct scsi_host_template iscsi_iser_sht = {
977 	.module                 = THIS_MODULE,
978 	.name                   = "iSCSI Initiator over iSER",
979 	.queuecommand           = iscsi_queuecommand,
980 	.change_queue_depth	= scsi_change_queue_depth,
981 	.sg_tablesize           = ISCSI_ISER_DEF_SG_TABLESIZE,
982 	.max_sectors            = ISER_DEF_MAX_SECTORS,
983 	.cmd_per_lun            = ISER_DEF_CMD_PER_LUN,
984 	.eh_abort_handler       = iscsi_eh_abort,
985 	.eh_device_reset_handler= iscsi_eh_device_reset,
986 	.eh_target_reset_handler = iscsi_eh_recover_target,
987 	.target_alloc		= iscsi_target_alloc,
988 	.use_clustering         = DISABLE_CLUSTERING,
989 	.proc_name              = "iscsi_iser",
990 	.this_id                = -1,
991 	.track_queue_depth	= 1,
992 };
993 
994 static struct iscsi_transport iscsi_iser_transport = {
995 	.owner                  = THIS_MODULE,
996 	.name                   = "iser",
997 	.caps                   = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_TEXT_NEGO,
998 	/* session management */
999 	.create_session         = iscsi_iser_session_create,
1000 	.destroy_session        = iscsi_iser_session_destroy,
1001 	/* connection management */
1002 	.create_conn            = iscsi_iser_conn_create,
1003 	.bind_conn              = iscsi_iser_conn_bind,
1004 	.destroy_conn           = iscsi_conn_teardown,
1005 	.attr_is_visible	= iser_attr_is_visible,
1006 	.set_param              = iscsi_iser_set_param,
1007 	.get_conn_param		= iscsi_conn_get_param,
1008 	.get_ep_param		= iscsi_iser_get_ep_param,
1009 	.get_session_param	= iscsi_session_get_param,
1010 	.start_conn             = iscsi_iser_conn_start,
1011 	.stop_conn              = iscsi_iser_conn_stop,
1012 	/* iscsi host params */
1013 	.get_host_param		= iscsi_host_get_param,
1014 	.set_host_param		= iscsi_host_set_param,
1015 	/* IO */
1016 	.send_pdu		= iscsi_conn_send_pdu,
1017 	.get_stats		= iscsi_iser_conn_get_stats,
1018 	.init_task		= iscsi_iser_task_init,
1019 	.xmit_task		= iscsi_iser_task_xmit,
1020 	.cleanup_task		= iscsi_iser_cleanup_task,
1021 	.alloc_pdu		= iscsi_iser_pdu_alloc,
1022 	.check_protection	= iscsi_iser_check_protection,
1023 	/* recovery */
1024 	.session_recovery_timedout = iscsi_session_recovery_timedout,
1025 
1026 	.ep_connect             = iscsi_iser_ep_connect,
1027 	.ep_poll                = iscsi_iser_ep_poll,
1028 	.ep_disconnect          = iscsi_iser_ep_disconnect
1029 };
1030 
1031 static int __init iser_init(void)
1032 {
1033 	int err;
1034 
1035 	iser_dbg("Starting iSER datamover...\n");
1036 
1037 	if (iscsi_max_lun < 1) {
1038 		iser_err("Invalid max_lun value of %u\n", iscsi_max_lun);
1039 		return -EINVAL;
1040 	}
1041 
1042 	memset(&ig, 0, sizeof(struct iser_global));
1043 
1044 	ig.desc_cache = kmem_cache_create("iser_descriptors",
1045 					  sizeof(struct iser_tx_desc),
1046 					  0, SLAB_HWCACHE_ALIGN,
1047 					  NULL);
1048 	if (ig.desc_cache == NULL)
1049 		return -ENOMEM;
1050 
1051 	/* device init is called only after the first addr resolution */
1052 	mutex_init(&ig.device_list_mutex);
1053 	INIT_LIST_HEAD(&ig.device_list);
1054 	mutex_init(&ig.connlist_mutex);
1055 	INIT_LIST_HEAD(&ig.connlist);
1056 
1057 	release_wq = alloc_workqueue("release workqueue", 0, 0);
1058 	if (!release_wq) {
1059 		iser_err("failed to allocate release workqueue\n");
1060 		return -ENOMEM;
1061 	}
1062 
1063 	iscsi_iser_scsi_transport = iscsi_register_transport(
1064 							&iscsi_iser_transport);
1065 	if (!iscsi_iser_scsi_transport) {
1066 		iser_err("iscsi_register_transport failed\n");
1067 		err = -EINVAL;
1068 		goto register_transport_failure;
1069 	}
1070 
1071 	return 0;
1072 
1073 register_transport_failure:
1074 	kmem_cache_destroy(ig.desc_cache);
1075 
1076 	return err;
1077 }
1078 
1079 static void __exit iser_exit(void)
1080 {
1081 	struct iser_conn *iser_conn, *n;
1082 	int connlist_empty;
1083 
1084 	iser_dbg("Removing iSER datamover...\n");
1085 	destroy_workqueue(release_wq);
1086 
1087 	mutex_lock(&ig.connlist_mutex);
1088 	connlist_empty = list_empty(&ig.connlist);
1089 	mutex_unlock(&ig.connlist_mutex);
1090 
1091 	if (!connlist_empty) {
1092 		iser_err("Error cleanup stage completed but we still have iser "
1093 			 "connections, destroying them anyway\n");
1094 		list_for_each_entry_safe(iser_conn, n, &ig.connlist,
1095 					 conn_list) {
1096 			iser_conn_release(iser_conn);
1097 		}
1098 	}
1099 
1100 	iscsi_unregister_transport(&iscsi_iser_transport);
1101 	kmem_cache_destroy(ig.desc_cache);
1102 }
1103 
1104 module_init(iser_init);
1105 module_exit(iser_exit);
1106