xref: /openbmc/linux/drivers/scsi/iscsi_tcp.c (revision e23feb16)
1 /*
2  * iSCSI Initiator over TCP/IP Data-Path
3  *
4  * Copyright (C) 2004 Dmitry Yusupov
5  * Copyright (C) 2004 Alex Aizman
6  * Copyright (C) 2005 - 2006 Mike Christie
7  * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
8  * maintained by open-iscsi@googlegroups.com
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published
12  * by the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful, but
16  * WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18  * General Public License for more details.
19  *
20  * See the file COPYING included with this distribution for more details.
21  *
22  * Credits:
23  *	Christoph Hellwig
24  *	FUJITA Tomonori
25  *	Arne Redlich
26  *	Zhenyu Wang
27  */
28 
29 #include <linux/types.h>
30 #include <linux/inet.h>
31 #include <linux/slab.h>
32 #include <linux/file.h>
33 #include <linux/blkdev.h>
34 #include <linux/crypto.h>
35 #include <linux/delay.h>
36 #include <linux/kfifo.h>
37 #include <linux/scatterlist.h>
38 #include <linux/module.h>
39 #include <net/tcp.h>
40 #include <scsi/scsi_cmnd.h>
41 #include <scsi/scsi_device.h>
42 #include <scsi/scsi_host.h>
43 #include <scsi/scsi.h>
44 #include <scsi/scsi_transport_iscsi.h>
45 
46 #include "iscsi_tcp.h"
47 
48 MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "
49 	      "Dmitry Yusupov <dmitry_yus@yahoo.com>, "
50 	      "Alex Aizman <itn780@yahoo.com>");
51 MODULE_DESCRIPTION("iSCSI/TCP data-path");
52 MODULE_LICENSE("GPL");
53 
54 static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport;
55 static struct scsi_host_template iscsi_sw_tcp_sht;
56 static struct iscsi_transport iscsi_sw_tcp_transport;
57 
58 static unsigned int iscsi_max_lun = ~0;
59 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
60 
61 static int iscsi_sw_tcp_dbg;
62 module_param_named(debug_iscsi_tcp, iscsi_sw_tcp_dbg, int,
63 		   S_IRUGO | S_IWUSR);
64 MODULE_PARM_DESC(debug_iscsi_tcp, "Turn on debugging for iscsi_tcp module "
65 		 "Set to 1 to turn on, and zero to turn off. Default is off.");
66 
67 #define ISCSI_SW_TCP_DBG(_conn, dbg_fmt, arg...)		\
68 	do {							\
69 		if (iscsi_sw_tcp_dbg)				\
70 			iscsi_conn_printk(KERN_INFO, _conn,	\
71 					     "%s " dbg_fmt,	\
72 					     __func__, ##arg);	\
73 	} while (0);
74 
75 
76 /**
77  * iscsi_sw_tcp_recv - TCP receive in sendfile fashion
78  * @rd_desc: read descriptor
79  * @skb: socket buffer
80  * @offset: offset in skb
81  * @len: skb->len - offset
82  */
83 static int iscsi_sw_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
84 			     unsigned int offset, size_t len)
85 {
86 	struct iscsi_conn *conn = rd_desc->arg.data;
87 	unsigned int consumed, total_consumed = 0;
88 	int status;
89 
90 	ISCSI_SW_TCP_DBG(conn, "in %d bytes\n", skb->len - offset);
91 
92 	do {
93 		status = 0;
94 		consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status);
95 		offset += consumed;
96 		total_consumed += consumed;
97 	} while (consumed != 0 && status != ISCSI_TCP_SKB_DONE);
98 
99 	ISCSI_SW_TCP_DBG(conn, "read %d bytes status %d\n",
100 			 skb->len - offset, status);
101 	return total_consumed;
102 }
103 
104 /**
105  * iscsi_sw_sk_state_check - check socket state
106  * @sk: socket
107  *
108  * If the socket is in CLOSE or CLOSE_WAIT we should
109  * not close the connection if there is still some
110  * data pending.
111  *
112  * Must be called with sk_callback_lock.
113  */
114 static inline int iscsi_sw_sk_state_check(struct sock *sk)
115 {
116 	struct iscsi_conn *conn = sk->sk_user_data;
117 
118 	if ((sk->sk_state == TCP_CLOSE_WAIT || sk->sk_state == TCP_CLOSE) &&
119 	    !atomic_read(&sk->sk_rmem_alloc)) {
120 		ISCSI_SW_TCP_DBG(conn, "TCP_CLOSE|TCP_CLOSE_WAIT\n");
121 		iscsi_conn_failure(conn, ISCSI_ERR_TCP_CONN_CLOSE);
122 		return -ECONNRESET;
123 	}
124 	return 0;
125 }
126 
127 static void iscsi_sw_tcp_data_ready(struct sock *sk, int flag)
128 {
129 	struct iscsi_conn *conn;
130 	struct iscsi_tcp_conn *tcp_conn;
131 	read_descriptor_t rd_desc;
132 
133 	read_lock(&sk->sk_callback_lock);
134 	conn = sk->sk_user_data;
135 	if (!conn) {
136 		read_unlock(&sk->sk_callback_lock);
137 		return;
138 	}
139 	tcp_conn = conn->dd_data;
140 
141 	/*
142 	 * Use rd_desc to pass 'conn' to iscsi_tcp_recv.
143 	 * We set count to 1 because we want the network layer to
144 	 * hand us all the skbs that are available. iscsi_tcp_recv
145 	 * handled pdus that cross buffers or pdus that still need data.
146 	 */
147 	rd_desc.arg.data = conn;
148 	rd_desc.count = 1;
149 	tcp_read_sock(sk, &rd_desc, iscsi_sw_tcp_recv);
150 
151 	iscsi_sw_sk_state_check(sk);
152 
153 	/* If we had to (atomically) map a highmem page,
154 	 * unmap it now. */
155 	iscsi_tcp_segment_unmap(&tcp_conn->in.segment);
156 	read_unlock(&sk->sk_callback_lock);
157 }
158 
159 static void iscsi_sw_tcp_state_change(struct sock *sk)
160 {
161 	struct iscsi_tcp_conn *tcp_conn;
162 	struct iscsi_sw_tcp_conn *tcp_sw_conn;
163 	struct iscsi_conn *conn;
164 	struct iscsi_session *session;
165 	void (*old_state_change)(struct sock *);
166 
167 	read_lock(&sk->sk_callback_lock);
168 	conn = sk->sk_user_data;
169 	if (!conn) {
170 		read_unlock(&sk->sk_callback_lock);
171 		return;
172 	}
173 	session = conn->session;
174 
175 	iscsi_sw_sk_state_check(sk);
176 
177 	tcp_conn = conn->dd_data;
178 	tcp_sw_conn = tcp_conn->dd_data;
179 	old_state_change = tcp_sw_conn->old_state_change;
180 
181 	read_unlock(&sk->sk_callback_lock);
182 
183 	old_state_change(sk);
184 }
185 
186 /**
187  * iscsi_write_space - Called when more output buffer space is available
188  * @sk: socket space is available for
189  **/
190 static void iscsi_sw_tcp_write_space(struct sock *sk)
191 {
192 	struct iscsi_conn *conn;
193 	struct iscsi_tcp_conn *tcp_conn;
194 	struct iscsi_sw_tcp_conn *tcp_sw_conn;
195 	void (*old_write_space)(struct sock *);
196 
197 	read_lock_bh(&sk->sk_callback_lock);
198 	conn = sk->sk_user_data;
199 	if (!conn) {
200 		read_unlock_bh(&sk->sk_callback_lock);
201 		return;
202 	}
203 
204 	tcp_conn = conn->dd_data;
205 	tcp_sw_conn = tcp_conn->dd_data;
206 	old_write_space = tcp_sw_conn->old_write_space;
207 	read_unlock_bh(&sk->sk_callback_lock);
208 
209 	old_write_space(sk);
210 
211 	ISCSI_SW_TCP_DBG(conn, "iscsi_write_space\n");
212 	iscsi_conn_queue_work(conn);
213 }
214 
215 static void iscsi_sw_tcp_conn_set_callbacks(struct iscsi_conn *conn)
216 {
217 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
218 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
219 	struct sock *sk = tcp_sw_conn->sock->sk;
220 
221 	/* assign new callbacks */
222 	write_lock_bh(&sk->sk_callback_lock);
223 	sk->sk_user_data = conn;
224 	tcp_sw_conn->old_data_ready = sk->sk_data_ready;
225 	tcp_sw_conn->old_state_change = sk->sk_state_change;
226 	tcp_sw_conn->old_write_space = sk->sk_write_space;
227 	sk->sk_data_ready = iscsi_sw_tcp_data_ready;
228 	sk->sk_state_change = iscsi_sw_tcp_state_change;
229 	sk->sk_write_space = iscsi_sw_tcp_write_space;
230 	write_unlock_bh(&sk->sk_callback_lock);
231 }
232 
233 static void
234 iscsi_sw_tcp_conn_restore_callbacks(struct iscsi_conn *conn)
235 {
236 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
237 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
238 	struct sock *sk = tcp_sw_conn->sock->sk;
239 
240 	/* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
241 	write_lock_bh(&sk->sk_callback_lock);
242 	sk->sk_user_data    = NULL;
243 	sk->sk_data_ready   = tcp_sw_conn->old_data_ready;
244 	sk->sk_state_change = tcp_sw_conn->old_state_change;
245 	sk->sk_write_space  = tcp_sw_conn->old_write_space;
246 	sk->sk_no_check	 = 0;
247 	write_unlock_bh(&sk->sk_callback_lock);
248 }
249 
250 /**
251  * iscsi_sw_tcp_xmit_segment - transmit segment
252  * @tcp_conn: the iSCSI TCP connection
253  * @segment: the buffer to transmnit
254  *
255  * This function transmits as much of the buffer as
256  * the network layer will accept, and returns the number of
257  * bytes transmitted.
258  *
259  * If CRC hashing is enabled, the function will compute the
260  * hash as it goes. When the entire segment has been transmitted,
261  * it will retrieve the hash value and send it as well.
262  */
263 static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
264 				     struct iscsi_segment *segment)
265 {
266 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
267 	struct socket *sk = tcp_sw_conn->sock;
268 	unsigned int copied = 0;
269 	int r = 0;
270 
271 	while (!iscsi_tcp_segment_done(tcp_conn, segment, 0, r)) {
272 		struct scatterlist *sg;
273 		unsigned int offset, copy;
274 		int flags = 0;
275 
276 		r = 0;
277 		offset = segment->copied;
278 		copy = segment->size - offset;
279 
280 		if (segment->total_copied + segment->size < segment->total_size)
281 			flags |= MSG_MORE;
282 
283 		/* Use sendpage if we can; else fall back to sendmsg */
284 		if (!segment->data) {
285 			sg = segment->sg;
286 			offset += segment->sg_offset + sg->offset;
287 			r = tcp_sw_conn->sendpage(sk, sg_page(sg), offset,
288 						  copy, flags);
289 		} else {
290 			struct msghdr msg = { .msg_flags = flags };
291 			struct kvec iov = {
292 				.iov_base = segment->data + offset,
293 				.iov_len = copy
294 			};
295 
296 			r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
297 		}
298 
299 		if (r < 0) {
300 			iscsi_tcp_segment_unmap(segment);
301 			return r;
302 		}
303 		copied += r;
304 	}
305 	return copied;
306 }
307 
308 /**
309  * iscsi_sw_tcp_xmit - TCP transmit
310  **/
311 static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn)
312 {
313 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
314 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
315 	struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
316 	unsigned int consumed = 0;
317 	int rc = 0;
318 
319 	while (1) {
320 		rc = iscsi_sw_tcp_xmit_segment(tcp_conn, segment);
321 		/*
322 		 * We may not have been able to send data because the conn
323 		 * is getting stopped. libiscsi will know so propagate err
324 		 * for it to do the right thing.
325 		 */
326 		if (rc == -EAGAIN)
327 			return rc;
328 		else if (rc < 0) {
329 			rc = ISCSI_ERR_XMIT_FAILED;
330 			goto error;
331 		} else if (rc == 0)
332 			break;
333 
334 		consumed += rc;
335 
336 		if (segment->total_copied >= segment->total_size) {
337 			if (segment->done != NULL) {
338 				rc = segment->done(tcp_conn, segment);
339 				if (rc != 0)
340 					goto error;
341 			}
342 		}
343 	}
344 
345 	ISCSI_SW_TCP_DBG(conn, "xmit %d bytes\n", consumed);
346 
347 	conn->txdata_octets += consumed;
348 	return consumed;
349 
350 error:
351 	/* Transmit error. We could initiate error recovery
352 	 * here. */
353 	ISCSI_SW_TCP_DBG(conn, "Error sending PDU, errno=%d\n", rc);
354 	iscsi_conn_failure(conn, rc);
355 	return -EIO;
356 }
357 
358 /**
359  * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
360  */
361 static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn)
362 {
363 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
364 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
365 	struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
366 
367 	return segment->total_copied - segment->total_size;
368 }
369 
370 static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task)
371 {
372 	struct iscsi_conn *conn = task->conn;
373 	unsigned long pflags = current->flags;
374 	int rc = 0;
375 
376 	current->flags |= PF_MEMALLOC;
377 
378 	while (iscsi_sw_tcp_xmit_qlen(conn)) {
379 		rc = iscsi_sw_tcp_xmit(conn);
380 		if (rc == 0) {
381 			rc = -EAGAIN;
382 			break;
383 		}
384 		if (rc < 0)
385 			break;
386 		rc = 0;
387 	}
388 
389 	tsk_restore_flags(current, pflags, PF_MEMALLOC);
390 	return rc;
391 }
392 
393 /*
394  * This is called when we're done sending the header.
395  * Simply copy the data_segment to the send segment, and return.
396  */
397 static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
398 				      struct iscsi_segment *segment)
399 {
400 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
401 
402 	tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment;
403 	ISCSI_SW_TCP_DBG(tcp_conn->iscsi_conn,
404 			 "Header done. Next segment size %u total_size %u\n",
405 			 tcp_sw_conn->out.segment.size,
406 			 tcp_sw_conn->out.segment.total_size);
407 	return 0;
408 }
409 
410 static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr,
411 				       size_t hdrlen)
412 {
413 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
414 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
415 
416 	ISCSI_SW_TCP_DBG(conn, "%s\n", conn->hdrdgst_en ?
417 			 "digest enabled" : "digest disabled");
418 
419 	/* Clear the data segment - needs to be filled in by the
420 	 * caller using iscsi_tcp_send_data_prep() */
421 	memset(&tcp_sw_conn->out.data_segment, 0,
422 	       sizeof(struct iscsi_segment));
423 
424 	/* If header digest is enabled, compute the CRC and
425 	 * place the digest into the same buffer. We make
426 	 * sure that both iscsi_tcp_task and mtask have
427 	 * sufficient room.
428 	 */
429 	if (conn->hdrdgst_en) {
430 		iscsi_tcp_dgst_header(&tcp_sw_conn->tx_hash, hdr, hdrlen,
431 				      hdr + hdrlen);
432 		hdrlen += ISCSI_DIGEST_SIZE;
433 	}
434 
435 	/* Remember header pointer for later, when we need
436 	 * to decide whether there's a payload to go along
437 	 * with the header. */
438 	tcp_sw_conn->out.hdr = hdr;
439 
440 	iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen,
441 				  iscsi_sw_tcp_send_hdr_done, NULL);
442 }
443 
444 /*
445  * Prepare the send buffer for the payload data.
446  * Padding and checksumming will all be taken care
447  * of by the iscsi_segment routines.
448  */
449 static int
450 iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
451 			    unsigned int count, unsigned int offset,
452 			    unsigned int len)
453 {
454 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
455 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
456 	struct hash_desc *tx_hash = NULL;
457 	unsigned int hdr_spec_len;
458 
459 	ISCSI_SW_TCP_DBG(conn, "offset=%d, datalen=%d %s\n", offset, len,
460 			 conn->datadgst_en ?
461 			 "digest enabled" : "digest disabled");
462 
463 	/* Make sure the datalen matches what the caller
464 	   said he would send. */
465 	hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
466 	WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
467 
468 	if (conn->datadgst_en)
469 		tx_hash = &tcp_sw_conn->tx_hash;
470 
471 	return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment,
472 				     sg, count, offset, len,
473 				     NULL, tx_hash);
474 }
475 
476 static void
477 iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
478 				   size_t len)
479 {
480 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
481 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
482 	struct hash_desc *tx_hash = NULL;
483 	unsigned int hdr_spec_len;
484 
485 	ISCSI_SW_TCP_DBG(conn, "datalen=%zd %s\n", len, conn->datadgst_en ?
486 			 "digest enabled" : "digest disabled");
487 
488 	/* Make sure the datalen matches what the caller
489 	   said he would send. */
490 	hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
491 	WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
492 
493 	if (conn->datadgst_en)
494 		tx_hash = &tcp_sw_conn->tx_hash;
495 
496 	iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment,
497 				data, len, NULL, tx_hash);
498 }
499 
500 static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task,
501 				 unsigned int offset, unsigned int count)
502 {
503 	struct iscsi_conn *conn = task->conn;
504 	int err = 0;
505 
506 	iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
507 
508 	if (!count)
509 		return 0;
510 
511 	if (!task->sc)
512 		iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count);
513 	else {
514 		struct scsi_data_buffer *sdb = scsi_out(task->sc);
515 
516 		err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl,
517 						  sdb->table.nents, offset,
518 						  count);
519 	}
520 
521 	if (err) {
522 		/* got invalid offset/len */
523 		return -EIO;
524 	}
525 	return 0;
526 }
527 
528 static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
529 {
530 	struct iscsi_tcp_task *tcp_task = task->dd_data;
531 
532 	task->hdr = task->dd_data + sizeof(*tcp_task);
533 	task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE;
534 	return 0;
535 }
536 
537 static struct iscsi_cls_conn *
538 iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
539 			 uint32_t conn_idx)
540 {
541 	struct iscsi_conn *conn;
542 	struct iscsi_cls_conn *cls_conn;
543 	struct iscsi_tcp_conn *tcp_conn;
544 	struct iscsi_sw_tcp_conn *tcp_sw_conn;
545 
546 	cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn),
547 					conn_idx);
548 	if (!cls_conn)
549 		return NULL;
550 	conn = cls_conn->dd_data;
551 	tcp_conn = conn->dd_data;
552 	tcp_sw_conn = tcp_conn->dd_data;
553 
554 	tcp_sw_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
555 						     CRYPTO_ALG_ASYNC);
556 	tcp_sw_conn->tx_hash.flags = 0;
557 	if (IS_ERR(tcp_sw_conn->tx_hash.tfm))
558 		goto free_conn;
559 
560 	tcp_sw_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
561 						     CRYPTO_ALG_ASYNC);
562 	tcp_sw_conn->rx_hash.flags = 0;
563 	if (IS_ERR(tcp_sw_conn->rx_hash.tfm))
564 		goto free_tx_tfm;
565 	tcp_conn->rx_hash = &tcp_sw_conn->rx_hash;
566 
567 	return cls_conn;
568 
569 free_tx_tfm:
570 	crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
571 free_conn:
572 	iscsi_conn_printk(KERN_ERR, conn,
573 			  "Could not create connection due to crc32c "
574 			  "loading error. Make sure the crc32c "
575 			  "module is built as a module or into the "
576 			  "kernel\n");
577 	iscsi_tcp_conn_teardown(cls_conn);
578 	return NULL;
579 }
580 
581 static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
582 {
583 	struct iscsi_session *session = conn->session;
584 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
585 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
586 	struct socket *sock = tcp_sw_conn->sock;
587 
588 	if (!sock)
589 		return;
590 
591 	sock_hold(sock->sk);
592 	iscsi_sw_tcp_conn_restore_callbacks(conn);
593 	sock_put(sock->sk);
594 
595 	spin_lock_bh(&session->lock);
596 	tcp_sw_conn->sock = NULL;
597 	spin_unlock_bh(&session->lock);
598 	sockfd_put(sock);
599 }
600 
601 static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
602 {
603 	struct iscsi_conn *conn = cls_conn->dd_data;
604 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
605 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
606 
607 	iscsi_sw_tcp_release_conn(conn);
608 
609 	if (tcp_sw_conn->tx_hash.tfm)
610 		crypto_free_hash(tcp_sw_conn->tx_hash.tfm);
611 	if (tcp_sw_conn->rx_hash.tfm)
612 		crypto_free_hash(tcp_sw_conn->rx_hash.tfm);
613 
614 	iscsi_tcp_conn_teardown(cls_conn);
615 }
616 
617 static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
618 {
619 	struct iscsi_conn *conn = cls_conn->dd_data;
620 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
621 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
622 	struct socket *sock = tcp_sw_conn->sock;
623 
624 	/* userspace may have goofed up and not bound us */
625 	if (!sock)
626 		return;
627 
628 	sock->sk->sk_err = EIO;
629 	wake_up_interruptible(sk_sleep(sock->sk));
630 
631 	/* stop xmit side */
632 	iscsi_suspend_tx(conn);
633 
634 	/* stop recv side and release socket */
635 	iscsi_sw_tcp_release_conn(conn);
636 
637 	iscsi_conn_stop(cls_conn, flag);
638 }
639 
640 static int
641 iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session,
642 		       struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
643 		       int is_leading)
644 {
645 	struct iscsi_session *session = cls_session->dd_data;
646 	struct iscsi_conn *conn = cls_conn->dd_data;
647 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
648 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
649 	struct sock *sk;
650 	struct socket *sock;
651 	int err;
652 
653 	/* lookup for existing socket */
654 	sock = sockfd_lookup((int)transport_eph, &err);
655 	if (!sock) {
656 		iscsi_conn_printk(KERN_ERR, conn,
657 				  "sockfd_lookup failed %d\n", err);
658 		return -EEXIST;
659 	}
660 
661 	err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
662 	if (err)
663 		goto free_socket;
664 
665 	spin_lock_bh(&session->lock);
666 	/* bind iSCSI connection and socket */
667 	tcp_sw_conn->sock = sock;
668 	spin_unlock_bh(&session->lock);
669 
670 	/* setup Socket parameters */
671 	sk = sock->sk;
672 	sk->sk_reuse = SK_CAN_REUSE;
673 	sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
674 	sk->sk_allocation = GFP_ATOMIC;
675 	sk_set_memalloc(sk);
676 
677 	iscsi_sw_tcp_conn_set_callbacks(conn);
678 	tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage;
679 	/*
680 	 * set receive state machine into initial state
681 	 */
682 	iscsi_tcp_hdr_recv_prep(tcp_conn);
683 	return 0;
684 
685 free_socket:
686 	sockfd_put(sock);
687 	return err;
688 }
689 
690 static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
691 				       enum iscsi_param param, char *buf,
692 				       int buflen)
693 {
694 	struct iscsi_conn *conn = cls_conn->dd_data;
695 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
696 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
697 
698 	switch(param) {
699 	case ISCSI_PARAM_HDRDGST_EN:
700 		iscsi_set_param(cls_conn, param, buf, buflen);
701 		break;
702 	case ISCSI_PARAM_DATADGST_EN:
703 		iscsi_set_param(cls_conn, param, buf, buflen);
704 		tcp_sw_conn->sendpage = conn->datadgst_en ?
705 			sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage;
706 		break;
707 	case ISCSI_PARAM_MAX_R2T:
708 		return iscsi_tcp_set_max_r2t(conn, buf);
709 	default:
710 		return iscsi_set_param(cls_conn, param, buf, buflen);
711 	}
712 
713 	return 0;
714 }
715 
716 static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
717 				       enum iscsi_param param, char *buf)
718 {
719 	struct iscsi_conn *conn = cls_conn->dd_data;
720 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
721 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
722 	struct sockaddr_in6 addr;
723 	int rc, len;
724 
725 	switch(param) {
726 	case ISCSI_PARAM_CONN_PORT:
727 	case ISCSI_PARAM_CONN_ADDRESS:
728 		spin_lock_bh(&conn->session->lock);
729 		if (!tcp_sw_conn || !tcp_sw_conn->sock) {
730 			spin_unlock_bh(&conn->session->lock);
731 			return -ENOTCONN;
732 		}
733 		rc = kernel_getpeername(tcp_sw_conn->sock,
734 					(struct sockaddr *)&addr, &len);
735 		spin_unlock_bh(&conn->session->lock);
736 		if (rc)
737 			return rc;
738 
739 		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
740 						 &addr, param, buf);
741 	default:
742 		return iscsi_conn_get_param(cls_conn, param, buf);
743 	}
744 
745 	return 0;
746 }
747 
748 static int iscsi_sw_tcp_host_get_param(struct Scsi_Host *shost,
749 				       enum iscsi_host_param param, char *buf)
750 {
751 	struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(shost);
752 	struct iscsi_session *session = tcp_sw_host->session;
753 	struct iscsi_conn *conn;
754 	struct iscsi_tcp_conn *tcp_conn;
755 	struct iscsi_sw_tcp_conn *tcp_sw_conn;
756 	struct sockaddr_in6 addr;
757 	int rc, len;
758 
759 	switch (param) {
760 	case ISCSI_HOST_PARAM_IPADDRESS:
761 		spin_lock_bh(&session->lock);
762 		conn = session->leadconn;
763 		if (!conn) {
764 			spin_unlock_bh(&session->lock);
765 			return -ENOTCONN;
766 		}
767 		tcp_conn = conn->dd_data;
768 
769 		tcp_sw_conn = tcp_conn->dd_data;
770 		if (!tcp_sw_conn->sock) {
771 			spin_unlock_bh(&session->lock);
772 			return -ENOTCONN;
773 		}
774 
775 		rc = kernel_getsockname(tcp_sw_conn->sock,
776 					(struct sockaddr *)&addr, &len);
777 		spin_unlock_bh(&session->lock);
778 		if (rc)
779 			return rc;
780 
781 		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
782 						 &addr, param, buf);
783 	default:
784 		return iscsi_host_get_param(shost, param, buf);
785 	}
786 
787 	return 0;
788 }
789 
790 static void
791 iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,
792 			    struct iscsi_stats *stats)
793 {
794 	struct iscsi_conn *conn = cls_conn->dd_data;
795 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
796 	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
797 
798 	stats->custom_length = 3;
799 	strcpy(stats->custom[0].desc, "tx_sendpage_failures");
800 	stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt;
801 	strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
802 	stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt;
803 	strcpy(stats->custom[2].desc, "eh_abort_cnt");
804 	stats->custom[2].value = conn->eh_abort_cnt;
805 
806 	iscsi_tcp_conn_get_stats(cls_conn, stats);
807 }
808 
809 static struct iscsi_cls_session *
810 iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
811 			    uint16_t qdepth, uint32_t initial_cmdsn)
812 {
813 	struct iscsi_cls_session *cls_session;
814 	struct iscsi_session *session;
815 	struct iscsi_sw_tcp_host *tcp_sw_host;
816 	struct Scsi_Host *shost;
817 
818 	if (ep) {
819 		printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep);
820 		return NULL;
821 	}
822 
823 	shost = iscsi_host_alloc(&iscsi_sw_tcp_sht,
824 				 sizeof(struct iscsi_sw_tcp_host), 1);
825 	if (!shost)
826 		return NULL;
827 	shost->transportt = iscsi_sw_tcp_scsi_transport;
828 	shost->cmd_per_lun = qdepth;
829 	shost->max_lun = iscsi_max_lun;
830 	shost->max_id = 0;
831 	shost->max_channel = 0;
832 	shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
833 
834 	if (iscsi_host_add(shost, NULL))
835 		goto free_host;
836 
837 	cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost,
838 					  cmds_max, 0,
839 					  sizeof(struct iscsi_tcp_task) +
840 					  sizeof(struct iscsi_sw_tcp_hdrbuf),
841 					  initial_cmdsn, 0);
842 	if (!cls_session)
843 		goto remove_host;
844 	session = cls_session->dd_data;
845 	tcp_sw_host = iscsi_host_priv(shost);
846 	tcp_sw_host->session = session;
847 
848 	shost->can_queue = session->scsi_cmds_max;
849 	if (iscsi_tcp_r2tpool_alloc(session))
850 		goto remove_session;
851 	return cls_session;
852 
853 remove_session:
854 	iscsi_session_teardown(cls_session);
855 remove_host:
856 	iscsi_host_remove(shost);
857 free_host:
858 	iscsi_host_free(shost);
859 	return NULL;
860 }
861 
862 static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session)
863 {
864 	struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
865 
866 	iscsi_tcp_r2tpool_free(cls_session->dd_data);
867 	iscsi_session_teardown(cls_session);
868 
869 	iscsi_host_remove(shost);
870 	iscsi_host_free(shost);
871 }
872 
873 static umode_t iscsi_sw_tcp_attr_is_visible(int param_type, int param)
874 {
875 	switch (param_type) {
876 	case ISCSI_HOST_PARAM:
877 		switch (param) {
878 		case ISCSI_HOST_PARAM_NETDEV_NAME:
879 		case ISCSI_HOST_PARAM_HWADDRESS:
880 		case ISCSI_HOST_PARAM_IPADDRESS:
881 		case ISCSI_HOST_PARAM_INITIATOR_NAME:
882 			return S_IRUGO;
883 		default:
884 			return 0;
885 		}
886 	case ISCSI_PARAM:
887 		switch (param) {
888 		case ISCSI_PARAM_MAX_RECV_DLENGTH:
889 		case ISCSI_PARAM_MAX_XMIT_DLENGTH:
890 		case ISCSI_PARAM_HDRDGST_EN:
891 		case ISCSI_PARAM_DATADGST_EN:
892 		case ISCSI_PARAM_CONN_ADDRESS:
893 		case ISCSI_PARAM_CONN_PORT:
894 		case ISCSI_PARAM_EXP_STATSN:
895 		case ISCSI_PARAM_PERSISTENT_ADDRESS:
896 		case ISCSI_PARAM_PERSISTENT_PORT:
897 		case ISCSI_PARAM_PING_TMO:
898 		case ISCSI_PARAM_RECV_TMO:
899 		case ISCSI_PARAM_INITIAL_R2T_EN:
900 		case ISCSI_PARAM_MAX_R2T:
901 		case ISCSI_PARAM_IMM_DATA_EN:
902 		case ISCSI_PARAM_FIRST_BURST:
903 		case ISCSI_PARAM_MAX_BURST:
904 		case ISCSI_PARAM_PDU_INORDER_EN:
905 		case ISCSI_PARAM_DATASEQ_INORDER_EN:
906 		case ISCSI_PARAM_ERL:
907 		case ISCSI_PARAM_TARGET_NAME:
908 		case ISCSI_PARAM_TPGT:
909 		case ISCSI_PARAM_USERNAME:
910 		case ISCSI_PARAM_PASSWORD:
911 		case ISCSI_PARAM_USERNAME_IN:
912 		case ISCSI_PARAM_PASSWORD_IN:
913 		case ISCSI_PARAM_FAST_ABORT:
914 		case ISCSI_PARAM_ABORT_TMO:
915 		case ISCSI_PARAM_LU_RESET_TMO:
916 		case ISCSI_PARAM_TGT_RESET_TMO:
917 		case ISCSI_PARAM_IFACE_NAME:
918 		case ISCSI_PARAM_INITIATOR_NAME:
919 			return S_IRUGO;
920 		default:
921 			return 0;
922 		}
923 	}
924 
925 	return 0;
926 }
927 
928 static int iscsi_sw_tcp_slave_alloc(struct scsi_device *sdev)
929 {
930 	set_bit(QUEUE_FLAG_BIDI, &sdev->request_queue->queue_flags);
931 	return 0;
932 }
933 
934 static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
935 {
936 	blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
937 	blk_queue_dma_alignment(sdev->request_queue, 0);
938 	return 0;
939 }
940 
941 static struct scsi_host_template iscsi_sw_tcp_sht = {
942 	.module			= THIS_MODULE,
943 	.name			= "iSCSI Initiator over TCP/IP",
944 	.queuecommand           = iscsi_queuecommand,
945 	.change_queue_depth	= iscsi_change_queue_depth,
946 	.can_queue		= ISCSI_DEF_XMIT_CMDS_MAX - 1,
947 	.sg_tablesize		= 4096,
948 	.max_sectors		= 0xFFFF,
949 	.cmd_per_lun		= ISCSI_DEF_CMD_PER_LUN,
950 	.eh_abort_handler       = iscsi_eh_abort,
951 	.eh_device_reset_handler= iscsi_eh_device_reset,
952 	.eh_target_reset_handler = iscsi_eh_recover_target,
953 	.use_clustering         = DISABLE_CLUSTERING,
954 	.slave_alloc            = iscsi_sw_tcp_slave_alloc,
955 	.slave_configure        = iscsi_sw_tcp_slave_configure,
956 	.target_alloc		= iscsi_target_alloc,
957 	.proc_name		= "iscsi_tcp",
958 	.this_id		= -1,
959 };
960 
961 static struct iscsi_transport iscsi_sw_tcp_transport = {
962 	.owner			= THIS_MODULE,
963 	.name			= "tcp",
964 	.caps			= CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
965 				  | CAP_DATADGST,
966 	/* session management */
967 	.create_session		= iscsi_sw_tcp_session_create,
968 	.destroy_session	= iscsi_sw_tcp_session_destroy,
969 	/* connection management */
970 	.create_conn		= iscsi_sw_tcp_conn_create,
971 	.bind_conn		= iscsi_sw_tcp_conn_bind,
972 	.destroy_conn		= iscsi_sw_tcp_conn_destroy,
973 	.attr_is_visible	= iscsi_sw_tcp_attr_is_visible,
974 	.set_param		= iscsi_sw_tcp_conn_set_param,
975 	.get_conn_param		= iscsi_sw_tcp_conn_get_param,
976 	.get_session_param	= iscsi_session_get_param,
977 	.start_conn		= iscsi_conn_start,
978 	.stop_conn		= iscsi_sw_tcp_conn_stop,
979 	/* iscsi host params */
980 	.get_host_param		= iscsi_sw_tcp_host_get_param,
981 	.set_host_param		= iscsi_host_set_param,
982 	/* IO */
983 	.send_pdu		= iscsi_conn_send_pdu,
984 	.get_stats		= iscsi_sw_tcp_conn_get_stats,
985 	/* iscsi task/cmd helpers */
986 	.init_task		= iscsi_tcp_task_init,
987 	.xmit_task		= iscsi_tcp_task_xmit,
988 	.cleanup_task		= iscsi_tcp_cleanup_task,
989 	/* low level pdu helpers */
990 	.xmit_pdu		= iscsi_sw_tcp_pdu_xmit,
991 	.init_pdu		= iscsi_sw_tcp_pdu_init,
992 	.alloc_pdu		= iscsi_sw_tcp_pdu_alloc,
993 	/* recovery */
994 	.session_recovery_timedout = iscsi_session_recovery_timedout,
995 };
996 
997 static int __init iscsi_sw_tcp_init(void)
998 {
999 	if (iscsi_max_lun < 1) {
1000 		printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
1001 		       iscsi_max_lun);
1002 		return -EINVAL;
1003 	}
1004 
1005 	iscsi_sw_tcp_scsi_transport = iscsi_register_transport(
1006 						&iscsi_sw_tcp_transport);
1007 	if (!iscsi_sw_tcp_scsi_transport)
1008 		return -ENODEV;
1009 
1010 	return 0;
1011 }
1012 
1013 static void __exit iscsi_sw_tcp_exit(void)
1014 {
1015 	iscsi_unregister_transport(&iscsi_sw_tcp_transport);
1016 }
1017 
1018 module_init(iscsi_sw_tcp_init);
1019 module_exit(iscsi_sw_tcp_exit);
1020