1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * iSCSI Initiator over TCP/IP Data-Path 4 * 5 * Copyright (C) 2004 Dmitry Yusupov 6 * Copyright (C) 2004 Alex Aizman 7 * Copyright (C) 2005 - 2006 Mike Christie 8 * Copyright (C) 2006 Red Hat, Inc. All rights reserved. 9 * maintained by open-iscsi@googlegroups.com 10 * 11 * See the file COPYING included with this distribution for more details. 12 * 13 * Credits: 14 * Christoph Hellwig 15 * FUJITA Tomonori 16 * Arne Redlich 17 * Zhenyu Wang 18 */ 19 20 #include <crypto/hash.h> 21 #include <linux/types.h> 22 #include <linux/inet.h> 23 #include <linux/slab.h> 24 #include <linux/sched/mm.h> 25 #include <linux/file.h> 26 #include <linux/blkdev.h> 27 #include <linux/delay.h> 28 #include <linux/kfifo.h> 29 #include <linux/scatterlist.h> 30 #include <linux/module.h> 31 #include <linux/backing-dev.h> 32 #include <net/tcp.h> 33 #include <scsi/scsi_cmnd.h> 34 #include <scsi/scsi_device.h> 35 #include <scsi/scsi_host.h> 36 #include <scsi/scsi.h> 37 #include <scsi/scsi_transport_iscsi.h> 38 #include <trace/events/iscsi.h> 39 40 #include "iscsi_tcp.h" 41 42 MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, " 43 "Dmitry Yusupov <dmitry_yus@yahoo.com>, " 44 "Alex Aizman <itn780@yahoo.com>"); 45 MODULE_DESCRIPTION("iSCSI/TCP data-path"); 46 MODULE_LICENSE("GPL"); 47 48 static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport; 49 static struct scsi_host_template iscsi_sw_tcp_sht; 50 static struct iscsi_transport iscsi_sw_tcp_transport; 51 52 static unsigned int iscsi_max_lun = ~0; 53 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO); 54 55 static bool iscsi_recv_from_iscsi_q; 56 module_param_named(recv_from_iscsi_q, iscsi_recv_from_iscsi_q, bool, 0644); 57 MODULE_PARM_DESC(recv_from_iscsi_q, "Set to true to read iSCSI data/headers from the iscsi_q workqueue. The default is false which will perform reads from the network softirq context."); 58 59 static int iscsi_sw_tcp_dbg; 60 module_param_named(debug_iscsi_tcp, iscsi_sw_tcp_dbg, int, 61 S_IRUGO | S_IWUSR); 62 MODULE_PARM_DESC(debug_iscsi_tcp, "Turn on debugging for iscsi_tcp module " 63 "Set to 1 to turn on, and zero to turn off. Default is off."); 64 65 #define ISCSI_SW_TCP_DBG(_conn, dbg_fmt, arg...) \ 66 do { \ 67 if (iscsi_sw_tcp_dbg) \ 68 iscsi_conn_printk(KERN_INFO, _conn, \ 69 "%s " dbg_fmt, \ 70 __func__, ##arg); \ 71 iscsi_dbg_trace(trace_iscsi_dbg_sw_tcp, \ 72 &(_conn)->cls_conn->dev, \ 73 "%s " dbg_fmt, __func__, ##arg);\ 74 } while (0); 75 76 77 /** 78 * iscsi_sw_tcp_recv - TCP receive in sendfile fashion 79 * @rd_desc: read descriptor 80 * @skb: socket buffer 81 * @offset: offset in skb 82 * @len: skb->len - offset 83 */ 84 static int iscsi_sw_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb, 85 unsigned int offset, size_t len) 86 { 87 struct iscsi_conn *conn = rd_desc->arg.data; 88 unsigned int consumed, total_consumed = 0; 89 int status; 90 91 ISCSI_SW_TCP_DBG(conn, "in %d bytes\n", skb->len - offset); 92 93 do { 94 status = 0; 95 consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status); 96 offset += consumed; 97 total_consumed += consumed; 98 } while (consumed != 0 && status != ISCSI_TCP_SKB_DONE); 99 100 ISCSI_SW_TCP_DBG(conn, "read %d bytes status %d\n", 101 skb->len - offset, status); 102 return total_consumed; 103 } 104 105 /** 106 * iscsi_sw_sk_state_check - check socket state 107 * @sk: socket 108 * 109 * If the socket is in CLOSE or CLOSE_WAIT we should 110 * not close the connection if there is still some 111 * data pending. 112 * 113 * Must be called with sk_callback_lock. 114 */ 115 static inline int iscsi_sw_sk_state_check(struct sock *sk) 116 { 117 struct iscsi_conn *conn = sk->sk_user_data; 118 119 if ((sk->sk_state == TCP_CLOSE_WAIT || sk->sk_state == TCP_CLOSE) && 120 (conn->session->state != ISCSI_STATE_LOGGING_OUT) && 121 !atomic_read(&sk->sk_rmem_alloc)) { 122 ISCSI_SW_TCP_DBG(conn, "TCP_CLOSE|TCP_CLOSE_WAIT\n"); 123 iscsi_conn_failure(conn, ISCSI_ERR_TCP_CONN_CLOSE); 124 return -ECONNRESET; 125 } 126 return 0; 127 } 128 129 static void iscsi_sw_tcp_recv_data(struct iscsi_conn *conn) 130 { 131 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 132 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 133 struct sock *sk = tcp_sw_conn->sock->sk; 134 read_descriptor_t rd_desc; 135 136 /* 137 * Use rd_desc to pass 'conn' to iscsi_tcp_recv. 138 * We set count to 1 because we want the network layer to 139 * hand us all the skbs that are available. iscsi_tcp_recv 140 * handled pdus that cross buffers or pdus that still need data. 141 */ 142 rd_desc.arg.data = conn; 143 rd_desc.count = 1; 144 145 tcp_read_sock(sk, &rd_desc, iscsi_sw_tcp_recv); 146 147 /* If we had to (atomically) map a highmem page, 148 * unmap it now. */ 149 iscsi_tcp_segment_unmap(&tcp_conn->in.segment); 150 151 iscsi_sw_sk_state_check(sk); 152 } 153 154 static void iscsi_sw_tcp_recv_data_work(struct work_struct *work) 155 { 156 struct iscsi_conn *conn = container_of(work, struct iscsi_conn, 157 recvwork); 158 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 159 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 160 struct sock *sk = tcp_sw_conn->sock->sk; 161 162 lock_sock(sk); 163 iscsi_sw_tcp_recv_data(conn); 164 release_sock(sk); 165 } 166 167 static void iscsi_sw_tcp_data_ready(struct sock *sk) 168 { 169 struct iscsi_sw_tcp_conn *tcp_sw_conn; 170 struct iscsi_tcp_conn *tcp_conn; 171 struct iscsi_conn *conn; 172 173 read_lock_bh(&sk->sk_callback_lock); 174 conn = sk->sk_user_data; 175 if (!conn) { 176 read_unlock_bh(&sk->sk_callback_lock); 177 return; 178 } 179 tcp_conn = conn->dd_data; 180 tcp_sw_conn = tcp_conn->dd_data; 181 182 if (tcp_sw_conn->queue_recv) 183 iscsi_conn_queue_recv(conn); 184 else 185 iscsi_sw_tcp_recv_data(conn); 186 read_unlock_bh(&sk->sk_callback_lock); 187 } 188 189 static void iscsi_sw_tcp_state_change(struct sock *sk) 190 { 191 struct iscsi_tcp_conn *tcp_conn; 192 struct iscsi_sw_tcp_conn *tcp_sw_conn; 193 struct iscsi_conn *conn; 194 void (*old_state_change)(struct sock *); 195 196 read_lock_bh(&sk->sk_callback_lock); 197 conn = sk->sk_user_data; 198 if (!conn) { 199 read_unlock_bh(&sk->sk_callback_lock); 200 return; 201 } 202 203 iscsi_sw_sk_state_check(sk); 204 205 tcp_conn = conn->dd_data; 206 tcp_sw_conn = tcp_conn->dd_data; 207 old_state_change = tcp_sw_conn->old_state_change; 208 209 read_unlock_bh(&sk->sk_callback_lock); 210 211 old_state_change(sk); 212 } 213 214 /** 215 * iscsi_sw_tcp_write_space - Called when more output buffer space is available 216 * @sk: socket space is available for 217 **/ 218 static void iscsi_sw_tcp_write_space(struct sock *sk) 219 { 220 struct iscsi_conn *conn; 221 struct iscsi_tcp_conn *tcp_conn; 222 struct iscsi_sw_tcp_conn *tcp_sw_conn; 223 void (*old_write_space)(struct sock *); 224 225 read_lock_bh(&sk->sk_callback_lock); 226 conn = sk->sk_user_data; 227 if (!conn) { 228 read_unlock_bh(&sk->sk_callback_lock); 229 return; 230 } 231 232 tcp_conn = conn->dd_data; 233 tcp_sw_conn = tcp_conn->dd_data; 234 old_write_space = tcp_sw_conn->old_write_space; 235 read_unlock_bh(&sk->sk_callback_lock); 236 237 old_write_space(sk); 238 239 ISCSI_SW_TCP_DBG(conn, "iscsi_write_space\n"); 240 iscsi_conn_queue_xmit(conn); 241 } 242 243 static void iscsi_sw_tcp_conn_set_callbacks(struct iscsi_conn *conn) 244 { 245 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 246 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 247 struct sock *sk = tcp_sw_conn->sock->sk; 248 249 /* assign new callbacks */ 250 write_lock_bh(&sk->sk_callback_lock); 251 sk->sk_user_data = conn; 252 tcp_sw_conn->old_data_ready = sk->sk_data_ready; 253 tcp_sw_conn->old_state_change = sk->sk_state_change; 254 tcp_sw_conn->old_write_space = sk->sk_write_space; 255 sk->sk_data_ready = iscsi_sw_tcp_data_ready; 256 sk->sk_state_change = iscsi_sw_tcp_state_change; 257 sk->sk_write_space = iscsi_sw_tcp_write_space; 258 write_unlock_bh(&sk->sk_callback_lock); 259 } 260 261 static void 262 iscsi_sw_tcp_conn_restore_callbacks(struct iscsi_conn *conn) 263 { 264 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 265 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 266 struct sock *sk = tcp_sw_conn->sock->sk; 267 268 /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */ 269 write_lock_bh(&sk->sk_callback_lock); 270 sk->sk_user_data = NULL; 271 sk->sk_data_ready = tcp_sw_conn->old_data_ready; 272 sk->sk_state_change = tcp_sw_conn->old_state_change; 273 sk->sk_write_space = tcp_sw_conn->old_write_space; 274 sk->sk_no_check_tx = 0; 275 write_unlock_bh(&sk->sk_callback_lock); 276 } 277 278 /** 279 * iscsi_sw_tcp_xmit_segment - transmit segment 280 * @tcp_conn: the iSCSI TCP connection 281 * @segment: the buffer to transmnit 282 * 283 * This function transmits as much of the buffer as 284 * the network layer will accept, and returns the number of 285 * bytes transmitted. 286 * 287 * If CRC hashing is enabled, the function will compute the 288 * hash as it goes. When the entire segment has been transmitted, 289 * it will retrieve the hash value and send it as well. 290 */ 291 static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn, 292 struct iscsi_segment *segment) 293 { 294 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 295 struct socket *sk = tcp_sw_conn->sock; 296 unsigned int copied = 0; 297 int r = 0; 298 299 while (!iscsi_tcp_segment_done(tcp_conn, segment, 0, r)) { 300 struct scatterlist *sg; 301 unsigned int offset, copy; 302 int flags = 0; 303 304 r = 0; 305 offset = segment->copied; 306 copy = segment->size - offset; 307 308 if (segment->total_copied + segment->size < segment->total_size) 309 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 310 311 if (tcp_sw_conn->queue_recv) 312 flags |= MSG_DONTWAIT; 313 314 /* Use sendpage if we can; else fall back to sendmsg */ 315 if (!segment->data) { 316 sg = segment->sg; 317 offset += segment->sg_offset + sg->offset; 318 r = tcp_sw_conn->sendpage(sk, sg_page(sg), offset, 319 copy, flags); 320 } else { 321 struct msghdr msg = { .msg_flags = flags }; 322 struct kvec iov = { 323 .iov_base = segment->data + offset, 324 .iov_len = copy 325 }; 326 327 r = kernel_sendmsg(sk, &msg, &iov, 1, copy); 328 } 329 330 if (r < 0) { 331 iscsi_tcp_segment_unmap(segment); 332 return r; 333 } 334 copied += r; 335 } 336 return copied; 337 } 338 339 /** 340 * iscsi_sw_tcp_xmit - TCP transmit 341 * @conn: iscsi connection 342 **/ 343 static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn) 344 { 345 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 346 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 347 struct iscsi_segment *segment = &tcp_sw_conn->out.segment; 348 unsigned int consumed = 0; 349 int rc = 0; 350 351 while (1) { 352 rc = iscsi_sw_tcp_xmit_segment(tcp_conn, segment); 353 /* 354 * We may not have been able to send data because the conn 355 * is getting stopped. libiscsi will know so propagate err 356 * for it to do the right thing. 357 */ 358 if (rc == -EAGAIN) 359 return rc; 360 else if (rc < 0) { 361 rc = ISCSI_ERR_XMIT_FAILED; 362 goto error; 363 } else if (rc == 0) 364 break; 365 366 consumed += rc; 367 368 if (segment->total_copied >= segment->total_size) { 369 if (segment->done != NULL) { 370 rc = segment->done(tcp_conn, segment); 371 if (rc != 0) 372 goto error; 373 } 374 } 375 } 376 377 ISCSI_SW_TCP_DBG(conn, "xmit %d bytes\n", consumed); 378 379 conn->txdata_octets += consumed; 380 return consumed; 381 382 error: 383 /* Transmit error. We could initiate error recovery 384 * here. */ 385 ISCSI_SW_TCP_DBG(conn, "Error sending PDU, errno=%d\n", rc); 386 iscsi_conn_failure(conn, rc); 387 return -EIO; 388 } 389 390 /** 391 * iscsi_sw_tcp_xmit_qlen - return the number of bytes queued for xmit 392 * @conn: iscsi connection 393 */ 394 static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn) 395 { 396 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 397 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 398 struct iscsi_segment *segment = &tcp_sw_conn->out.segment; 399 400 return segment->total_copied - segment->total_size; 401 } 402 403 static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task) 404 { 405 struct iscsi_conn *conn = task->conn; 406 unsigned int noreclaim_flag; 407 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 408 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 409 int rc = 0; 410 411 if (!tcp_sw_conn->sock) { 412 iscsi_conn_printk(KERN_ERR, conn, 413 "Transport not bound to socket!\n"); 414 return -EINVAL; 415 } 416 417 noreclaim_flag = memalloc_noreclaim_save(); 418 419 while (iscsi_sw_tcp_xmit_qlen(conn)) { 420 rc = iscsi_sw_tcp_xmit(conn); 421 if (rc == 0) { 422 rc = -EAGAIN; 423 break; 424 } 425 if (rc < 0) 426 break; 427 rc = 0; 428 } 429 430 memalloc_noreclaim_restore(noreclaim_flag); 431 return rc; 432 } 433 434 /* 435 * This is called when we're done sending the header. 436 * Simply copy the data_segment to the send segment, and return. 437 */ 438 static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn, 439 struct iscsi_segment *segment) 440 { 441 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 442 443 tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment; 444 ISCSI_SW_TCP_DBG(tcp_conn->iscsi_conn, 445 "Header done. Next segment size %u total_size %u\n", 446 tcp_sw_conn->out.segment.size, 447 tcp_sw_conn->out.segment.total_size); 448 return 0; 449 } 450 451 static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, 452 size_t hdrlen) 453 { 454 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 455 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 456 457 ISCSI_SW_TCP_DBG(conn, "%s\n", conn->hdrdgst_en ? 458 "digest enabled" : "digest disabled"); 459 460 /* Clear the data segment - needs to be filled in by the 461 * caller using iscsi_tcp_send_data_prep() */ 462 memset(&tcp_sw_conn->out.data_segment, 0, 463 sizeof(struct iscsi_segment)); 464 465 /* If header digest is enabled, compute the CRC and 466 * place the digest into the same buffer. We make 467 * sure that both iscsi_tcp_task and mtask have 468 * sufficient room. 469 */ 470 if (conn->hdrdgst_en) { 471 iscsi_tcp_dgst_header(tcp_sw_conn->tx_hash, hdr, hdrlen, 472 hdr + hdrlen); 473 hdrlen += ISCSI_DIGEST_SIZE; 474 } 475 476 /* Remember header pointer for later, when we need 477 * to decide whether there's a payload to go along 478 * with the header. */ 479 tcp_sw_conn->out.hdr = hdr; 480 481 iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen, 482 iscsi_sw_tcp_send_hdr_done, NULL); 483 } 484 485 /* 486 * Prepare the send buffer for the payload data. 487 * Padding and checksumming will all be taken care 488 * of by the iscsi_segment routines. 489 */ 490 static int 491 iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg, 492 unsigned int count, unsigned int offset, 493 unsigned int len) 494 { 495 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 496 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 497 struct ahash_request *tx_hash = NULL; 498 unsigned int hdr_spec_len; 499 500 ISCSI_SW_TCP_DBG(conn, "offset=%d, datalen=%d %s\n", offset, len, 501 conn->datadgst_en ? 502 "digest enabled" : "digest disabled"); 503 504 /* Make sure the datalen matches what the caller 505 said he would send. */ 506 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength); 507 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 508 509 if (conn->datadgst_en) 510 tx_hash = tcp_sw_conn->tx_hash; 511 512 return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment, 513 sg, count, offset, len, 514 NULL, tx_hash); 515 } 516 517 static void 518 iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data, 519 size_t len) 520 { 521 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 522 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 523 struct ahash_request *tx_hash = NULL; 524 unsigned int hdr_spec_len; 525 526 ISCSI_SW_TCP_DBG(conn, "datalen=%zd %s\n", len, conn->datadgst_en ? 527 "digest enabled" : "digest disabled"); 528 529 /* Make sure the datalen matches what the caller 530 said he would send. */ 531 hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength); 532 WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len)); 533 534 if (conn->datadgst_en) 535 tx_hash = tcp_sw_conn->tx_hash; 536 537 iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment, 538 data, len, NULL, tx_hash); 539 } 540 541 static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task, 542 unsigned int offset, unsigned int count) 543 { 544 struct iscsi_conn *conn = task->conn; 545 int err = 0; 546 547 iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len); 548 549 if (!count) 550 return 0; 551 552 if (!task->sc) 553 iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count); 554 else { 555 struct scsi_data_buffer *sdb = &task->sc->sdb; 556 557 err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl, 558 sdb->table.nents, offset, 559 count); 560 } 561 562 if (err) { 563 /* got invalid offset/len */ 564 return -EIO; 565 } 566 return 0; 567 } 568 569 static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task, uint8_t opcode) 570 { 571 struct iscsi_tcp_task *tcp_task = task->dd_data; 572 573 task->hdr = task->dd_data + sizeof(*tcp_task); 574 task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE; 575 return 0; 576 } 577 578 static struct iscsi_cls_conn * 579 iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session, 580 uint32_t conn_idx) 581 { 582 struct iscsi_conn *conn; 583 struct iscsi_cls_conn *cls_conn; 584 struct iscsi_tcp_conn *tcp_conn; 585 struct iscsi_sw_tcp_conn *tcp_sw_conn; 586 struct crypto_ahash *tfm; 587 588 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn), 589 conn_idx); 590 if (!cls_conn) 591 return NULL; 592 conn = cls_conn->dd_data; 593 tcp_conn = conn->dd_data; 594 tcp_sw_conn = tcp_conn->dd_data; 595 INIT_WORK(&conn->recvwork, iscsi_sw_tcp_recv_data_work); 596 tcp_sw_conn->queue_recv = iscsi_recv_from_iscsi_q; 597 598 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 599 if (IS_ERR(tfm)) 600 goto free_conn; 601 602 tcp_sw_conn->tx_hash = ahash_request_alloc(tfm, GFP_KERNEL); 603 if (!tcp_sw_conn->tx_hash) 604 goto free_tfm; 605 ahash_request_set_callback(tcp_sw_conn->tx_hash, 0, NULL, NULL); 606 607 tcp_sw_conn->rx_hash = ahash_request_alloc(tfm, GFP_KERNEL); 608 if (!tcp_sw_conn->rx_hash) 609 goto free_tx_hash; 610 ahash_request_set_callback(tcp_sw_conn->rx_hash, 0, NULL, NULL); 611 612 tcp_conn->rx_hash = tcp_sw_conn->rx_hash; 613 614 return cls_conn; 615 616 free_tx_hash: 617 ahash_request_free(tcp_sw_conn->tx_hash); 618 free_tfm: 619 crypto_free_ahash(tfm); 620 free_conn: 621 iscsi_conn_printk(KERN_ERR, conn, 622 "Could not create connection due to crc32c " 623 "loading error. Make sure the crc32c " 624 "module is built as a module or into the " 625 "kernel\n"); 626 iscsi_tcp_conn_teardown(cls_conn); 627 return NULL; 628 } 629 630 static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn) 631 { 632 struct iscsi_session *session = conn->session; 633 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 634 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 635 struct socket *sock = tcp_sw_conn->sock; 636 637 if (!sock) 638 return; 639 640 /* 641 * Make sure we start socket shutdown now in case userspace is up 642 * but delayed in releasing the socket. 643 */ 644 kernel_sock_shutdown(sock, SHUT_RDWR); 645 646 sock_hold(sock->sk); 647 iscsi_sw_tcp_conn_restore_callbacks(conn); 648 sock_put(sock->sk); 649 650 iscsi_suspend_rx(conn); 651 652 spin_lock_bh(&session->frwd_lock); 653 tcp_sw_conn->sock = NULL; 654 spin_unlock_bh(&session->frwd_lock); 655 sockfd_put(sock); 656 } 657 658 static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn) 659 { 660 struct iscsi_conn *conn = cls_conn->dd_data; 661 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 662 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 663 664 iscsi_sw_tcp_release_conn(conn); 665 666 ahash_request_free(tcp_sw_conn->rx_hash); 667 if (tcp_sw_conn->tx_hash) { 668 struct crypto_ahash *tfm; 669 670 tfm = crypto_ahash_reqtfm(tcp_sw_conn->tx_hash); 671 ahash_request_free(tcp_sw_conn->tx_hash); 672 crypto_free_ahash(tfm); 673 } 674 675 iscsi_tcp_conn_teardown(cls_conn); 676 } 677 678 static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag) 679 { 680 struct iscsi_conn *conn = cls_conn->dd_data; 681 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 682 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 683 struct socket *sock = tcp_sw_conn->sock; 684 685 /* userspace may have goofed up and not bound us */ 686 if (!sock) 687 return; 688 689 sock->sk->sk_err = EIO; 690 wake_up_interruptible(sk_sleep(sock->sk)); 691 692 /* stop xmit side */ 693 iscsi_suspend_tx(conn); 694 695 /* stop recv side and release socket */ 696 iscsi_sw_tcp_release_conn(conn); 697 698 iscsi_conn_stop(cls_conn, flag); 699 } 700 701 static int 702 iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session, 703 struct iscsi_cls_conn *cls_conn, uint64_t transport_eph, 704 int is_leading) 705 { 706 struct iscsi_session *session = cls_session->dd_data; 707 struct iscsi_conn *conn = cls_conn->dd_data; 708 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 709 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 710 struct sock *sk; 711 struct socket *sock; 712 int err; 713 714 /* lookup for existing socket */ 715 sock = sockfd_lookup((int)transport_eph, &err); 716 if (!sock) { 717 iscsi_conn_printk(KERN_ERR, conn, 718 "sockfd_lookup failed %d\n", err); 719 return -EEXIST; 720 } 721 722 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 723 if (err) 724 goto free_socket; 725 726 spin_lock_bh(&session->frwd_lock); 727 /* bind iSCSI connection and socket */ 728 tcp_sw_conn->sock = sock; 729 spin_unlock_bh(&session->frwd_lock); 730 731 /* setup Socket parameters */ 732 sk = sock->sk; 733 sk->sk_reuse = SK_CAN_REUSE; 734 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */ 735 sk->sk_allocation = GFP_ATOMIC; 736 sk_set_memalloc(sk); 737 sock_no_linger(sk); 738 739 iscsi_sw_tcp_conn_set_callbacks(conn); 740 tcp_sw_conn->sendpage = tcp_sw_conn->sock->ops->sendpage; 741 /* 742 * set receive state machine into initial state 743 */ 744 iscsi_tcp_hdr_recv_prep(tcp_conn); 745 return 0; 746 747 free_socket: 748 sockfd_put(sock); 749 return err; 750 } 751 752 static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn, 753 enum iscsi_param param, char *buf, 754 int buflen) 755 { 756 struct iscsi_conn *conn = cls_conn->dd_data; 757 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 758 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 759 760 switch(param) { 761 case ISCSI_PARAM_HDRDGST_EN: 762 iscsi_set_param(cls_conn, param, buf, buflen); 763 break; 764 case ISCSI_PARAM_DATADGST_EN: 765 iscsi_set_param(cls_conn, param, buf, buflen); 766 tcp_sw_conn->sendpage = conn->datadgst_en ? 767 sock_no_sendpage : tcp_sw_conn->sock->ops->sendpage; 768 break; 769 case ISCSI_PARAM_MAX_R2T: 770 return iscsi_tcp_set_max_r2t(conn, buf); 771 default: 772 return iscsi_set_param(cls_conn, param, buf, buflen); 773 } 774 775 return 0; 776 } 777 778 static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn, 779 enum iscsi_param param, char *buf) 780 { 781 struct iscsi_conn *conn = cls_conn->dd_data; 782 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 783 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 784 struct sockaddr_in6 addr; 785 struct socket *sock; 786 int rc; 787 788 switch(param) { 789 case ISCSI_PARAM_CONN_PORT: 790 case ISCSI_PARAM_CONN_ADDRESS: 791 case ISCSI_PARAM_LOCAL_PORT: 792 spin_lock_bh(&conn->session->frwd_lock); 793 if (!tcp_sw_conn || !tcp_sw_conn->sock) { 794 spin_unlock_bh(&conn->session->frwd_lock); 795 return -ENOTCONN; 796 } 797 sock = tcp_sw_conn->sock; 798 sock_hold(sock->sk); 799 spin_unlock_bh(&conn->session->frwd_lock); 800 801 if (param == ISCSI_PARAM_LOCAL_PORT) 802 rc = kernel_getsockname(sock, 803 (struct sockaddr *)&addr); 804 else 805 rc = kernel_getpeername(sock, 806 (struct sockaddr *)&addr); 807 sock_put(sock->sk); 808 if (rc < 0) 809 return rc; 810 811 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 812 &addr, param, buf); 813 default: 814 return iscsi_conn_get_param(cls_conn, param, buf); 815 } 816 817 return 0; 818 } 819 820 static int iscsi_sw_tcp_host_get_param(struct Scsi_Host *shost, 821 enum iscsi_host_param param, char *buf) 822 { 823 struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(shost); 824 struct iscsi_session *session = tcp_sw_host->session; 825 struct iscsi_conn *conn; 826 struct iscsi_tcp_conn *tcp_conn; 827 struct iscsi_sw_tcp_conn *tcp_sw_conn; 828 struct sockaddr_in6 addr; 829 struct socket *sock; 830 int rc; 831 832 switch (param) { 833 case ISCSI_HOST_PARAM_IPADDRESS: 834 if (!session) 835 return -ENOTCONN; 836 837 spin_lock_bh(&session->frwd_lock); 838 conn = session->leadconn; 839 if (!conn) { 840 spin_unlock_bh(&session->frwd_lock); 841 return -ENOTCONN; 842 } 843 tcp_conn = conn->dd_data; 844 tcp_sw_conn = tcp_conn->dd_data; 845 sock = tcp_sw_conn->sock; 846 if (!sock) { 847 spin_unlock_bh(&session->frwd_lock); 848 return -ENOTCONN; 849 } 850 sock_hold(sock->sk); 851 spin_unlock_bh(&session->frwd_lock); 852 853 rc = kernel_getsockname(sock, 854 (struct sockaddr *)&addr); 855 sock_put(sock->sk); 856 if (rc < 0) 857 return rc; 858 859 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 860 &addr, 861 (enum iscsi_param)param, buf); 862 default: 863 return iscsi_host_get_param(shost, param, buf); 864 } 865 866 return 0; 867 } 868 869 static void 870 iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn, 871 struct iscsi_stats *stats) 872 { 873 struct iscsi_conn *conn = cls_conn->dd_data; 874 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 875 struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data; 876 877 stats->custom_length = 3; 878 strcpy(stats->custom[0].desc, "tx_sendpage_failures"); 879 stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt; 880 strcpy(stats->custom[1].desc, "rx_discontiguous_hdr"); 881 stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt; 882 strcpy(stats->custom[2].desc, "eh_abort_cnt"); 883 stats->custom[2].value = conn->eh_abort_cnt; 884 885 iscsi_tcp_conn_get_stats(cls_conn, stats); 886 } 887 888 static struct iscsi_cls_session * 889 iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max, 890 uint16_t qdepth, uint32_t initial_cmdsn) 891 { 892 struct iscsi_cls_session *cls_session; 893 struct iscsi_session *session; 894 struct iscsi_sw_tcp_host *tcp_sw_host; 895 struct Scsi_Host *shost; 896 int rc; 897 898 if (ep) { 899 printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep); 900 return NULL; 901 } 902 903 shost = iscsi_host_alloc(&iscsi_sw_tcp_sht, 904 sizeof(struct iscsi_sw_tcp_host), 1); 905 if (!shost) 906 return NULL; 907 shost->transportt = iscsi_sw_tcp_scsi_transport; 908 shost->cmd_per_lun = qdepth; 909 shost->max_lun = iscsi_max_lun; 910 shost->max_id = 0; 911 shost->max_channel = 0; 912 shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE; 913 914 rc = iscsi_host_get_max_scsi_cmds(shost, cmds_max); 915 if (rc < 0) 916 goto free_host; 917 shost->can_queue = rc; 918 919 if (iscsi_host_add(shost, NULL)) 920 goto free_host; 921 922 cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost, 923 cmds_max, 0, 924 sizeof(struct iscsi_tcp_task) + 925 sizeof(struct iscsi_sw_tcp_hdrbuf), 926 initial_cmdsn, 0); 927 if (!cls_session) 928 goto remove_host; 929 session = cls_session->dd_data; 930 tcp_sw_host = iscsi_host_priv(shost); 931 tcp_sw_host->session = session; 932 933 if (iscsi_tcp_r2tpool_alloc(session)) 934 goto remove_session; 935 return cls_session; 936 937 remove_session: 938 iscsi_session_teardown(cls_session); 939 remove_host: 940 iscsi_host_remove(shost, false); 941 free_host: 942 iscsi_host_free(shost); 943 return NULL; 944 } 945 946 static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session) 947 { 948 struct Scsi_Host *shost = iscsi_session_to_shost(cls_session); 949 struct iscsi_session *session = cls_session->dd_data; 950 951 if (WARN_ON_ONCE(session->leadconn)) 952 return; 953 954 iscsi_tcp_r2tpool_free(cls_session->dd_data); 955 iscsi_session_teardown(cls_session); 956 957 iscsi_host_remove(shost, false); 958 iscsi_host_free(shost); 959 } 960 961 static umode_t iscsi_sw_tcp_attr_is_visible(int param_type, int param) 962 { 963 switch (param_type) { 964 case ISCSI_HOST_PARAM: 965 switch (param) { 966 case ISCSI_HOST_PARAM_NETDEV_NAME: 967 case ISCSI_HOST_PARAM_HWADDRESS: 968 case ISCSI_HOST_PARAM_IPADDRESS: 969 case ISCSI_HOST_PARAM_INITIATOR_NAME: 970 return S_IRUGO; 971 default: 972 return 0; 973 } 974 case ISCSI_PARAM: 975 switch (param) { 976 case ISCSI_PARAM_MAX_RECV_DLENGTH: 977 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 978 case ISCSI_PARAM_HDRDGST_EN: 979 case ISCSI_PARAM_DATADGST_EN: 980 case ISCSI_PARAM_CONN_ADDRESS: 981 case ISCSI_PARAM_CONN_PORT: 982 case ISCSI_PARAM_LOCAL_PORT: 983 case ISCSI_PARAM_EXP_STATSN: 984 case ISCSI_PARAM_PERSISTENT_ADDRESS: 985 case ISCSI_PARAM_PERSISTENT_PORT: 986 case ISCSI_PARAM_PING_TMO: 987 case ISCSI_PARAM_RECV_TMO: 988 case ISCSI_PARAM_INITIAL_R2T_EN: 989 case ISCSI_PARAM_MAX_R2T: 990 case ISCSI_PARAM_IMM_DATA_EN: 991 case ISCSI_PARAM_FIRST_BURST: 992 case ISCSI_PARAM_MAX_BURST: 993 case ISCSI_PARAM_PDU_INORDER_EN: 994 case ISCSI_PARAM_DATASEQ_INORDER_EN: 995 case ISCSI_PARAM_ERL: 996 case ISCSI_PARAM_TARGET_NAME: 997 case ISCSI_PARAM_TPGT: 998 case ISCSI_PARAM_USERNAME: 999 case ISCSI_PARAM_PASSWORD: 1000 case ISCSI_PARAM_USERNAME_IN: 1001 case ISCSI_PARAM_PASSWORD_IN: 1002 case ISCSI_PARAM_FAST_ABORT: 1003 case ISCSI_PARAM_ABORT_TMO: 1004 case ISCSI_PARAM_LU_RESET_TMO: 1005 case ISCSI_PARAM_TGT_RESET_TMO: 1006 case ISCSI_PARAM_IFACE_NAME: 1007 case ISCSI_PARAM_INITIATOR_NAME: 1008 return S_IRUGO; 1009 default: 1010 return 0; 1011 } 1012 } 1013 1014 return 0; 1015 } 1016 1017 static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev) 1018 { 1019 struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(sdev->host); 1020 struct iscsi_session *session = tcp_sw_host->session; 1021 struct iscsi_conn *conn = session->leadconn; 1022 1023 if (conn->datadgst_en) 1024 blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, 1025 sdev->request_queue); 1026 blk_queue_dma_alignment(sdev->request_queue, 0); 1027 return 0; 1028 } 1029 1030 static struct scsi_host_template iscsi_sw_tcp_sht = { 1031 .module = THIS_MODULE, 1032 .name = "iSCSI Initiator over TCP/IP", 1033 .queuecommand = iscsi_queuecommand, 1034 .change_queue_depth = scsi_change_queue_depth, 1035 .can_queue = ISCSI_TOTAL_CMDS_MAX, 1036 .sg_tablesize = 4096, 1037 .max_sectors = 0xFFFF, 1038 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 1039 .eh_timed_out = iscsi_eh_cmd_timed_out, 1040 .eh_abort_handler = iscsi_eh_abort, 1041 .eh_device_reset_handler= iscsi_eh_device_reset, 1042 .eh_target_reset_handler = iscsi_eh_recover_target, 1043 .dma_boundary = PAGE_SIZE - 1, 1044 .slave_configure = iscsi_sw_tcp_slave_configure, 1045 .proc_name = "iscsi_tcp", 1046 .this_id = -1, 1047 .track_queue_depth = 1, 1048 .cmd_size = sizeof(struct iscsi_cmd), 1049 }; 1050 1051 static struct iscsi_transport iscsi_sw_tcp_transport = { 1052 .owner = THIS_MODULE, 1053 .name = "tcp", 1054 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 1055 | CAP_DATADGST, 1056 /* session management */ 1057 .create_session = iscsi_sw_tcp_session_create, 1058 .destroy_session = iscsi_sw_tcp_session_destroy, 1059 /* connection management */ 1060 .create_conn = iscsi_sw_tcp_conn_create, 1061 .bind_conn = iscsi_sw_tcp_conn_bind, 1062 .destroy_conn = iscsi_sw_tcp_conn_destroy, 1063 .attr_is_visible = iscsi_sw_tcp_attr_is_visible, 1064 .set_param = iscsi_sw_tcp_conn_set_param, 1065 .get_conn_param = iscsi_sw_tcp_conn_get_param, 1066 .get_session_param = iscsi_session_get_param, 1067 .start_conn = iscsi_conn_start, 1068 .stop_conn = iscsi_sw_tcp_conn_stop, 1069 /* iscsi host params */ 1070 .get_host_param = iscsi_sw_tcp_host_get_param, 1071 .set_host_param = iscsi_host_set_param, 1072 /* IO */ 1073 .send_pdu = iscsi_conn_send_pdu, 1074 .get_stats = iscsi_sw_tcp_conn_get_stats, 1075 /* iscsi task/cmd helpers */ 1076 .init_task = iscsi_tcp_task_init, 1077 .xmit_task = iscsi_tcp_task_xmit, 1078 .cleanup_task = iscsi_tcp_cleanup_task, 1079 /* low level pdu helpers */ 1080 .xmit_pdu = iscsi_sw_tcp_pdu_xmit, 1081 .init_pdu = iscsi_sw_tcp_pdu_init, 1082 .alloc_pdu = iscsi_sw_tcp_pdu_alloc, 1083 /* recovery */ 1084 .session_recovery_timedout = iscsi_session_recovery_timedout, 1085 }; 1086 1087 static int __init iscsi_sw_tcp_init(void) 1088 { 1089 if (iscsi_max_lun < 1) { 1090 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n", 1091 iscsi_max_lun); 1092 return -EINVAL; 1093 } 1094 1095 iscsi_sw_tcp_scsi_transport = iscsi_register_transport( 1096 &iscsi_sw_tcp_transport); 1097 if (!iscsi_sw_tcp_scsi_transport) 1098 return -ENODEV; 1099 1100 return 0; 1101 } 1102 1103 static void __exit iscsi_sw_tcp_exit(void) 1104 { 1105 iscsi_unregister_transport(&iscsi_sw_tcp_transport); 1106 } 1107 1108 module_init(iscsi_sw_tcp_init); 1109 module_exit(iscsi_sw_tcp_exit); 1110