1 /* 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/workqueue.h> 10 #include <linux/kthread.h> 11 #include <asm/unaligned.h> 12 #include <net/tcp.h> 13 #include <target/target_core_base.h> 14 #include <target/target_core_fabric.h> 15 #include "cxgbit.h" 16 17 struct sge_opaque_hdr { 18 void *dev; 19 dma_addr_t addr[MAX_SKB_FRAGS + 1]; 20 }; 21 22 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 23 24 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 25 sizeof(struct fw_ofld_tx_data_wr)) 26 27 static struct sk_buff * 28 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 29 { 30 struct sk_buff *skb = NULL; 31 u8 submode = 0; 32 int errcode; 33 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 34 35 if (len) { 36 skb = alloc_skb_with_frags(hdr_len, len, 37 0, &errcode, 38 GFP_KERNEL); 39 if (!skb) 40 return NULL; 41 42 skb_reserve(skb, TX_HDR_LEN); 43 skb_reset_transport_header(skb); 44 __skb_put(skb, ISCSI_HDR_LEN); 45 skb->data_len = len; 46 skb->len += len; 47 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 48 49 } else { 50 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 51 52 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 53 if (!skb) 54 return NULL; 55 56 skb_reserve(skb, TX_HDR_LEN + iso_len); 57 skb_reset_transport_header(skb); 58 __skb_put(skb, ISCSI_HDR_LEN); 59 } 60 61 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 62 cxgbit_skcb_submode(skb) = submode; 63 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 64 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 65 return skb; 66 } 67 68 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 69 { 70 return __cxgbit_alloc_skb(csk, len, false); 71 } 72 73 /* 74 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 75 * @skb: the packet 76 * 77 * Returns true if a packet can be sent as an offload WR with immediate 78 * data. We currently use the same limit as for Ethernet packets. 79 */ 80 static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 81 { 82 int length = skb->len; 83 84 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 85 length += sizeof(struct fw_ofld_tx_data_wr); 86 87 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 88 length += sizeof(struct cpl_tx_data_iso); 89 90 #define MAX_IMM_TX_PKT_LEN 256 91 return length <= MAX_IMM_TX_PKT_LEN; 92 } 93 94 /* 95 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 96 * @n: the number of SGL entries 97 * Calculates the number of flits needed for a scatter/gather list that 98 * can hold the given number of entries. 99 */ 100 static inline unsigned int cxgbit_sgl_len(unsigned int n) 101 { 102 n--; 103 return (3 * n) / 2 + (n & 1) + 2; 104 } 105 106 /* 107 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 108 * @skb: the packet 109 * 110 * Returns the number of flits needed for the given offload packet. 111 * These packets are already fully constructed and no additional headers 112 * will be added. 113 */ 114 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 115 { 116 unsigned int flits, cnt; 117 118 if (cxgbit_is_ofld_imm(skb)) 119 return DIV_ROUND_UP(skb->len, 8); 120 flits = skb_transport_offset(skb) / 8; 121 cnt = skb_shinfo(skb)->nr_frags; 122 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 123 cnt++; 124 return flits + cxgbit_sgl_len(cnt); 125 } 126 127 #define CXGBIT_ISO_FSLICE 0x1 128 #define CXGBIT_ISO_LSLICE 0x2 129 static void 130 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 131 { 132 struct cpl_tx_data_iso *cpl; 133 unsigned int submode = cxgbit_skcb_submode(skb); 134 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 135 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 136 137 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl)); 138 139 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 140 CPL_TX_DATA_ISO_FIRST_V(fslice) | 141 CPL_TX_DATA_ISO_LAST_V(lslice) | 142 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 143 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 144 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 145 CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 146 CPL_TX_DATA_ISO_SCSI_V(2)); 147 148 cpl->ahs_len = 0; 149 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 150 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 151 cpl->len = htonl(iso_info->len); 152 cpl->reserved2_seglen_offset = htonl(0); 153 cpl->datasn_offset = htonl(0); 154 cpl->buffer_offset = htonl(0); 155 cpl->reserved3 = 0; 156 157 __skb_pull(skb, sizeof(*cpl)); 158 } 159 160 static void 161 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 162 u32 len, u32 credits, u32 compl) 163 { 164 struct fw_ofld_tx_data_wr *req; 165 u32 submode = cxgbit_skcb_submode(skb); 166 u32 wr_ulp_mode = 0; 167 u32 hdr_size = sizeof(*req); 168 u32 opcode = FW_OFLD_TX_DATA_WR; 169 u32 immlen = 0; 170 u32 force = TX_FORCE_V(!submode); 171 172 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 173 opcode = FW_ISCSI_TX_DATA_WR; 174 immlen += sizeof(struct cpl_tx_data_iso); 175 hdr_size += sizeof(struct cpl_tx_data_iso); 176 submode |= 8; 177 } 178 179 if (cxgbit_is_ofld_imm(skb)) 180 immlen += dlen; 181 182 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, 183 hdr_size); 184 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 185 FW_WR_COMPL_V(compl) | 186 FW_WR_IMMDLEN_V(immlen)); 187 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 188 FW_WR_LEN16_V(credits)); 189 req->plen = htonl(len); 190 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 191 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 192 193 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 194 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 195 } 196 197 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 198 { 199 kfree_skb(skb); 200 } 201 202 void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 203 { 204 struct sk_buff *skb; 205 206 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 207 u32 dlen = skb->len; 208 u32 len = skb->len; 209 u32 credits_needed; 210 u32 compl = 0; 211 u32 flowclen16 = 0; 212 u32 iso_cpl_len = 0; 213 214 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 215 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 216 217 if (cxgbit_is_ofld_imm(skb)) 218 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 219 else 220 credits_needed = DIV_ROUND_UP((8 * 221 cxgbit_calc_tx_flits_ofld(skb)) + 222 iso_cpl_len, 16); 223 224 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 225 credits_needed += DIV_ROUND_UP( 226 sizeof(struct fw_ofld_tx_data_wr), 16); 227 /* 228 * Assumes the initial credits is large enough to support 229 * fw_flowc_wr plus largest possible first payload 230 */ 231 232 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 233 flowclen16 = cxgbit_send_tx_flowc_wr(csk); 234 csk->wr_cred -= flowclen16; 235 csk->wr_una_cred += flowclen16; 236 } 237 238 if (csk->wr_cred < credits_needed) { 239 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 240 csk, skb->len, skb->data_len, 241 credits_needed, csk->wr_cred); 242 break; 243 } 244 __skb_unlink(skb, &csk->txq); 245 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 246 skb->csum = credits_needed + flowclen16; 247 csk->wr_cred -= credits_needed; 248 csk->wr_una_cred += credits_needed; 249 250 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 251 csk, skb->len, skb->data_len, credits_needed, 252 csk->wr_cred, csk->wr_una_cred); 253 254 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 255 len += cxgbit_skcb_tx_extralen(skb); 256 257 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 258 (!before(csk->write_seq, 259 csk->snd_una + csk->snd_win))) { 260 compl = 1; 261 csk->wr_una_cred = 0; 262 } 263 264 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 265 compl); 266 csk->snd_nxt += len; 267 268 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 269 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 270 struct cpl_close_con_req *req = 271 (struct cpl_close_con_req *)skb->data; 272 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 273 csk->wr_una_cred = 0; 274 } 275 276 cxgbit_sock_enqueue_wr(csk, skb); 277 t4_set_arp_err_handler(skb, csk, 278 cxgbit_arp_failure_skb_discard); 279 280 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 281 csk, csk->tid, skb, len); 282 283 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 284 } 285 } 286 287 static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 288 { 289 spin_lock_bh(&csk->lock); 290 291 if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 292 csk->lock_owner = true; 293 294 spin_unlock_bh(&csk->lock); 295 296 return csk->lock_owner; 297 } 298 299 static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 300 { 301 struct sk_buff_head backlogq; 302 struct sk_buff *skb; 303 void (*fn)(struct cxgbit_sock *, struct sk_buff *); 304 305 skb_queue_head_init(&backlogq); 306 307 spin_lock_bh(&csk->lock); 308 while (skb_queue_len(&csk->backlogq)) { 309 skb_queue_splice_init(&csk->backlogq, &backlogq); 310 spin_unlock_bh(&csk->lock); 311 312 while ((skb = __skb_dequeue(&backlogq))) { 313 fn = cxgbit_skcb_rx_backlog_fn(skb); 314 fn(csk, skb); 315 } 316 317 spin_lock_bh(&csk->lock); 318 } 319 320 csk->lock_owner = false; 321 spin_unlock_bh(&csk->lock); 322 } 323 324 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 325 { 326 int ret = 0; 327 328 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 329 330 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 331 signal_pending(current))) { 332 __kfree_skb(skb); 333 __skb_queue_purge(&csk->ppodq); 334 ret = -1; 335 spin_lock_bh(&csk->lock); 336 if (csk->lock_owner) { 337 spin_unlock_bh(&csk->lock); 338 goto unlock; 339 } 340 spin_unlock_bh(&csk->lock); 341 return ret; 342 } 343 344 csk->write_seq += skb->len + 345 cxgbit_skcb_tx_extralen(skb); 346 347 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 348 __skb_queue_tail(&csk->txq, skb); 349 cxgbit_push_tx_frames(csk); 350 351 unlock: 352 cxgbit_unlock_sock(csk); 353 return ret; 354 } 355 356 static int 357 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 358 u32 data_length) 359 { 360 u32 i = 0, nr_frags = MAX_SKB_FRAGS; 361 u32 padding = ((-data_length) & 3); 362 struct scatterlist *sg; 363 struct page *page; 364 unsigned int page_off; 365 366 if (padding) 367 nr_frags--; 368 369 /* 370 * We know each entry in t_data_sg contains a page. 371 */ 372 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 373 page_off = (data_offset % PAGE_SIZE); 374 375 while (data_length && (i < nr_frags)) { 376 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 377 378 page = sg_page(sg); 379 380 get_page(page); 381 skb_fill_page_desc(skb, i, page, sg->offset + page_off, 382 cur_len); 383 skb->data_len += cur_len; 384 skb->len += cur_len; 385 skb->truesize += cur_len; 386 387 data_length -= cur_len; 388 page_off = 0; 389 sg = sg_next(sg); 390 i++; 391 } 392 393 if (data_length) 394 return -1; 395 396 if (padding) { 397 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 398 if (!page) 399 return -1; 400 skb_fill_page_desc(skb, i, page, 0, padding); 401 skb->data_len += padding; 402 skb->len += padding; 403 skb->truesize += padding; 404 } 405 406 return 0; 407 } 408 409 static int 410 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 411 struct iscsi_datain_req *dr) 412 { 413 struct iscsi_conn *conn = csk->conn; 414 struct sk_buff *skb; 415 struct iscsi_datain datain; 416 struct cxgbit_iso_info iso_info; 417 u32 data_length = cmd->se_cmd.data_length; 418 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 419 u32 num_pdu, plen, tx_data = 0; 420 bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 421 SCF_TRANSPORT_TASK_SENSE); 422 bool set_statsn = false; 423 int ret = -1; 424 425 while (data_length) { 426 num_pdu = (data_length + mrdsl - 1) / mrdsl; 427 if (num_pdu > csk->max_iso_npdu) 428 num_pdu = csk->max_iso_npdu; 429 430 plen = num_pdu * mrdsl; 431 if (plen > data_length) 432 plen = data_length; 433 434 skb = __cxgbit_alloc_skb(csk, 0, true); 435 if (unlikely(!skb)) 436 return -ENOMEM; 437 438 memset(skb->data, 0, ISCSI_HDR_LEN); 439 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 440 cxgbit_skcb_submode(skb) |= (csk->submode & 441 CXGBIT_SUBMODE_DCRC); 442 cxgbit_skcb_tx_extralen(skb) = (num_pdu * 443 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 444 ((num_pdu - 1) * ISCSI_HDR_LEN); 445 446 memset(&datain, 0, sizeof(struct iscsi_datain)); 447 memset(&iso_info, 0, sizeof(iso_info)); 448 449 if (!tx_data) 450 iso_info.flags |= CXGBIT_ISO_FSLICE; 451 452 if (!(data_length - plen)) { 453 iso_info.flags |= CXGBIT_ISO_LSLICE; 454 if (!task_sense) { 455 datain.flags = ISCSI_FLAG_DATA_STATUS; 456 iscsit_increment_maxcmdsn(cmd, conn->sess); 457 cmd->stat_sn = conn->stat_sn++; 458 set_statsn = true; 459 } 460 } 461 462 iso_info.burst_len = num_pdu * mrdsl; 463 iso_info.mpdu = mrdsl; 464 iso_info.len = ISCSI_HDR_LEN + plen; 465 466 cxgbit_cpl_tx_data_iso(skb, &iso_info); 467 468 datain.offset = tx_data; 469 datain.data_sn = cmd->data_sn - 1; 470 471 iscsit_build_datain_pdu(cmd, conn, &datain, 472 (struct iscsi_data_rsp *)skb->data, 473 set_statsn); 474 475 ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 476 if (unlikely(ret)) { 477 __kfree_skb(skb); 478 goto out; 479 } 480 481 ret = cxgbit_queue_skb(csk, skb); 482 if (unlikely(ret)) 483 goto out; 484 485 tx_data += plen; 486 data_length -= plen; 487 488 cmd->read_data_done += plen; 489 cmd->data_sn += num_pdu; 490 } 491 492 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 493 494 return 0; 495 496 out: 497 return ret; 498 } 499 500 static int 501 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 502 const struct iscsi_datain *datain) 503 { 504 struct sk_buff *skb; 505 int ret = 0; 506 507 skb = cxgbit_alloc_skb(csk, 0); 508 if (unlikely(!skb)) 509 return -ENOMEM; 510 511 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 512 513 if (datain->length) { 514 cxgbit_skcb_submode(skb) |= (csk->submode & 515 CXGBIT_SUBMODE_DCRC); 516 cxgbit_skcb_tx_extralen(skb) = 517 cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 518 } 519 520 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 521 if (ret < 0) { 522 __kfree_skb(skb); 523 return ret; 524 } 525 526 return cxgbit_queue_skb(csk, skb); 527 } 528 529 static int 530 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 531 struct iscsi_datain_req *dr, 532 const struct iscsi_datain *datain) 533 { 534 struct cxgbit_sock *csk = conn->context; 535 u32 data_length = cmd->se_cmd.data_length; 536 u32 padding = ((-data_length) & 3); 537 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 538 539 if ((data_length > mrdsl) && (!dr->recovery) && 540 (!padding) && (!datain->offset) && csk->max_iso_npdu) { 541 atomic_long_add(data_length - datain->length, 542 &conn->sess->tx_data_octets); 543 return cxgbit_tx_datain_iso(csk, cmd, dr); 544 } 545 546 return cxgbit_tx_datain(csk, cmd, datain); 547 } 548 549 static int 550 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 551 const void *data_buf, u32 data_buf_len) 552 { 553 struct cxgbit_sock *csk = conn->context; 554 struct sk_buff *skb; 555 u32 padding = ((-data_buf_len) & 3); 556 557 skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 558 if (unlikely(!skb)) 559 return -ENOMEM; 560 561 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 562 563 if (data_buf_len) { 564 u32 pad_bytes = 0; 565 566 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 567 568 if (padding) 569 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 570 &pad_bytes, padding); 571 } 572 573 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 574 cxgbit_skcb_submode(skb)]; 575 576 return cxgbit_queue_skb(csk, skb); 577 } 578 579 int 580 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 581 struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 582 { 583 if (dr) 584 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 585 else 586 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 587 } 588 589 int cxgbit_validate_params(struct iscsi_conn *conn) 590 { 591 struct cxgbit_sock *csk = conn->context; 592 struct cxgbit_device *cdev = csk->com.cdev; 593 struct iscsi_param *param; 594 u32 max_xmitdsl; 595 596 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 597 conn->param_list); 598 if (!param) 599 return -1; 600 601 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 602 return -1; 603 604 if (max_xmitdsl > cdev->mdsl) { 605 if (iscsi_change_param_sprintf( 606 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 607 return -1; 608 } 609 610 return 0; 611 } 612 613 static int cxgbit_set_digest(struct cxgbit_sock *csk) 614 { 615 struct iscsi_conn *conn = csk->conn; 616 struct iscsi_param *param; 617 618 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 619 if (!param) { 620 pr_err("param not found key %s\n", HEADERDIGEST); 621 return -1; 622 } 623 624 if (!strcmp(param->value, CRC32C)) 625 csk->submode |= CXGBIT_SUBMODE_HCRC; 626 627 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 628 if (!param) { 629 csk->submode = 0; 630 pr_err("param not found key %s\n", DATADIGEST); 631 return -1; 632 } 633 634 if (!strcmp(param->value, CRC32C)) 635 csk->submode |= CXGBIT_SUBMODE_DCRC; 636 637 if (cxgbit_setup_conn_digest(csk)) { 638 csk->submode = 0; 639 return -1; 640 } 641 642 return 0; 643 } 644 645 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 646 { 647 struct iscsi_conn *conn = csk->conn; 648 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 649 struct iscsi_param *param; 650 u32 mrdsl, mbl; 651 u32 max_npdu, max_iso_npdu; 652 653 if (conn->login->leading_connection) { 654 param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 655 conn->param_list); 656 if (!param) { 657 pr_err("param not found key %s\n", DATASEQUENCEINORDER); 658 return -1; 659 } 660 661 if (strcmp(param->value, YES)) 662 return 0; 663 664 param = iscsi_find_param_from_key(DATAPDUINORDER, 665 conn->param_list); 666 if (!param) { 667 pr_err("param not found key %s\n", DATAPDUINORDER); 668 return -1; 669 } 670 671 if (strcmp(param->value, YES)) 672 return 0; 673 674 param = iscsi_find_param_from_key(MAXBURSTLENGTH, 675 conn->param_list); 676 if (!param) { 677 pr_err("param not found key %s\n", MAXBURSTLENGTH); 678 return -1; 679 } 680 681 if (kstrtou32(param->value, 0, &mbl) < 0) 682 return -1; 683 } else { 684 if (!conn->sess->sess_ops->DataSequenceInOrder) 685 return 0; 686 if (!conn->sess->sess_ops->DataPDUInOrder) 687 return 0; 688 689 mbl = conn->sess->sess_ops->MaxBurstLength; 690 } 691 692 mrdsl = conn_ops->MaxRecvDataSegmentLength; 693 max_npdu = mbl / mrdsl; 694 695 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / 696 (ISCSI_HDR_LEN + mrdsl + 697 cxgbit_digest_len[csk->submode]); 698 699 csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 700 701 if (csk->max_iso_npdu <= 1) 702 csk->max_iso_npdu = 0; 703 704 return 0; 705 } 706 707 static int cxgbit_set_params(struct iscsi_conn *conn) 708 { 709 struct cxgbit_sock *csk = conn->context; 710 struct cxgbit_device *cdev = csk->com.cdev; 711 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 712 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 713 struct iscsi_param *param; 714 u8 erl; 715 716 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 717 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 718 719 if (conn->login->leading_connection) { 720 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 721 conn->param_list); 722 if (!param) { 723 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 724 return -1; 725 } 726 if (kstrtou8(param->value, 0, &erl) < 0) 727 return -1; 728 } else { 729 erl = conn->sess->sess_ops->ErrorRecoveryLevel; 730 } 731 732 if (!erl) { 733 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 734 if (cxgbit_set_iso_npdu(csk)) 735 return -1; 736 } 737 738 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 739 if (cxgbit_setup_conn_pgidx(csk, 740 ppm->tformat.pgsz_idx_dflt)) 741 return -1; 742 set_bit(CSK_DDP_ENABLE, &csk->com.flags); 743 } 744 } 745 746 if (cxgbit_set_digest(csk)) 747 return -1; 748 749 return 0; 750 } 751 752 int 753 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 754 u32 length) 755 { 756 struct cxgbit_sock *csk = conn->context; 757 struct sk_buff *skb; 758 u32 padding_buf = 0; 759 u8 padding = ((-length) & 3); 760 761 skb = cxgbit_alloc_skb(csk, length + padding); 762 if (!skb) 763 return -ENOMEM; 764 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 765 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 766 767 if (padding) 768 skb_store_bits(skb, ISCSI_HDR_LEN + length, 769 &padding_buf, padding); 770 771 if (login->login_complete) { 772 if (cxgbit_set_params(conn)) { 773 kfree_skb(skb); 774 return -1; 775 } 776 777 set_bit(CSK_LOGIN_DONE, &csk->com.flags); 778 } 779 780 if (cxgbit_queue_skb(csk, skb)) 781 return -1; 782 783 if ((!login->login_complete) && (!login->login_failed)) 784 schedule_delayed_work(&conn->login_work, 0); 785 786 return 0; 787 } 788 789 static void 790 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 791 unsigned int nents) 792 { 793 struct skb_seq_state st; 794 const u8 *buf; 795 unsigned int consumed = 0, buf_len; 796 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 797 798 skb_prepare_seq_read(skb, pdu_cb->doffset, 799 pdu_cb->doffset + pdu_cb->dlen, 800 &st); 801 802 while (true) { 803 buf_len = skb_seq_read(consumed, &buf, &st); 804 if (!buf_len) { 805 skb_abort_seq_read(&st); 806 break; 807 } 808 809 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 810 buf_len, consumed); 811 } 812 } 813 814 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 815 { 816 struct iscsi_conn *conn = csk->conn; 817 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 818 struct cxgbit_cmd *ccmd; 819 struct iscsi_cmd *cmd; 820 821 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 822 if (!cmd) { 823 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 824 return NULL; 825 } 826 827 ccmd = iscsit_priv_cmd(cmd); 828 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 829 ccmd->setup_ddp = true; 830 831 return cmd; 832 } 833 834 static int 835 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 836 u32 length) 837 { 838 struct iscsi_conn *conn = cmd->conn; 839 struct cxgbit_sock *csk = conn->context; 840 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 841 842 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 843 pr_err("ImmediateData CRC32C DataDigest error\n"); 844 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 845 pr_err("Unable to recover from" 846 " Immediate Data digest failure while" 847 " in ERL=0.\n"); 848 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 849 (unsigned char *)hdr); 850 return IMMEDIATE_DATA_CANNOT_RECOVER; 851 } 852 853 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 854 (unsigned char *)hdr); 855 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 856 } 857 858 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 859 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 860 struct skb_shared_info *ssi = skb_shinfo(csk->skb); 861 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 862 863 sg_init_table(&ccmd->sg, 1); 864 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 865 dfrag->page_offset); 866 get_page(dfrag->page.p); 867 868 cmd->se_cmd.t_data_sg = &ccmd->sg; 869 cmd->se_cmd.t_data_nents = 1; 870 871 ccmd->release = true; 872 } else { 873 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 874 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 875 876 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); 877 } 878 879 cmd->write_data_done += pdu_cb->dlen; 880 881 if (cmd->write_data_done == cmd->se_cmd.data_length) { 882 spin_lock_bh(&cmd->istate_lock); 883 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 884 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 885 spin_unlock_bh(&cmd->istate_lock); 886 } 887 888 return IMMEDIATE_DATA_NORMAL_OPERATION; 889 } 890 891 static int 892 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 893 bool dump_payload) 894 { 895 struct iscsi_conn *conn = cmd->conn; 896 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 897 /* 898 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 899 */ 900 if (dump_payload) 901 goto after_immediate_data; 902 903 immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 904 cmd->first_burst_len); 905 after_immediate_data: 906 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 907 /* 908 * A PDU/CmdSN carrying Immediate Data passed 909 * DataCRC, check against ExpCmdSN/MaxCmdSN if 910 * Immediate Bit is not set. 911 */ 912 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 913 (unsigned char *)hdr, 914 hdr->cmdsn); 915 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 916 return -1; 917 918 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 919 target_put_sess_cmd(&cmd->se_cmd); 920 return 0; 921 } else if (cmd->unsolicited_data) { 922 iscsit_set_unsoliticed_dataout(cmd); 923 } 924 925 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 926 /* 927 * Immediate Data failed DataCRC and ERL>=1, 928 * silently drop this PDU and let the initiator 929 * plug the CmdSN gap. 930 * 931 * FIXME: Send Unsolicited NOPIN with reserved 932 * TTT here to help the initiator figure out 933 * the missing CmdSN, although they should be 934 * intelligent enough to determine the missing 935 * CmdSN and issue a retry to plug the sequence. 936 */ 937 cmd->i_state = ISTATE_REMOVE; 938 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 939 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 940 return -1; 941 942 return 0; 943 } 944 945 static int 946 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 947 { 948 struct iscsi_conn *conn = csk->conn; 949 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 950 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 951 int rc; 952 bool dump_payload = false; 953 954 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 955 if (rc < 0) 956 return rc; 957 958 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 959 (pdu_cb->nr_dfrags == 1)) 960 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 961 962 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 963 if (rc < 0) 964 return 0; 965 else if (rc > 0) 966 dump_payload = true; 967 968 if (!pdu_cb->dlen) 969 return 0; 970 971 return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 972 } 973 974 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 975 { 976 struct scatterlist *sg_start; 977 struct iscsi_conn *conn = csk->conn; 978 struct iscsi_cmd *cmd = NULL; 979 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 980 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 981 u32 data_offset = be32_to_cpu(hdr->offset); 982 u32 data_len = pdu_cb->dlen; 983 int rc, sg_nents, sg_off; 984 bool dcrc_err = false; 985 986 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 987 if (rc < 0) 988 return rc; 989 else if (!cmd) 990 return 0; 991 992 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 993 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 994 " DataSN: 0x%08x\n", 995 hdr->itt, hdr->offset, data_len, 996 hdr->datasn); 997 998 dcrc_err = true; 999 goto check_payload; 1000 } 1001 1002 pr_debug("DataOut data_len: %u, " 1003 "write_data_done: %u, data_length: %u\n", 1004 data_len, cmd->write_data_done, 1005 cmd->se_cmd.data_length); 1006 1007 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1008 sg_off = data_offset / PAGE_SIZE; 1009 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1010 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); 1011 1012 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); 1013 } 1014 1015 check_payload: 1016 1017 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1018 if (rc < 0) 1019 return rc; 1020 1021 return 0; 1022 } 1023 1024 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1025 { 1026 struct iscsi_conn *conn = csk->conn; 1027 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1028 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1029 unsigned char *ping_data = NULL; 1030 u32 payload_length = pdu_cb->dlen; 1031 int ret; 1032 1033 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1034 if (ret < 0) 1035 return 0; 1036 1037 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1038 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1039 pr_err("Unable to recover from" 1040 " NOPOUT Ping DataCRC failure while in" 1041 " ERL=0.\n"); 1042 ret = -1; 1043 goto out; 1044 } else { 1045 /* 1046 * drop this PDU and let the 1047 * initiator plug the CmdSN gap. 1048 */ 1049 pr_info("Dropping NOPOUT" 1050 " Command CmdSN: 0x%08x due to" 1051 " DataCRC error.\n", hdr->cmdsn); 1052 ret = 0; 1053 goto out; 1054 } 1055 } 1056 1057 /* 1058 * Handle NOP-OUT payload for traditional iSCSI sockets 1059 */ 1060 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1061 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1062 if (!ping_data) { 1063 pr_err("Unable to allocate memory for" 1064 " NOPOUT ping data.\n"); 1065 ret = -1; 1066 goto out; 1067 } 1068 1069 skb_copy_bits(csk->skb, pdu_cb->doffset, 1070 ping_data, payload_length); 1071 1072 ping_data[payload_length] = '\0'; 1073 /* 1074 * Attach ping data to struct iscsi_cmd->buf_ptr. 1075 */ 1076 cmd->buf_ptr = ping_data; 1077 cmd->buf_ptr_size = payload_length; 1078 1079 pr_debug("Got %u bytes of NOPOUT ping" 1080 " data.\n", payload_length); 1081 pr_debug("Ping Data: \"%s\"\n", ping_data); 1082 } 1083 1084 return iscsit_process_nop_out(conn, cmd, hdr); 1085 out: 1086 if (cmd) 1087 iscsit_free_cmd(cmd, false); 1088 return ret; 1089 } 1090 1091 static int 1092 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1093 { 1094 struct iscsi_conn *conn = csk->conn; 1095 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1096 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1097 u32 payload_length = pdu_cb->dlen; 1098 int rc; 1099 unsigned char *text_in = NULL; 1100 1101 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1102 if (rc < 0) 1103 return rc; 1104 1105 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1106 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1107 pr_err("Unable to recover from" 1108 " Text Data digest failure while in" 1109 " ERL=0.\n"); 1110 goto reject; 1111 } else { 1112 /* 1113 * drop this PDU and let the 1114 * initiator plug the CmdSN gap. 1115 */ 1116 pr_info("Dropping Text" 1117 " Command CmdSN: 0x%08x due to" 1118 " DataCRC error.\n", hdr->cmdsn); 1119 return 0; 1120 } 1121 } 1122 1123 if (payload_length) { 1124 text_in = kzalloc(payload_length, GFP_KERNEL); 1125 if (!text_in) { 1126 pr_err("Unable to allocate text_in of payload_length: %u\n", 1127 payload_length); 1128 return -ENOMEM; 1129 } 1130 skb_copy_bits(csk->skb, pdu_cb->doffset, 1131 text_in, payload_length); 1132 1133 text_in[payload_length - 1] = '\0'; 1134 1135 cmd->text_in_ptr = text_in; 1136 } 1137 1138 return iscsit_process_text_cmd(conn, cmd, hdr); 1139 1140 reject: 1141 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1142 pdu_cb->hdr); 1143 } 1144 1145 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1146 { 1147 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1148 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1149 struct iscsi_conn *conn = csk->conn; 1150 struct iscsi_cmd *cmd = NULL; 1151 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1152 int ret = -EINVAL; 1153 1154 switch (opcode) { 1155 case ISCSI_OP_SCSI_CMD: 1156 cmd = cxgbit_allocate_cmd(csk); 1157 if (!cmd) 1158 goto reject; 1159 1160 ret = cxgbit_handle_scsi_cmd(csk, cmd); 1161 break; 1162 case ISCSI_OP_SCSI_DATA_OUT: 1163 ret = cxgbit_handle_iscsi_dataout(csk); 1164 break; 1165 case ISCSI_OP_NOOP_OUT: 1166 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1167 cmd = cxgbit_allocate_cmd(csk); 1168 if (!cmd) 1169 goto reject; 1170 } 1171 1172 ret = cxgbit_handle_nop_out(csk, cmd); 1173 break; 1174 case ISCSI_OP_SCSI_TMFUNC: 1175 cmd = cxgbit_allocate_cmd(csk); 1176 if (!cmd) 1177 goto reject; 1178 1179 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1180 (unsigned char *)hdr); 1181 break; 1182 case ISCSI_OP_TEXT: 1183 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1184 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1185 if (!cmd) 1186 goto reject; 1187 } else { 1188 cmd = cxgbit_allocate_cmd(csk); 1189 if (!cmd) 1190 goto reject; 1191 } 1192 1193 ret = cxgbit_handle_text_cmd(csk, cmd); 1194 break; 1195 case ISCSI_OP_LOGOUT: 1196 cmd = cxgbit_allocate_cmd(csk); 1197 if (!cmd) 1198 goto reject; 1199 1200 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1201 if (ret > 0) 1202 wait_for_completion_timeout(&conn->conn_logout_comp, 1203 SECONDS_FOR_LOGOUT_COMP 1204 * HZ); 1205 break; 1206 case ISCSI_OP_SNACK: 1207 ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1208 break; 1209 default: 1210 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1211 dump_stack(); 1212 break; 1213 } 1214 1215 return ret; 1216 1217 reject: 1218 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1219 (unsigned char *)hdr); 1220 return ret; 1221 } 1222 1223 static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1224 { 1225 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1226 struct iscsi_conn *conn = csk->conn; 1227 struct iscsi_hdr *hdr = pdu_cb->hdr; 1228 u8 opcode; 1229 1230 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1231 atomic_long_inc(&conn->sess->conn_digest_errors); 1232 goto transport_err; 1233 } 1234 1235 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1236 goto transport_err; 1237 1238 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1239 1240 if (conn->sess->sess_ops->SessionType && 1241 ((!(opcode & ISCSI_OP_TEXT)) || 1242 (!(opcode & ISCSI_OP_LOGOUT)))) { 1243 pr_err("Received illegal iSCSI Opcode: 0x%02x" 1244 " while in Discovery Session, rejecting.\n", opcode); 1245 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1246 (unsigned char *)hdr); 1247 goto transport_err; 1248 } 1249 1250 if (cxgbit_target_rx_opcode(csk) < 0) 1251 goto transport_err; 1252 1253 return 0; 1254 1255 transport_err: 1256 return -1; 1257 } 1258 1259 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1260 { 1261 struct iscsi_conn *conn = csk->conn; 1262 struct iscsi_login *login = conn->login; 1263 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1264 struct iscsi_login_req *login_req; 1265 1266 login_req = (struct iscsi_login_req *)login->req; 1267 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1268 1269 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1270 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1271 login_req->flags, login_req->itt, login_req->cmdsn, 1272 login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1273 /* 1274 * Setup the initial iscsi_login values from the leading 1275 * login request PDU. 1276 */ 1277 if (login->first_request) { 1278 login_req = (struct iscsi_login_req *)login->req; 1279 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1280 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1281 login_req->flags); 1282 login->version_min = login_req->min_version; 1283 login->version_max = login_req->max_version; 1284 memcpy(login->isid, login_req->isid, 6); 1285 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1286 login->init_task_tag = login_req->itt; 1287 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1288 login->cid = be16_to_cpu(login_req->cid); 1289 login->tsih = be16_to_cpu(login_req->tsih); 1290 } 1291 1292 if (iscsi_target_check_login_request(conn, login) < 0) 1293 return -1; 1294 1295 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1296 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1297 1298 return 0; 1299 } 1300 1301 static int 1302 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1303 { 1304 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1305 int ret; 1306 1307 cxgbit_rx_pdu_cb(skb) = pdu_cb; 1308 1309 csk->skb = skb; 1310 1311 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1312 ret = cxgbit_rx_login_pdu(csk); 1313 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1314 } else { 1315 ret = cxgbit_rx_opcode(csk); 1316 } 1317 1318 return ret; 1319 } 1320 1321 static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1322 { 1323 struct skb_shared_info *ssi = skb_shinfo(skb); 1324 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1325 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1326 u8 i; 1327 1328 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1329 skb, skb->head, skb->data, skb->len, skb->data_len, 1330 ssi->nr_frags); 1331 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1332 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1333 1334 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1335 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1336 "frags %u.\n", 1337 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1338 pdu_cb->ddigest, pdu_cb->frags); 1339 for (i = 0; i < ssi->nr_frags; i++) 1340 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1341 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 1342 } 1343 1344 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1345 { 1346 struct sk_buff *skb = csk->lro_hskb; 1347 struct skb_shared_info *ssi = skb_shinfo(skb); 1348 u8 i; 1349 1350 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1351 for (i = 0; i < ssi->nr_frags; i++) 1352 put_page(skb_frag_page(&ssi->frags[i])); 1353 ssi->nr_frags = 0; 1354 } 1355 1356 static void 1357 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1358 { 1359 struct sk_buff *hskb = csk->lro_hskb; 1360 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1361 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1362 struct skb_shared_info *hssi = skb_shinfo(hskb); 1363 struct skb_shared_info *ssi = skb_shinfo(skb); 1364 unsigned int len = 0; 1365 1366 if (pdu_cb->flags & PDUCBF_RX_HDR) { 1367 hpdu_cb->flags = pdu_cb->flags; 1368 hpdu_cb->seq = pdu_cb->seq; 1369 hpdu_cb->hdr = pdu_cb->hdr; 1370 hpdu_cb->hlen = pdu_cb->hlen; 1371 1372 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx], 1373 sizeof(skb_frag_t)); 1374 1375 get_page(skb_frag_page(&hssi->frags[0])); 1376 hssi->nr_frags = 1; 1377 hpdu_cb->frags = 1; 1378 hpdu_cb->hfrag_idx = 0; 1379 1380 len = hssi->frags[0].size; 1381 hskb->len = len; 1382 hskb->data_len = len; 1383 hskb->truesize = len; 1384 } 1385 1386 if (pdu_cb->flags & PDUCBF_RX_DATA) { 1387 u8 hfrag_idx = 1, i; 1388 1389 hpdu_cb->flags |= pdu_cb->flags; 1390 1391 len = 0; 1392 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) { 1393 memcpy(&hssi->frags[hfrag_idx], 1394 &ssi->frags[pdu_cb->dfrag_idx + i], 1395 sizeof(skb_frag_t)); 1396 1397 get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1398 1399 len += hssi->frags[hfrag_idx].size; 1400 1401 hssi->nr_frags++; 1402 hpdu_cb->frags++; 1403 } 1404 1405 hpdu_cb->dlen = pdu_cb->dlen; 1406 hpdu_cb->doffset = hpdu_cb->hlen; 1407 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1408 hpdu_cb->dfrag_idx = 1; 1409 hskb->len += len; 1410 hskb->data_len += len; 1411 hskb->truesize += len; 1412 } 1413 1414 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1415 hpdu_cb->flags |= pdu_cb->flags; 1416 1417 if (hpdu_cb->flags & PDUCBF_RX_DATA) 1418 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1419 1420 hpdu_cb->ddigest = pdu_cb->ddigest; 1421 hpdu_cb->pdulen = pdu_cb->pdulen; 1422 } 1423 } 1424 1425 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1426 { 1427 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1428 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1429 u8 pdu_idx = 0, last_idx = 0; 1430 int ret = 0; 1431 1432 if (!pdu_cb->complete) { 1433 cxgbit_lro_skb_merge(csk, skb, 0); 1434 1435 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1436 struct sk_buff *hskb = csk->lro_hskb; 1437 1438 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1439 1440 cxgbit_lro_hskb_reset(csk); 1441 1442 if (ret < 0) 1443 goto out; 1444 } 1445 1446 pdu_idx = 1; 1447 } 1448 1449 if (lro_cb->pdu_idx) 1450 last_idx = lro_cb->pdu_idx - 1; 1451 1452 for (; pdu_idx <= last_idx; pdu_idx++) { 1453 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1454 if (ret < 0) 1455 goto out; 1456 } 1457 1458 if ((!lro_cb->complete) && lro_cb->pdu_idx) 1459 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1460 1461 out: 1462 return ret; 1463 } 1464 1465 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1466 { 1467 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1468 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1469 int ret = -1; 1470 1471 if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1472 (pdu_cb->seq != csk->rcv_nxt)) { 1473 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1474 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1475 cxgbit_lro_skb_dump(skb); 1476 return ret; 1477 } 1478 1479 csk->rcv_nxt += lro_cb->pdu_totallen; 1480 1481 ret = cxgbit_process_lro_skb(csk, skb); 1482 1483 csk->rx_credits += lro_cb->pdu_totallen; 1484 1485 if (csk->rx_credits >= (csk->rcv_win / 4)) 1486 cxgbit_rx_data_ack(csk); 1487 1488 return ret; 1489 } 1490 1491 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1492 { 1493 int ret = -1; 1494 1495 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) 1496 ret = cxgbit_rx_lro_skb(csk, skb); 1497 1498 __kfree_skb(skb); 1499 return ret; 1500 } 1501 1502 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1503 { 1504 spin_lock_bh(&csk->rxq.lock); 1505 if (skb_queue_len(&csk->rxq)) { 1506 skb_queue_splice_init(&csk->rxq, rxq); 1507 spin_unlock_bh(&csk->rxq.lock); 1508 return true; 1509 } 1510 spin_unlock_bh(&csk->rxq.lock); 1511 return false; 1512 } 1513 1514 static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1515 { 1516 struct sk_buff *skb; 1517 struct sk_buff_head rxq; 1518 1519 skb_queue_head_init(&rxq); 1520 1521 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1522 1523 if (signal_pending(current)) 1524 goto out; 1525 1526 while ((skb = __skb_dequeue(&rxq))) { 1527 if (cxgbit_rx_skb(csk, skb)) 1528 goto out; 1529 } 1530 1531 return 0; 1532 out: 1533 __skb_queue_purge(&rxq); 1534 return -1; 1535 } 1536 1537 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1538 { 1539 struct cxgbit_sock *csk = conn->context; 1540 int ret = -1; 1541 1542 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1543 ret = cxgbit_wait_rxq(csk); 1544 if (ret) { 1545 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1546 break; 1547 } 1548 } 1549 1550 return ret; 1551 } 1552 1553 void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1554 { 1555 struct cxgbit_sock *csk = conn->context; 1556 1557 while (!kthread_should_stop()) { 1558 iscsit_thread_check_cpumask(conn, current, 0); 1559 if (cxgbit_wait_rxq(csk)) 1560 return; 1561 } 1562 } 1563