1 /* 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/workqueue.h> 10 #include <linux/kthread.h> 11 #include <asm/unaligned.h> 12 #include <net/tcp.h> 13 #include <target/target_core_base.h> 14 #include <target/target_core_fabric.h> 15 #include "cxgbit.h" 16 17 struct sge_opaque_hdr { 18 void *dev; 19 dma_addr_t addr[MAX_SKB_FRAGS + 1]; 20 }; 21 22 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 23 24 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 25 sizeof(struct fw_ofld_tx_data_wr)) 26 27 static struct sk_buff * 28 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 29 { 30 struct sk_buff *skb = NULL; 31 u8 submode = 0; 32 int errcode; 33 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 34 35 if (len) { 36 skb = alloc_skb_with_frags(hdr_len, len, 37 0, &errcode, 38 GFP_KERNEL); 39 if (!skb) 40 return NULL; 41 42 skb_reserve(skb, TX_HDR_LEN); 43 skb_reset_transport_header(skb); 44 __skb_put(skb, ISCSI_HDR_LEN); 45 skb->data_len = len; 46 skb->len += len; 47 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 48 49 } else { 50 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 51 52 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 53 if (!skb) 54 return NULL; 55 56 skb_reserve(skb, TX_HDR_LEN + iso_len); 57 skb_reset_transport_header(skb); 58 __skb_put(skb, ISCSI_HDR_LEN); 59 } 60 61 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 62 cxgbit_skcb_submode(skb) = submode; 63 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 64 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 65 return skb; 66 } 67 68 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 69 { 70 return __cxgbit_alloc_skb(csk, len, false); 71 } 72 73 /* 74 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 75 * @skb: the packet 76 * 77 * Returns true if a packet can be sent as an offload WR with immediate 78 * data. We currently use the same limit as for Ethernet packets. 79 */ 80 static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 81 { 82 int length = skb->len; 83 84 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 85 length += sizeof(struct fw_ofld_tx_data_wr); 86 87 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 88 length += sizeof(struct cpl_tx_data_iso); 89 90 #define MAX_IMM_TX_PKT_LEN 256 91 return length <= MAX_IMM_TX_PKT_LEN; 92 } 93 94 /* 95 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 96 * @n: the number of SGL entries 97 * Calculates the number of flits needed for a scatter/gather list that 98 * can hold the given number of entries. 99 */ 100 static inline unsigned int cxgbit_sgl_len(unsigned int n) 101 { 102 n--; 103 return (3 * n) / 2 + (n & 1) + 2; 104 } 105 106 /* 107 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 108 * @skb: the packet 109 * 110 * Returns the number of flits needed for the given offload packet. 111 * These packets are already fully constructed and no additional headers 112 * will be added. 113 */ 114 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 115 { 116 unsigned int flits, cnt; 117 118 if (cxgbit_is_ofld_imm(skb)) 119 return DIV_ROUND_UP(skb->len, 8); 120 flits = skb_transport_offset(skb) / 8; 121 cnt = skb_shinfo(skb)->nr_frags; 122 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 123 cnt++; 124 return flits + cxgbit_sgl_len(cnt); 125 } 126 127 #define CXGBIT_ISO_FSLICE 0x1 128 #define CXGBIT_ISO_LSLICE 0x2 129 static void 130 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 131 { 132 struct cpl_tx_data_iso *cpl; 133 unsigned int submode = cxgbit_skcb_submode(skb); 134 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 135 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 136 137 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl)); 138 139 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 140 CPL_TX_DATA_ISO_FIRST_V(fslice) | 141 CPL_TX_DATA_ISO_LAST_V(lslice) | 142 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 143 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 144 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 145 CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 146 CPL_TX_DATA_ISO_SCSI_V(2)); 147 148 cpl->ahs_len = 0; 149 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 150 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 151 cpl->len = htonl(iso_info->len); 152 cpl->reserved2_seglen_offset = htonl(0); 153 cpl->datasn_offset = htonl(0); 154 cpl->buffer_offset = htonl(0); 155 cpl->reserved3 = 0; 156 157 __skb_pull(skb, sizeof(*cpl)); 158 } 159 160 static void 161 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 162 u32 len, u32 credits, u32 compl) 163 { 164 struct fw_ofld_tx_data_wr *req; 165 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 166 u32 submode = cxgbit_skcb_submode(skb); 167 u32 wr_ulp_mode = 0; 168 u32 hdr_size = sizeof(*req); 169 u32 opcode = FW_OFLD_TX_DATA_WR; 170 u32 immlen = 0; 171 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : 172 T6_TX_FORCE_F; 173 174 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 175 opcode = FW_ISCSI_TX_DATA_WR; 176 immlen += sizeof(struct cpl_tx_data_iso); 177 hdr_size += sizeof(struct cpl_tx_data_iso); 178 submode |= 8; 179 } 180 181 if (cxgbit_is_ofld_imm(skb)) 182 immlen += dlen; 183 184 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, 185 hdr_size); 186 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 187 FW_WR_COMPL_V(compl) | 188 FW_WR_IMMDLEN_V(immlen)); 189 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 190 FW_WR_LEN16_V(credits)); 191 req->plen = htonl(len); 192 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 193 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 194 195 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 196 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 197 } 198 199 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 200 { 201 kfree_skb(skb); 202 } 203 204 void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 205 { 206 struct sk_buff *skb; 207 208 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 209 u32 dlen = skb->len; 210 u32 len = skb->len; 211 u32 credits_needed; 212 u32 compl = 0; 213 u32 flowclen16 = 0; 214 u32 iso_cpl_len = 0; 215 216 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 217 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 218 219 if (cxgbit_is_ofld_imm(skb)) 220 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 221 else 222 credits_needed = DIV_ROUND_UP((8 * 223 cxgbit_calc_tx_flits_ofld(skb)) + 224 iso_cpl_len, 16); 225 226 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 227 credits_needed += DIV_ROUND_UP( 228 sizeof(struct fw_ofld_tx_data_wr), 16); 229 /* 230 * Assumes the initial credits is large enough to support 231 * fw_flowc_wr plus largest possible first payload 232 */ 233 234 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 235 flowclen16 = cxgbit_send_tx_flowc_wr(csk); 236 csk->wr_cred -= flowclen16; 237 csk->wr_una_cred += flowclen16; 238 } 239 240 if (csk->wr_cred < credits_needed) { 241 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 242 csk, skb->len, skb->data_len, 243 credits_needed, csk->wr_cred); 244 break; 245 } 246 __skb_unlink(skb, &csk->txq); 247 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 248 skb->csum = (__force __wsum)(credits_needed + flowclen16); 249 csk->wr_cred -= credits_needed; 250 csk->wr_una_cred += credits_needed; 251 252 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 253 csk, skb->len, skb->data_len, credits_needed, 254 csk->wr_cred, csk->wr_una_cred); 255 256 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 257 len += cxgbit_skcb_tx_extralen(skb); 258 259 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 260 (!before(csk->write_seq, 261 csk->snd_una + csk->snd_win))) { 262 compl = 1; 263 csk->wr_una_cred = 0; 264 } 265 266 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 267 compl); 268 csk->snd_nxt += len; 269 270 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 271 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 272 struct cpl_close_con_req *req = 273 (struct cpl_close_con_req *)skb->data; 274 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 275 csk->wr_una_cred = 0; 276 } 277 278 cxgbit_sock_enqueue_wr(csk, skb); 279 t4_set_arp_err_handler(skb, csk, 280 cxgbit_arp_failure_skb_discard); 281 282 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 283 csk, csk->tid, skb, len); 284 285 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 286 } 287 } 288 289 static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 290 { 291 spin_lock_bh(&csk->lock); 292 293 if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 294 csk->lock_owner = true; 295 296 spin_unlock_bh(&csk->lock); 297 298 return csk->lock_owner; 299 } 300 301 static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 302 { 303 struct sk_buff_head backlogq; 304 struct sk_buff *skb; 305 void (*fn)(struct cxgbit_sock *, struct sk_buff *); 306 307 skb_queue_head_init(&backlogq); 308 309 spin_lock_bh(&csk->lock); 310 while (skb_queue_len(&csk->backlogq)) { 311 skb_queue_splice_init(&csk->backlogq, &backlogq); 312 spin_unlock_bh(&csk->lock); 313 314 while ((skb = __skb_dequeue(&backlogq))) { 315 fn = cxgbit_skcb_rx_backlog_fn(skb); 316 fn(csk, skb); 317 } 318 319 spin_lock_bh(&csk->lock); 320 } 321 322 csk->lock_owner = false; 323 spin_unlock_bh(&csk->lock); 324 } 325 326 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 327 { 328 int ret = 0; 329 330 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 331 332 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 333 signal_pending(current))) { 334 __kfree_skb(skb); 335 __skb_queue_purge(&csk->ppodq); 336 ret = -1; 337 spin_lock_bh(&csk->lock); 338 if (csk->lock_owner) { 339 spin_unlock_bh(&csk->lock); 340 goto unlock; 341 } 342 spin_unlock_bh(&csk->lock); 343 return ret; 344 } 345 346 csk->write_seq += skb->len + 347 cxgbit_skcb_tx_extralen(skb); 348 349 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 350 __skb_queue_tail(&csk->txq, skb); 351 cxgbit_push_tx_frames(csk); 352 353 unlock: 354 cxgbit_unlock_sock(csk); 355 return ret; 356 } 357 358 static int 359 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 360 u32 data_length) 361 { 362 u32 i = 0, nr_frags = MAX_SKB_FRAGS; 363 u32 padding = ((-data_length) & 3); 364 struct scatterlist *sg; 365 struct page *page; 366 unsigned int page_off; 367 368 if (padding) 369 nr_frags--; 370 371 /* 372 * We know each entry in t_data_sg contains a page. 373 */ 374 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 375 page_off = (data_offset % PAGE_SIZE); 376 377 while (data_length && (i < nr_frags)) { 378 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 379 380 page = sg_page(sg); 381 382 get_page(page); 383 skb_fill_page_desc(skb, i, page, sg->offset + page_off, 384 cur_len); 385 skb->data_len += cur_len; 386 skb->len += cur_len; 387 skb->truesize += cur_len; 388 389 data_length -= cur_len; 390 page_off = 0; 391 sg = sg_next(sg); 392 i++; 393 } 394 395 if (data_length) 396 return -1; 397 398 if (padding) { 399 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 400 if (!page) 401 return -1; 402 skb_fill_page_desc(skb, i, page, 0, padding); 403 skb->data_len += padding; 404 skb->len += padding; 405 skb->truesize += padding; 406 } 407 408 return 0; 409 } 410 411 static int 412 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 413 struct iscsi_datain_req *dr) 414 { 415 struct iscsi_conn *conn = csk->conn; 416 struct sk_buff *skb; 417 struct iscsi_datain datain; 418 struct cxgbit_iso_info iso_info; 419 u32 data_length = cmd->se_cmd.data_length; 420 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 421 u32 num_pdu, plen, tx_data = 0; 422 bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 423 SCF_TRANSPORT_TASK_SENSE); 424 bool set_statsn = false; 425 int ret = -1; 426 427 while (data_length) { 428 num_pdu = (data_length + mrdsl - 1) / mrdsl; 429 if (num_pdu > csk->max_iso_npdu) 430 num_pdu = csk->max_iso_npdu; 431 432 plen = num_pdu * mrdsl; 433 if (plen > data_length) 434 plen = data_length; 435 436 skb = __cxgbit_alloc_skb(csk, 0, true); 437 if (unlikely(!skb)) 438 return -ENOMEM; 439 440 memset(skb->data, 0, ISCSI_HDR_LEN); 441 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 442 cxgbit_skcb_submode(skb) |= (csk->submode & 443 CXGBIT_SUBMODE_DCRC); 444 cxgbit_skcb_tx_extralen(skb) = (num_pdu * 445 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 446 ((num_pdu - 1) * ISCSI_HDR_LEN); 447 448 memset(&datain, 0, sizeof(struct iscsi_datain)); 449 memset(&iso_info, 0, sizeof(iso_info)); 450 451 if (!tx_data) 452 iso_info.flags |= CXGBIT_ISO_FSLICE; 453 454 if (!(data_length - plen)) { 455 iso_info.flags |= CXGBIT_ISO_LSLICE; 456 if (!task_sense) { 457 datain.flags = ISCSI_FLAG_DATA_STATUS; 458 iscsit_increment_maxcmdsn(cmd, conn->sess); 459 cmd->stat_sn = conn->stat_sn++; 460 set_statsn = true; 461 } 462 } 463 464 iso_info.burst_len = num_pdu * mrdsl; 465 iso_info.mpdu = mrdsl; 466 iso_info.len = ISCSI_HDR_LEN + plen; 467 468 cxgbit_cpl_tx_data_iso(skb, &iso_info); 469 470 datain.offset = tx_data; 471 datain.data_sn = cmd->data_sn - 1; 472 473 iscsit_build_datain_pdu(cmd, conn, &datain, 474 (struct iscsi_data_rsp *)skb->data, 475 set_statsn); 476 477 ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 478 if (unlikely(ret)) { 479 __kfree_skb(skb); 480 goto out; 481 } 482 483 ret = cxgbit_queue_skb(csk, skb); 484 if (unlikely(ret)) 485 goto out; 486 487 tx_data += plen; 488 data_length -= plen; 489 490 cmd->read_data_done += plen; 491 cmd->data_sn += num_pdu; 492 } 493 494 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 495 496 return 0; 497 498 out: 499 return ret; 500 } 501 502 static int 503 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 504 const struct iscsi_datain *datain) 505 { 506 struct sk_buff *skb; 507 int ret = 0; 508 509 skb = cxgbit_alloc_skb(csk, 0); 510 if (unlikely(!skb)) 511 return -ENOMEM; 512 513 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 514 515 if (datain->length) { 516 cxgbit_skcb_submode(skb) |= (csk->submode & 517 CXGBIT_SUBMODE_DCRC); 518 cxgbit_skcb_tx_extralen(skb) = 519 cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 520 } 521 522 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 523 if (ret < 0) { 524 __kfree_skb(skb); 525 return ret; 526 } 527 528 return cxgbit_queue_skb(csk, skb); 529 } 530 531 static int 532 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 533 struct iscsi_datain_req *dr, 534 const struct iscsi_datain *datain) 535 { 536 struct cxgbit_sock *csk = conn->context; 537 u32 data_length = cmd->se_cmd.data_length; 538 u32 padding = ((-data_length) & 3); 539 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 540 541 if ((data_length > mrdsl) && (!dr->recovery) && 542 (!padding) && (!datain->offset) && csk->max_iso_npdu) { 543 atomic_long_add(data_length - datain->length, 544 &conn->sess->tx_data_octets); 545 return cxgbit_tx_datain_iso(csk, cmd, dr); 546 } 547 548 return cxgbit_tx_datain(csk, cmd, datain); 549 } 550 551 static int 552 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 553 const void *data_buf, u32 data_buf_len) 554 { 555 struct cxgbit_sock *csk = conn->context; 556 struct sk_buff *skb; 557 u32 padding = ((-data_buf_len) & 3); 558 559 skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 560 if (unlikely(!skb)) 561 return -ENOMEM; 562 563 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 564 565 if (data_buf_len) { 566 u32 pad_bytes = 0; 567 568 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 569 570 if (padding) 571 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 572 &pad_bytes, padding); 573 } 574 575 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 576 cxgbit_skcb_submode(skb)]; 577 578 return cxgbit_queue_skb(csk, skb); 579 } 580 581 int 582 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 583 struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 584 { 585 if (dr) 586 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 587 else 588 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 589 } 590 591 int cxgbit_validate_params(struct iscsi_conn *conn) 592 { 593 struct cxgbit_sock *csk = conn->context; 594 struct cxgbit_device *cdev = csk->com.cdev; 595 struct iscsi_param *param; 596 u32 max_xmitdsl; 597 598 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 599 conn->param_list); 600 if (!param) 601 return -1; 602 603 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 604 return -1; 605 606 if (max_xmitdsl > cdev->mdsl) { 607 if (iscsi_change_param_sprintf( 608 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 609 return -1; 610 } 611 612 return 0; 613 } 614 615 static int cxgbit_set_digest(struct cxgbit_sock *csk) 616 { 617 struct iscsi_conn *conn = csk->conn; 618 struct iscsi_param *param; 619 620 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 621 if (!param) { 622 pr_err("param not found key %s\n", HEADERDIGEST); 623 return -1; 624 } 625 626 if (!strcmp(param->value, CRC32C)) 627 csk->submode |= CXGBIT_SUBMODE_HCRC; 628 629 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 630 if (!param) { 631 csk->submode = 0; 632 pr_err("param not found key %s\n", DATADIGEST); 633 return -1; 634 } 635 636 if (!strcmp(param->value, CRC32C)) 637 csk->submode |= CXGBIT_SUBMODE_DCRC; 638 639 if (cxgbit_setup_conn_digest(csk)) { 640 csk->submode = 0; 641 return -1; 642 } 643 644 return 0; 645 } 646 647 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 648 { 649 struct iscsi_conn *conn = csk->conn; 650 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 651 struct iscsi_param *param; 652 u32 mrdsl, mbl; 653 u32 max_npdu, max_iso_npdu; 654 655 if (conn->login->leading_connection) { 656 param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 657 conn->param_list); 658 if (!param) { 659 pr_err("param not found key %s\n", DATASEQUENCEINORDER); 660 return -1; 661 } 662 663 if (strcmp(param->value, YES)) 664 return 0; 665 666 param = iscsi_find_param_from_key(DATAPDUINORDER, 667 conn->param_list); 668 if (!param) { 669 pr_err("param not found key %s\n", DATAPDUINORDER); 670 return -1; 671 } 672 673 if (strcmp(param->value, YES)) 674 return 0; 675 676 param = iscsi_find_param_from_key(MAXBURSTLENGTH, 677 conn->param_list); 678 if (!param) { 679 pr_err("param not found key %s\n", MAXBURSTLENGTH); 680 return -1; 681 } 682 683 if (kstrtou32(param->value, 0, &mbl) < 0) 684 return -1; 685 } else { 686 if (!conn->sess->sess_ops->DataSequenceInOrder) 687 return 0; 688 if (!conn->sess->sess_ops->DataPDUInOrder) 689 return 0; 690 691 mbl = conn->sess->sess_ops->MaxBurstLength; 692 } 693 694 mrdsl = conn_ops->MaxRecvDataSegmentLength; 695 max_npdu = mbl / mrdsl; 696 697 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / 698 (ISCSI_HDR_LEN + mrdsl + 699 cxgbit_digest_len[csk->submode]); 700 701 csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 702 703 if (csk->max_iso_npdu <= 1) 704 csk->max_iso_npdu = 0; 705 706 return 0; 707 } 708 709 static int cxgbit_set_params(struct iscsi_conn *conn) 710 { 711 struct cxgbit_sock *csk = conn->context; 712 struct cxgbit_device *cdev = csk->com.cdev; 713 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 714 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 715 struct iscsi_param *param; 716 u8 erl; 717 718 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 719 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 720 721 if (conn->login->leading_connection) { 722 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 723 conn->param_list); 724 if (!param) { 725 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 726 return -1; 727 } 728 if (kstrtou8(param->value, 0, &erl) < 0) 729 return -1; 730 } else { 731 erl = conn->sess->sess_ops->ErrorRecoveryLevel; 732 } 733 734 if (!erl) { 735 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 736 if (cxgbit_set_iso_npdu(csk)) 737 return -1; 738 } 739 740 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 741 if (cxgbit_setup_conn_pgidx(csk, 742 ppm->tformat.pgsz_idx_dflt)) 743 return -1; 744 set_bit(CSK_DDP_ENABLE, &csk->com.flags); 745 } 746 } 747 748 if (cxgbit_set_digest(csk)) 749 return -1; 750 751 return 0; 752 } 753 754 int 755 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 756 u32 length) 757 { 758 struct cxgbit_sock *csk = conn->context; 759 struct sk_buff *skb; 760 u32 padding_buf = 0; 761 u8 padding = ((-length) & 3); 762 763 skb = cxgbit_alloc_skb(csk, length + padding); 764 if (!skb) 765 return -ENOMEM; 766 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 767 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 768 769 if (padding) 770 skb_store_bits(skb, ISCSI_HDR_LEN + length, 771 &padding_buf, padding); 772 773 if (login->login_complete) { 774 if (cxgbit_set_params(conn)) { 775 kfree_skb(skb); 776 return -1; 777 } 778 779 set_bit(CSK_LOGIN_DONE, &csk->com.flags); 780 } 781 782 if (cxgbit_queue_skb(csk, skb)) 783 return -1; 784 785 if ((!login->login_complete) && (!login->login_failed)) 786 schedule_delayed_work(&conn->login_work, 0); 787 788 return 0; 789 } 790 791 static void 792 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 793 unsigned int nents) 794 { 795 struct skb_seq_state st; 796 const u8 *buf; 797 unsigned int consumed = 0, buf_len; 798 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 799 800 skb_prepare_seq_read(skb, pdu_cb->doffset, 801 pdu_cb->doffset + pdu_cb->dlen, 802 &st); 803 804 while (true) { 805 buf_len = skb_seq_read(consumed, &buf, &st); 806 if (!buf_len) { 807 skb_abort_seq_read(&st); 808 break; 809 } 810 811 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 812 buf_len, consumed); 813 } 814 } 815 816 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 817 { 818 struct iscsi_conn *conn = csk->conn; 819 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 820 struct cxgbit_cmd *ccmd; 821 struct iscsi_cmd *cmd; 822 823 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 824 if (!cmd) { 825 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 826 return NULL; 827 } 828 829 ccmd = iscsit_priv_cmd(cmd); 830 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 831 ccmd->setup_ddp = true; 832 833 return cmd; 834 } 835 836 static int 837 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 838 u32 length) 839 { 840 struct iscsi_conn *conn = cmd->conn; 841 struct cxgbit_sock *csk = conn->context; 842 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 843 844 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 845 pr_err("ImmediateData CRC32C DataDigest error\n"); 846 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 847 pr_err("Unable to recover from" 848 " Immediate Data digest failure while" 849 " in ERL=0.\n"); 850 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 851 (unsigned char *)hdr); 852 return IMMEDIATE_DATA_CANNOT_RECOVER; 853 } 854 855 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 856 (unsigned char *)hdr); 857 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 858 } 859 860 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 861 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 862 struct skb_shared_info *ssi = skb_shinfo(csk->skb); 863 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 864 865 sg_init_table(&ccmd->sg, 1); 866 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 867 dfrag->page_offset); 868 get_page(dfrag->page.p); 869 870 cmd->se_cmd.t_data_sg = &ccmd->sg; 871 cmd->se_cmd.t_data_nents = 1; 872 873 ccmd->release = true; 874 } else { 875 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 876 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 877 878 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); 879 } 880 881 cmd->write_data_done += pdu_cb->dlen; 882 883 if (cmd->write_data_done == cmd->se_cmd.data_length) { 884 spin_lock_bh(&cmd->istate_lock); 885 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 886 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 887 spin_unlock_bh(&cmd->istate_lock); 888 } 889 890 return IMMEDIATE_DATA_NORMAL_OPERATION; 891 } 892 893 static int 894 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 895 bool dump_payload) 896 { 897 struct iscsi_conn *conn = cmd->conn; 898 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 899 /* 900 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 901 */ 902 if (dump_payload) 903 goto after_immediate_data; 904 905 immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 906 cmd->first_burst_len); 907 after_immediate_data: 908 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 909 /* 910 * A PDU/CmdSN carrying Immediate Data passed 911 * DataCRC, check against ExpCmdSN/MaxCmdSN if 912 * Immediate Bit is not set. 913 */ 914 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 915 (unsigned char *)hdr, 916 hdr->cmdsn); 917 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 918 return -1; 919 920 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 921 target_put_sess_cmd(&cmd->se_cmd); 922 return 0; 923 } else if (cmd->unsolicited_data) { 924 iscsit_set_unsoliticed_dataout(cmd); 925 } 926 927 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 928 /* 929 * Immediate Data failed DataCRC and ERL>=1, 930 * silently drop this PDU and let the initiator 931 * plug the CmdSN gap. 932 * 933 * FIXME: Send Unsolicited NOPIN with reserved 934 * TTT here to help the initiator figure out 935 * the missing CmdSN, although they should be 936 * intelligent enough to determine the missing 937 * CmdSN and issue a retry to plug the sequence. 938 */ 939 cmd->i_state = ISTATE_REMOVE; 940 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 941 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 942 return -1; 943 944 return 0; 945 } 946 947 static int 948 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 949 { 950 struct iscsi_conn *conn = csk->conn; 951 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 952 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 953 int rc; 954 bool dump_payload = false; 955 956 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 957 if (rc < 0) 958 return rc; 959 960 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 961 (pdu_cb->nr_dfrags == 1)) 962 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 963 964 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 965 if (rc < 0) 966 return 0; 967 else if (rc > 0) 968 dump_payload = true; 969 970 if (!pdu_cb->dlen) 971 return 0; 972 973 return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 974 } 975 976 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 977 { 978 struct scatterlist *sg_start; 979 struct iscsi_conn *conn = csk->conn; 980 struct iscsi_cmd *cmd = NULL; 981 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 982 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 983 u32 data_offset = be32_to_cpu(hdr->offset); 984 u32 data_len = pdu_cb->dlen; 985 int rc, sg_nents, sg_off; 986 bool dcrc_err = false; 987 988 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 989 if (rc < 0) 990 return rc; 991 else if (!cmd) 992 return 0; 993 994 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 995 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 996 " DataSN: 0x%08x\n", 997 hdr->itt, hdr->offset, data_len, 998 hdr->datasn); 999 1000 dcrc_err = true; 1001 goto check_payload; 1002 } 1003 1004 pr_debug("DataOut data_len: %u, " 1005 "write_data_done: %u, data_length: %u\n", 1006 data_len, cmd->write_data_done, 1007 cmd->se_cmd.data_length); 1008 1009 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1010 sg_off = data_offset / PAGE_SIZE; 1011 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1012 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); 1013 1014 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); 1015 } 1016 1017 check_payload: 1018 1019 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1020 if (rc < 0) 1021 return rc; 1022 1023 return 0; 1024 } 1025 1026 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1027 { 1028 struct iscsi_conn *conn = csk->conn; 1029 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1030 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1031 unsigned char *ping_data = NULL; 1032 u32 payload_length = pdu_cb->dlen; 1033 int ret; 1034 1035 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1036 if (ret < 0) 1037 return 0; 1038 1039 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1040 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1041 pr_err("Unable to recover from" 1042 " NOPOUT Ping DataCRC failure while in" 1043 " ERL=0.\n"); 1044 ret = -1; 1045 goto out; 1046 } else { 1047 /* 1048 * drop this PDU and let the 1049 * initiator plug the CmdSN gap. 1050 */ 1051 pr_info("Dropping NOPOUT" 1052 " Command CmdSN: 0x%08x due to" 1053 " DataCRC error.\n", hdr->cmdsn); 1054 ret = 0; 1055 goto out; 1056 } 1057 } 1058 1059 /* 1060 * Handle NOP-OUT payload for traditional iSCSI sockets 1061 */ 1062 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1063 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1064 if (!ping_data) { 1065 pr_err("Unable to allocate memory for" 1066 " NOPOUT ping data.\n"); 1067 ret = -1; 1068 goto out; 1069 } 1070 1071 skb_copy_bits(csk->skb, pdu_cb->doffset, 1072 ping_data, payload_length); 1073 1074 ping_data[payload_length] = '\0'; 1075 /* 1076 * Attach ping data to struct iscsi_cmd->buf_ptr. 1077 */ 1078 cmd->buf_ptr = ping_data; 1079 cmd->buf_ptr_size = payload_length; 1080 1081 pr_debug("Got %u bytes of NOPOUT ping" 1082 " data.\n", payload_length); 1083 pr_debug("Ping Data: \"%s\"\n", ping_data); 1084 } 1085 1086 return iscsit_process_nop_out(conn, cmd, hdr); 1087 out: 1088 if (cmd) 1089 iscsit_free_cmd(cmd, false); 1090 return ret; 1091 } 1092 1093 static int 1094 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1095 { 1096 struct iscsi_conn *conn = csk->conn; 1097 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1098 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1099 u32 payload_length = pdu_cb->dlen; 1100 int rc; 1101 unsigned char *text_in = NULL; 1102 1103 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1104 if (rc < 0) 1105 return rc; 1106 1107 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1108 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1109 pr_err("Unable to recover from" 1110 " Text Data digest failure while in" 1111 " ERL=0.\n"); 1112 goto reject; 1113 } else { 1114 /* 1115 * drop this PDU and let the 1116 * initiator plug the CmdSN gap. 1117 */ 1118 pr_info("Dropping Text" 1119 " Command CmdSN: 0x%08x due to" 1120 " DataCRC error.\n", hdr->cmdsn); 1121 return 0; 1122 } 1123 } 1124 1125 if (payload_length) { 1126 text_in = kzalloc(payload_length, GFP_KERNEL); 1127 if (!text_in) { 1128 pr_err("Unable to allocate text_in of payload_length: %u\n", 1129 payload_length); 1130 return -ENOMEM; 1131 } 1132 skb_copy_bits(csk->skb, pdu_cb->doffset, 1133 text_in, payload_length); 1134 1135 text_in[payload_length - 1] = '\0'; 1136 1137 cmd->text_in_ptr = text_in; 1138 } 1139 1140 return iscsit_process_text_cmd(conn, cmd, hdr); 1141 1142 reject: 1143 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1144 pdu_cb->hdr); 1145 } 1146 1147 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1148 { 1149 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1150 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1151 struct iscsi_conn *conn = csk->conn; 1152 struct iscsi_cmd *cmd = NULL; 1153 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1154 int ret = -EINVAL; 1155 1156 switch (opcode) { 1157 case ISCSI_OP_SCSI_CMD: 1158 cmd = cxgbit_allocate_cmd(csk); 1159 if (!cmd) 1160 goto reject; 1161 1162 ret = cxgbit_handle_scsi_cmd(csk, cmd); 1163 break; 1164 case ISCSI_OP_SCSI_DATA_OUT: 1165 ret = cxgbit_handle_iscsi_dataout(csk); 1166 break; 1167 case ISCSI_OP_NOOP_OUT: 1168 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1169 cmd = cxgbit_allocate_cmd(csk); 1170 if (!cmd) 1171 goto reject; 1172 } 1173 1174 ret = cxgbit_handle_nop_out(csk, cmd); 1175 break; 1176 case ISCSI_OP_SCSI_TMFUNC: 1177 cmd = cxgbit_allocate_cmd(csk); 1178 if (!cmd) 1179 goto reject; 1180 1181 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1182 (unsigned char *)hdr); 1183 break; 1184 case ISCSI_OP_TEXT: 1185 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1186 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1187 if (!cmd) 1188 goto reject; 1189 } else { 1190 cmd = cxgbit_allocate_cmd(csk); 1191 if (!cmd) 1192 goto reject; 1193 } 1194 1195 ret = cxgbit_handle_text_cmd(csk, cmd); 1196 break; 1197 case ISCSI_OP_LOGOUT: 1198 cmd = cxgbit_allocate_cmd(csk); 1199 if (!cmd) 1200 goto reject; 1201 1202 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1203 if (ret > 0) 1204 wait_for_completion_timeout(&conn->conn_logout_comp, 1205 SECONDS_FOR_LOGOUT_COMP 1206 * HZ); 1207 break; 1208 case ISCSI_OP_SNACK: 1209 ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1210 break; 1211 default: 1212 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1213 dump_stack(); 1214 break; 1215 } 1216 1217 return ret; 1218 1219 reject: 1220 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1221 (unsigned char *)hdr); 1222 return ret; 1223 } 1224 1225 static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1226 { 1227 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1228 struct iscsi_conn *conn = csk->conn; 1229 struct iscsi_hdr *hdr = pdu_cb->hdr; 1230 u8 opcode; 1231 1232 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1233 atomic_long_inc(&conn->sess->conn_digest_errors); 1234 goto transport_err; 1235 } 1236 1237 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1238 goto transport_err; 1239 1240 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1241 1242 if (conn->sess->sess_ops->SessionType && 1243 ((!(opcode & ISCSI_OP_TEXT)) || 1244 (!(opcode & ISCSI_OP_LOGOUT)))) { 1245 pr_err("Received illegal iSCSI Opcode: 0x%02x" 1246 " while in Discovery Session, rejecting.\n", opcode); 1247 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1248 (unsigned char *)hdr); 1249 goto transport_err; 1250 } 1251 1252 if (cxgbit_target_rx_opcode(csk) < 0) 1253 goto transport_err; 1254 1255 return 0; 1256 1257 transport_err: 1258 return -1; 1259 } 1260 1261 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1262 { 1263 struct iscsi_conn *conn = csk->conn; 1264 struct iscsi_login *login = conn->login; 1265 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1266 struct iscsi_login_req *login_req; 1267 1268 login_req = (struct iscsi_login_req *)login->req; 1269 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1270 1271 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1272 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1273 login_req->flags, login_req->itt, login_req->cmdsn, 1274 login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1275 /* 1276 * Setup the initial iscsi_login values from the leading 1277 * login request PDU. 1278 */ 1279 if (login->first_request) { 1280 login_req = (struct iscsi_login_req *)login->req; 1281 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1282 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1283 login_req->flags); 1284 login->version_min = login_req->min_version; 1285 login->version_max = login_req->max_version; 1286 memcpy(login->isid, login_req->isid, 6); 1287 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1288 login->init_task_tag = login_req->itt; 1289 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1290 login->cid = be16_to_cpu(login_req->cid); 1291 login->tsih = be16_to_cpu(login_req->tsih); 1292 } 1293 1294 if (iscsi_target_check_login_request(conn, login) < 0) 1295 return -1; 1296 1297 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1298 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1299 1300 return 0; 1301 } 1302 1303 static int 1304 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1305 { 1306 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1307 int ret; 1308 1309 cxgbit_rx_pdu_cb(skb) = pdu_cb; 1310 1311 csk->skb = skb; 1312 1313 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1314 ret = cxgbit_rx_login_pdu(csk); 1315 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1316 } else { 1317 ret = cxgbit_rx_opcode(csk); 1318 } 1319 1320 return ret; 1321 } 1322 1323 static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1324 { 1325 struct skb_shared_info *ssi = skb_shinfo(skb); 1326 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1327 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1328 u8 i; 1329 1330 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1331 skb, skb->head, skb->data, skb->len, skb->data_len, 1332 ssi->nr_frags); 1333 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1334 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1335 1336 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1337 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1338 "frags %u.\n", 1339 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1340 pdu_cb->ddigest, pdu_cb->frags); 1341 for (i = 0; i < ssi->nr_frags; i++) 1342 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1343 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 1344 } 1345 1346 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1347 { 1348 struct sk_buff *skb = csk->lro_hskb; 1349 struct skb_shared_info *ssi = skb_shinfo(skb); 1350 u8 i; 1351 1352 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1353 for (i = 0; i < ssi->nr_frags; i++) 1354 put_page(skb_frag_page(&ssi->frags[i])); 1355 ssi->nr_frags = 0; 1356 } 1357 1358 static void 1359 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1360 { 1361 struct sk_buff *hskb = csk->lro_hskb; 1362 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1363 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1364 struct skb_shared_info *hssi = skb_shinfo(hskb); 1365 struct skb_shared_info *ssi = skb_shinfo(skb); 1366 unsigned int len = 0; 1367 1368 if (pdu_cb->flags & PDUCBF_RX_HDR) { 1369 hpdu_cb->flags = pdu_cb->flags; 1370 hpdu_cb->seq = pdu_cb->seq; 1371 hpdu_cb->hdr = pdu_cb->hdr; 1372 hpdu_cb->hlen = pdu_cb->hlen; 1373 1374 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx], 1375 sizeof(skb_frag_t)); 1376 1377 get_page(skb_frag_page(&hssi->frags[0])); 1378 hssi->nr_frags = 1; 1379 hpdu_cb->frags = 1; 1380 hpdu_cb->hfrag_idx = 0; 1381 1382 len = hssi->frags[0].size; 1383 hskb->len = len; 1384 hskb->data_len = len; 1385 hskb->truesize = len; 1386 } 1387 1388 if (pdu_cb->flags & PDUCBF_RX_DATA) { 1389 u8 hfrag_idx = 1, i; 1390 1391 hpdu_cb->flags |= pdu_cb->flags; 1392 1393 len = 0; 1394 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) { 1395 memcpy(&hssi->frags[hfrag_idx], 1396 &ssi->frags[pdu_cb->dfrag_idx + i], 1397 sizeof(skb_frag_t)); 1398 1399 get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1400 1401 len += hssi->frags[hfrag_idx].size; 1402 1403 hssi->nr_frags++; 1404 hpdu_cb->frags++; 1405 } 1406 1407 hpdu_cb->dlen = pdu_cb->dlen; 1408 hpdu_cb->doffset = hpdu_cb->hlen; 1409 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1410 hpdu_cb->dfrag_idx = 1; 1411 hskb->len += len; 1412 hskb->data_len += len; 1413 hskb->truesize += len; 1414 } 1415 1416 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1417 hpdu_cb->flags |= pdu_cb->flags; 1418 1419 if (hpdu_cb->flags & PDUCBF_RX_DATA) 1420 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1421 1422 hpdu_cb->ddigest = pdu_cb->ddigest; 1423 hpdu_cb->pdulen = pdu_cb->pdulen; 1424 } 1425 } 1426 1427 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1428 { 1429 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1430 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1431 u8 pdu_idx = 0, last_idx = 0; 1432 int ret = 0; 1433 1434 if (!pdu_cb->complete) { 1435 cxgbit_lro_skb_merge(csk, skb, 0); 1436 1437 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1438 struct sk_buff *hskb = csk->lro_hskb; 1439 1440 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1441 1442 cxgbit_lro_hskb_reset(csk); 1443 1444 if (ret < 0) 1445 goto out; 1446 } 1447 1448 pdu_idx = 1; 1449 } 1450 1451 if (lro_cb->pdu_idx) 1452 last_idx = lro_cb->pdu_idx - 1; 1453 1454 for (; pdu_idx <= last_idx; pdu_idx++) { 1455 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1456 if (ret < 0) 1457 goto out; 1458 } 1459 1460 if ((!lro_cb->complete) && lro_cb->pdu_idx) 1461 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1462 1463 out: 1464 return ret; 1465 } 1466 1467 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1468 { 1469 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1470 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1471 int ret = -1; 1472 1473 if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1474 (pdu_cb->seq != csk->rcv_nxt)) { 1475 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1476 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1477 cxgbit_lro_skb_dump(skb); 1478 return ret; 1479 } 1480 1481 csk->rcv_nxt += lro_cb->pdu_totallen; 1482 1483 ret = cxgbit_process_lro_skb(csk, skb); 1484 1485 csk->rx_credits += lro_cb->pdu_totallen; 1486 1487 if (csk->rx_credits >= (csk->rcv_win / 4)) 1488 cxgbit_rx_data_ack(csk); 1489 1490 return ret; 1491 } 1492 1493 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1494 { 1495 int ret = -1; 1496 1497 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) 1498 ret = cxgbit_rx_lro_skb(csk, skb); 1499 1500 __kfree_skb(skb); 1501 return ret; 1502 } 1503 1504 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1505 { 1506 spin_lock_bh(&csk->rxq.lock); 1507 if (skb_queue_len(&csk->rxq)) { 1508 skb_queue_splice_init(&csk->rxq, rxq); 1509 spin_unlock_bh(&csk->rxq.lock); 1510 return true; 1511 } 1512 spin_unlock_bh(&csk->rxq.lock); 1513 return false; 1514 } 1515 1516 static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1517 { 1518 struct sk_buff *skb; 1519 struct sk_buff_head rxq; 1520 1521 skb_queue_head_init(&rxq); 1522 1523 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1524 1525 if (signal_pending(current)) 1526 goto out; 1527 1528 while ((skb = __skb_dequeue(&rxq))) { 1529 if (cxgbit_rx_skb(csk, skb)) 1530 goto out; 1531 } 1532 1533 return 0; 1534 out: 1535 __skb_queue_purge(&rxq); 1536 return -1; 1537 } 1538 1539 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1540 { 1541 struct cxgbit_sock *csk = conn->context; 1542 int ret = -1; 1543 1544 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1545 ret = cxgbit_wait_rxq(csk); 1546 if (ret) { 1547 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1548 break; 1549 } 1550 } 1551 1552 return ret; 1553 } 1554 1555 void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1556 { 1557 struct cxgbit_sock *csk = conn->context; 1558 1559 while (!kthread_should_stop()) { 1560 iscsit_thread_check_cpumask(conn, current, 0); 1561 if (cxgbit_wait_rxq(csk)) 1562 return; 1563 } 1564 } 1565