1 /* 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/workqueue.h> 10 #include <linux/kthread.h> 11 #include <linux/sched/signal.h> 12 13 #include <asm/unaligned.h> 14 #include <net/tcp.h> 15 #include <target/target_core_base.h> 16 #include <target/target_core_fabric.h> 17 #include "cxgbit.h" 18 19 struct sge_opaque_hdr { 20 void *dev; 21 dma_addr_t addr[MAX_SKB_FRAGS + 1]; 22 }; 23 24 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 25 26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 27 sizeof(struct fw_ofld_tx_data_wr)) 28 29 static struct sk_buff * 30 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 31 { 32 struct sk_buff *skb = NULL; 33 u8 submode = 0; 34 int errcode; 35 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 36 37 if (len) { 38 skb = alloc_skb_with_frags(hdr_len, len, 39 0, &errcode, 40 GFP_KERNEL); 41 if (!skb) 42 return NULL; 43 44 skb_reserve(skb, TX_HDR_LEN); 45 skb_reset_transport_header(skb); 46 __skb_put(skb, ISCSI_HDR_LEN); 47 skb->data_len = len; 48 skb->len += len; 49 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 50 51 } else { 52 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 53 54 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 55 if (!skb) 56 return NULL; 57 58 skb_reserve(skb, TX_HDR_LEN + iso_len); 59 skb_reset_transport_header(skb); 60 __skb_put(skb, ISCSI_HDR_LEN); 61 } 62 63 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 64 cxgbit_skcb_submode(skb) = submode; 65 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 66 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 67 return skb; 68 } 69 70 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 71 { 72 return __cxgbit_alloc_skb(csk, len, false); 73 } 74 75 /* 76 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 77 * @skb: the packet 78 * 79 * Returns true if a packet can be sent as an offload WR with immediate 80 * data. We currently use the same limit as for Ethernet packets. 81 */ 82 static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 83 { 84 int length = skb->len; 85 86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 87 length += sizeof(struct fw_ofld_tx_data_wr); 88 89 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 90 length += sizeof(struct cpl_tx_data_iso); 91 92 #define MAX_IMM_TX_PKT_LEN 256 93 return length <= MAX_IMM_TX_PKT_LEN; 94 } 95 96 /* 97 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 98 * @n: the number of SGL entries 99 * Calculates the number of flits needed for a scatter/gather list that 100 * can hold the given number of entries. 101 */ 102 static inline unsigned int cxgbit_sgl_len(unsigned int n) 103 { 104 n--; 105 return (3 * n) / 2 + (n & 1) + 2; 106 } 107 108 /* 109 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 110 * @skb: the packet 111 * 112 * Returns the number of flits needed for the given offload packet. 113 * These packets are already fully constructed and no additional headers 114 * will be added. 115 */ 116 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 117 { 118 unsigned int flits, cnt; 119 120 if (cxgbit_is_ofld_imm(skb)) 121 return DIV_ROUND_UP(skb->len, 8); 122 flits = skb_transport_offset(skb) / 8; 123 cnt = skb_shinfo(skb)->nr_frags; 124 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 125 cnt++; 126 return flits + cxgbit_sgl_len(cnt); 127 } 128 129 #define CXGBIT_ISO_FSLICE 0x1 130 #define CXGBIT_ISO_LSLICE 0x2 131 static void 132 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 133 { 134 struct cpl_tx_data_iso *cpl; 135 unsigned int submode = cxgbit_skcb_submode(skb); 136 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 137 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 138 139 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl)); 140 141 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 142 CPL_TX_DATA_ISO_FIRST_V(fslice) | 143 CPL_TX_DATA_ISO_LAST_V(lslice) | 144 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 145 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 146 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 147 CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 148 CPL_TX_DATA_ISO_SCSI_V(2)); 149 150 cpl->ahs_len = 0; 151 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 152 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 153 cpl->len = htonl(iso_info->len); 154 cpl->reserved2_seglen_offset = htonl(0); 155 cpl->datasn_offset = htonl(0); 156 cpl->buffer_offset = htonl(0); 157 cpl->reserved3 = 0; 158 159 __skb_pull(skb, sizeof(*cpl)); 160 } 161 162 static void 163 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 164 u32 len, u32 credits, u32 compl) 165 { 166 struct fw_ofld_tx_data_wr *req; 167 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 168 u32 submode = cxgbit_skcb_submode(skb); 169 u32 wr_ulp_mode = 0; 170 u32 hdr_size = sizeof(*req); 171 u32 opcode = FW_OFLD_TX_DATA_WR; 172 u32 immlen = 0; 173 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : 174 T6_TX_FORCE_F; 175 176 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 177 opcode = FW_ISCSI_TX_DATA_WR; 178 immlen += sizeof(struct cpl_tx_data_iso); 179 hdr_size += sizeof(struct cpl_tx_data_iso); 180 submode |= 8; 181 } 182 183 if (cxgbit_is_ofld_imm(skb)) 184 immlen += dlen; 185 186 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, 187 hdr_size); 188 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 189 FW_WR_COMPL_V(compl) | 190 FW_WR_IMMDLEN_V(immlen)); 191 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 192 FW_WR_LEN16_V(credits)); 193 req->plen = htonl(len); 194 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 195 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 196 197 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 198 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 199 } 200 201 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 202 { 203 kfree_skb(skb); 204 } 205 206 void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 207 { 208 struct sk_buff *skb; 209 210 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 211 u32 dlen = skb->len; 212 u32 len = skb->len; 213 u32 credits_needed; 214 u32 compl = 0; 215 u32 flowclen16 = 0; 216 u32 iso_cpl_len = 0; 217 218 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 219 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 220 221 if (cxgbit_is_ofld_imm(skb)) 222 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 223 else 224 credits_needed = DIV_ROUND_UP((8 * 225 cxgbit_calc_tx_flits_ofld(skb)) + 226 iso_cpl_len, 16); 227 228 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 229 credits_needed += DIV_ROUND_UP( 230 sizeof(struct fw_ofld_tx_data_wr), 16); 231 /* 232 * Assumes the initial credits is large enough to support 233 * fw_flowc_wr plus largest possible first payload 234 */ 235 236 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 237 flowclen16 = cxgbit_send_tx_flowc_wr(csk); 238 csk->wr_cred -= flowclen16; 239 csk->wr_una_cred += flowclen16; 240 } 241 242 if (csk->wr_cred < credits_needed) { 243 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 244 csk, skb->len, skb->data_len, 245 credits_needed, csk->wr_cred); 246 break; 247 } 248 __skb_unlink(skb, &csk->txq); 249 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 250 skb->csum = (__force __wsum)(credits_needed + flowclen16); 251 csk->wr_cred -= credits_needed; 252 csk->wr_una_cred += credits_needed; 253 254 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 255 csk, skb->len, skb->data_len, credits_needed, 256 csk->wr_cred, csk->wr_una_cred); 257 258 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 259 len += cxgbit_skcb_tx_extralen(skb); 260 261 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 262 (!before(csk->write_seq, 263 csk->snd_una + csk->snd_win))) { 264 compl = 1; 265 csk->wr_una_cred = 0; 266 } 267 268 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 269 compl); 270 csk->snd_nxt += len; 271 272 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 273 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 274 struct cpl_close_con_req *req = 275 (struct cpl_close_con_req *)skb->data; 276 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 277 csk->wr_una_cred = 0; 278 } 279 280 cxgbit_sock_enqueue_wr(csk, skb); 281 t4_set_arp_err_handler(skb, csk, 282 cxgbit_arp_failure_skb_discard); 283 284 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 285 csk, csk->tid, skb, len); 286 287 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 288 } 289 } 290 291 static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 292 { 293 spin_lock_bh(&csk->lock); 294 295 if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 296 csk->lock_owner = true; 297 298 spin_unlock_bh(&csk->lock); 299 300 return csk->lock_owner; 301 } 302 303 static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 304 { 305 struct sk_buff_head backlogq; 306 struct sk_buff *skb; 307 void (*fn)(struct cxgbit_sock *, struct sk_buff *); 308 309 skb_queue_head_init(&backlogq); 310 311 spin_lock_bh(&csk->lock); 312 while (skb_queue_len(&csk->backlogq)) { 313 skb_queue_splice_init(&csk->backlogq, &backlogq); 314 spin_unlock_bh(&csk->lock); 315 316 while ((skb = __skb_dequeue(&backlogq))) { 317 fn = cxgbit_skcb_rx_backlog_fn(skb); 318 fn(csk, skb); 319 } 320 321 spin_lock_bh(&csk->lock); 322 } 323 324 csk->lock_owner = false; 325 spin_unlock_bh(&csk->lock); 326 } 327 328 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 329 { 330 int ret = 0; 331 332 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 333 334 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 335 signal_pending(current))) { 336 __kfree_skb(skb); 337 __skb_queue_purge(&csk->ppodq); 338 ret = -1; 339 spin_lock_bh(&csk->lock); 340 if (csk->lock_owner) { 341 spin_unlock_bh(&csk->lock); 342 goto unlock; 343 } 344 spin_unlock_bh(&csk->lock); 345 return ret; 346 } 347 348 csk->write_seq += skb->len + 349 cxgbit_skcb_tx_extralen(skb); 350 351 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 352 __skb_queue_tail(&csk->txq, skb); 353 cxgbit_push_tx_frames(csk); 354 355 unlock: 356 cxgbit_unlock_sock(csk); 357 return ret; 358 } 359 360 static int 361 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 362 u32 data_length) 363 { 364 u32 i = 0, nr_frags = MAX_SKB_FRAGS; 365 u32 padding = ((-data_length) & 3); 366 struct scatterlist *sg; 367 struct page *page; 368 unsigned int page_off; 369 370 if (padding) 371 nr_frags--; 372 373 /* 374 * We know each entry in t_data_sg contains a page. 375 */ 376 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 377 page_off = (data_offset % PAGE_SIZE); 378 379 while (data_length && (i < nr_frags)) { 380 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 381 382 page = sg_page(sg); 383 384 get_page(page); 385 skb_fill_page_desc(skb, i, page, sg->offset + page_off, 386 cur_len); 387 skb->data_len += cur_len; 388 skb->len += cur_len; 389 skb->truesize += cur_len; 390 391 data_length -= cur_len; 392 page_off = 0; 393 sg = sg_next(sg); 394 i++; 395 } 396 397 if (data_length) 398 return -1; 399 400 if (padding) { 401 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 402 if (!page) 403 return -1; 404 skb_fill_page_desc(skb, i, page, 0, padding); 405 skb->data_len += padding; 406 skb->len += padding; 407 skb->truesize += padding; 408 } 409 410 return 0; 411 } 412 413 static int 414 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 415 struct iscsi_datain_req *dr) 416 { 417 struct iscsi_conn *conn = csk->conn; 418 struct sk_buff *skb; 419 struct iscsi_datain datain; 420 struct cxgbit_iso_info iso_info; 421 u32 data_length = cmd->se_cmd.data_length; 422 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 423 u32 num_pdu, plen, tx_data = 0; 424 bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 425 SCF_TRANSPORT_TASK_SENSE); 426 bool set_statsn = false; 427 int ret = -1; 428 429 while (data_length) { 430 num_pdu = (data_length + mrdsl - 1) / mrdsl; 431 if (num_pdu > csk->max_iso_npdu) 432 num_pdu = csk->max_iso_npdu; 433 434 plen = num_pdu * mrdsl; 435 if (plen > data_length) 436 plen = data_length; 437 438 skb = __cxgbit_alloc_skb(csk, 0, true); 439 if (unlikely(!skb)) 440 return -ENOMEM; 441 442 memset(skb->data, 0, ISCSI_HDR_LEN); 443 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 444 cxgbit_skcb_submode(skb) |= (csk->submode & 445 CXGBIT_SUBMODE_DCRC); 446 cxgbit_skcb_tx_extralen(skb) = (num_pdu * 447 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 448 ((num_pdu - 1) * ISCSI_HDR_LEN); 449 450 memset(&datain, 0, sizeof(struct iscsi_datain)); 451 memset(&iso_info, 0, sizeof(iso_info)); 452 453 if (!tx_data) 454 iso_info.flags |= CXGBIT_ISO_FSLICE; 455 456 if (!(data_length - plen)) { 457 iso_info.flags |= CXGBIT_ISO_LSLICE; 458 if (!task_sense) { 459 datain.flags = ISCSI_FLAG_DATA_STATUS; 460 iscsit_increment_maxcmdsn(cmd, conn->sess); 461 cmd->stat_sn = conn->stat_sn++; 462 set_statsn = true; 463 } 464 } 465 466 iso_info.burst_len = num_pdu * mrdsl; 467 iso_info.mpdu = mrdsl; 468 iso_info.len = ISCSI_HDR_LEN + plen; 469 470 cxgbit_cpl_tx_data_iso(skb, &iso_info); 471 472 datain.offset = tx_data; 473 datain.data_sn = cmd->data_sn - 1; 474 475 iscsit_build_datain_pdu(cmd, conn, &datain, 476 (struct iscsi_data_rsp *)skb->data, 477 set_statsn); 478 479 ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 480 if (unlikely(ret)) { 481 __kfree_skb(skb); 482 goto out; 483 } 484 485 ret = cxgbit_queue_skb(csk, skb); 486 if (unlikely(ret)) 487 goto out; 488 489 tx_data += plen; 490 data_length -= plen; 491 492 cmd->read_data_done += plen; 493 cmd->data_sn += num_pdu; 494 } 495 496 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 497 498 return 0; 499 500 out: 501 return ret; 502 } 503 504 static int 505 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 506 const struct iscsi_datain *datain) 507 { 508 struct sk_buff *skb; 509 int ret = 0; 510 511 skb = cxgbit_alloc_skb(csk, 0); 512 if (unlikely(!skb)) 513 return -ENOMEM; 514 515 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 516 517 if (datain->length) { 518 cxgbit_skcb_submode(skb) |= (csk->submode & 519 CXGBIT_SUBMODE_DCRC); 520 cxgbit_skcb_tx_extralen(skb) = 521 cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 522 } 523 524 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 525 if (ret < 0) { 526 __kfree_skb(skb); 527 return ret; 528 } 529 530 return cxgbit_queue_skb(csk, skb); 531 } 532 533 static int 534 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 535 struct iscsi_datain_req *dr, 536 const struct iscsi_datain *datain) 537 { 538 struct cxgbit_sock *csk = conn->context; 539 u32 data_length = cmd->se_cmd.data_length; 540 u32 padding = ((-data_length) & 3); 541 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 542 543 if ((data_length > mrdsl) && (!dr->recovery) && 544 (!padding) && (!datain->offset) && csk->max_iso_npdu) { 545 atomic_long_add(data_length - datain->length, 546 &conn->sess->tx_data_octets); 547 return cxgbit_tx_datain_iso(csk, cmd, dr); 548 } 549 550 return cxgbit_tx_datain(csk, cmd, datain); 551 } 552 553 static int 554 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 555 const void *data_buf, u32 data_buf_len) 556 { 557 struct cxgbit_sock *csk = conn->context; 558 struct sk_buff *skb; 559 u32 padding = ((-data_buf_len) & 3); 560 561 skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 562 if (unlikely(!skb)) 563 return -ENOMEM; 564 565 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 566 567 if (data_buf_len) { 568 u32 pad_bytes = 0; 569 570 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 571 572 if (padding) 573 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 574 &pad_bytes, padding); 575 } 576 577 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 578 cxgbit_skcb_submode(skb)]; 579 580 return cxgbit_queue_skb(csk, skb); 581 } 582 583 int 584 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 585 struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 586 { 587 if (dr) 588 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 589 else 590 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 591 } 592 593 int cxgbit_validate_params(struct iscsi_conn *conn) 594 { 595 struct cxgbit_sock *csk = conn->context; 596 struct cxgbit_device *cdev = csk->com.cdev; 597 struct iscsi_param *param; 598 u32 max_xmitdsl; 599 600 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 601 conn->param_list); 602 if (!param) 603 return -1; 604 605 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 606 return -1; 607 608 if (max_xmitdsl > cdev->mdsl) { 609 if (iscsi_change_param_sprintf( 610 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 611 return -1; 612 } 613 614 return 0; 615 } 616 617 static int cxgbit_set_digest(struct cxgbit_sock *csk) 618 { 619 struct iscsi_conn *conn = csk->conn; 620 struct iscsi_param *param; 621 622 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 623 if (!param) { 624 pr_err("param not found key %s\n", HEADERDIGEST); 625 return -1; 626 } 627 628 if (!strcmp(param->value, CRC32C)) 629 csk->submode |= CXGBIT_SUBMODE_HCRC; 630 631 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 632 if (!param) { 633 csk->submode = 0; 634 pr_err("param not found key %s\n", DATADIGEST); 635 return -1; 636 } 637 638 if (!strcmp(param->value, CRC32C)) 639 csk->submode |= CXGBIT_SUBMODE_DCRC; 640 641 if (cxgbit_setup_conn_digest(csk)) { 642 csk->submode = 0; 643 return -1; 644 } 645 646 return 0; 647 } 648 649 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 650 { 651 struct iscsi_conn *conn = csk->conn; 652 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 653 struct iscsi_param *param; 654 u32 mrdsl, mbl; 655 u32 max_npdu, max_iso_npdu; 656 657 if (conn->login->leading_connection) { 658 param = iscsi_find_param_from_key(MAXBURSTLENGTH, 659 conn->param_list); 660 if (!param) { 661 pr_err("param not found key %s\n", MAXBURSTLENGTH); 662 return -1; 663 } 664 665 if (kstrtou32(param->value, 0, &mbl) < 0) 666 return -1; 667 } else { 668 mbl = conn->sess->sess_ops->MaxBurstLength; 669 } 670 671 mrdsl = conn_ops->MaxRecvDataSegmentLength; 672 max_npdu = mbl / mrdsl; 673 674 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / 675 (ISCSI_HDR_LEN + mrdsl + 676 cxgbit_digest_len[csk->submode]); 677 678 csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 679 680 if (csk->max_iso_npdu <= 1) 681 csk->max_iso_npdu = 0; 682 683 return 0; 684 } 685 686 /* 687 * cxgbit_seq_pdu_inorder() 688 * @csk: pointer to cxgbit socket structure 689 * 690 * This function checks whether data sequence and data 691 * pdu are in order. 692 * 693 * Return: returns -1 on error, 0 if data sequence and 694 * data pdu are in order, 1 if data sequence or data pdu 695 * is not in order. 696 */ 697 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk) 698 { 699 struct iscsi_conn *conn = csk->conn; 700 struct iscsi_param *param; 701 702 if (conn->login->leading_connection) { 703 param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 704 conn->param_list); 705 if (!param) { 706 pr_err("param not found key %s\n", DATASEQUENCEINORDER); 707 return -1; 708 } 709 710 if (strcmp(param->value, YES)) 711 return 1; 712 713 param = iscsi_find_param_from_key(DATAPDUINORDER, 714 conn->param_list); 715 if (!param) { 716 pr_err("param not found key %s\n", DATAPDUINORDER); 717 return -1; 718 } 719 720 if (strcmp(param->value, YES)) 721 return 1; 722 723 } else { 724 if (!conn->sess->sess_ops->DataSequenceInOrder) 725 return 1; 726 if (!conn->sess->sess_ops->DataPDUInOrder) 727 return 1; 728 } 729 730 return 0; 731 } 732 733 static int cxgbit_set_params(struct iscsi_conn *conn) 734 { 735 struct cxgbit_sock *csk = conn->context; 736 struct cxgbit_device *cdev = csk->com.cdev; 737 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 738 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 739 struct iscsi_param *param; 740 u8 erl; 741 742 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 743 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 744 745 if (conn->login->leading_connection) { 746 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 747 conn->param_list); 748 if (!param) { 749 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 750 return -1; 751 } 752 if (kstrtou8(param->value, 0, &erl) < 0) 753 return -1; 754 } else { 755 erl = conn->sess->sess_ops->ErrorRecoveryLevel; 756 } 757 758 if (!erl) { 759 int ret; 760 761 ret = cxgbit_seq_pdu_inorder(csk); 762 if (ret < 0) { 763 return -1; 764 } else if (ret > 0) { 765 if (is_t5(cdev->lldi.adapter_type)) 766 goto enable_ddp; 767 else 768 goto enable_digest; 769 } 770 771 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 772 if (cxgbit_set_iso_npdu(csk)) 773 return -1; 774 } 775 776 enable_ddp: 777 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 778 if (cxgbit_setup_conn_pgidx(csk, 779 ppm->tformat.pgsz_idx_dflt)) 780 return -1; 781 set_bit(CSK_DDP_ENABLE, &csk->com.flags); 782 } 783 } 784 785 enable_digest: 786 if (cxgbit_set_digest(csk)) 787 return -1; 788 789 return 0; 790 } 791 792 int 793 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 794 u32 length) 795 { 796 struct cxgbit_sock *csk = conn->context; 797 struct sk_buff *skb; 798 u32 padding_buf = 0; 799 u8 padding = ((-length) & 3); 800 801 skb = cxgbit_alloc_skb(csk, length + padding); 802 if (!skb) 803 return -ENOMEM; 804 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 805 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 806 807 if (padding) 808 skb_store_bits(skb, ISCSI_HDR_LEN + length, 809 &padding_buf, padding); 810 811 if (login->login_complete) { 812 if (cxgbit_set_params(conn)) { 813 kfree_skb(skb); 814 return -1; 815 } 816 817 set_bit(CSK_LOGIN_DONE, &csk->com.flags); 818 } 819 820 if (cxgbit_queue_skb(csk, skb)) 821 return -1; 822 823 if ((!login->login_complete) && (!login->login_failed)) 824 schedule_delayed_work(&conn->login_work, 0); 825 826 return 0; 827 } 828 829 static void 830 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 831 unsigned int nents) 832 { 833 struct skb_seq_state st; 834 const u8 *buf; 835 unsigned int consumed = 0, buf_len; 836 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 837 838 skb_prepare_seq_read(skb, pdu_cb->doffset, 839 pdu_cb->doffset + pdu_cb->dlen, 840 &st); 841 842 while (true) { 843 buf_len = skb_seq_read(consumed, &buf, &st); 844 if (!buf_len) { 845 skb_abort_seq_read(&st); 846 break; 847 } 848 849 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 850 buf_len, consumed); 851 } 852 } 853 854 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 855 { 856 struct iscsi_conn *conn = csk->conn; 857 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 858 struct cxgbit_cmd *ccmd; 859 struct iscsi_cmd *cmd; 860 861 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 862 if (!cmd) { 863 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 864 return NULL; 865 } 866 867 ccmd = iscsit_priv_cmd(cmd); 868 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 869 ccmd->setup_ddp = true; 870 871 return cmd; 872 } 873 874 static int 875 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 876 u32 length) 877 { 878 struct iscsi_conn *conn = cmd->conn; 879 struct cxgbit_sock *csk = conn->context; 880 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 881 882 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 883 pr_err("ImmediateData CRC32C DataDigest error\n"); 884 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 885 pr_err("Unable to recover from" 886 " Immediate Data digest failure while" 887 " in ERL=0.\n"); 888 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 889 (unsigned char *)hdr); 890 return IMMEDIATE_DATA_CANNOT_RECOVER; 891 } 892 893 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 894 (unsigned char *)hdr); 895 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 896 } 897 898 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 899 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 900 struct skb_shared_info *ssi = skb_shinfo(csk->skb); 901 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 902 903 sg_init_table(&ccmd->sg, 1); 904 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 905 dfrag->page_offset); 906 get_page(dfrag->page.p); 907 908 cmd->se_cmd.t_data_sg = &ccmd->sg; 909 cmd->se_cmd.t_data_nents = 1; 910 911 ccmd->release = true; 912 } else { 913 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 914 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 915 916 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); 917 } 918 919 cmd->write_data_done += pdu_cb->dlen; 920 921 if (cmd->write_data_done == cmd->se_cmd.data_length) { 922 spin_lock_bh(&cmd->istate_lock); 923 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 924 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 925 spin_unlock_bh(&cmd->istate_lock); 926 } 927 928 return IMMEDIATE_DATA_NORMAL_OPERATION; 929 } 930 931 static int 932 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 933 bool dump_payload) 934 { 935 struct iscsi_conn *conn = cmd->conn; 936 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 937 /* 938 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 939 */ 940 if (dump_payload) 941 goto after_immediate_data; 942 943 immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 944 cmd->first_burst_len); 945 after_immediate_data: 946 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 947 /* 948 * A PDU/CmdSN carrying Immediate Data passed 949 * DataCRC, check against ExpCmdSN/MaxCmdSN if 950 * Immediate Bit is not set. 951 */ 952 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 953 (unsigned char *)hdr, 954 hdr->cmdsn); 955 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 956 return -1; 957 958 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 959 target_put_sess_cmd(&cmd->se_cmd); 960 return 0; 961 } else if (cmd->unsolicited_data) { 962 iscsit_set_unsoliticed_dataout(cmd); 963 } 964 965 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 966 /* 967 * Immediate Data failed DataCRC and ERL>=1, 968 * silently drop this PDU and let the initiator 969 * plug the CmdSN gap. 970 * 971 * FIXME: Send Unsolicited NOPIN with reserved 972 * TTT here to help the initiator figure out 973 * the missing CmdSN, although they should be 974 * intelligent enough to determine the missing 975 * CmdSN and issue a retry to plug the sequence. 976 */ 977 cmd->i_state = ISTATE_REMOVE; 978 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 979 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 980 return -1; 981 982 return 0; 983 } 984 985 static int 986 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 987 { 988 struct iscsi_conn *conn = csk->conn; 989 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 990 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 991 int rc; 992 bool dump_payload = false; 993 994 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 995 if (rc < 0) 996 return rc; 997 998 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 999 (pdu_cb->nr_dfrags == 1)) 1000 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1001 1002 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1003 if (rc < 0) 1004 return 0; 1005 else if (rc > 0) 1006 dump_payload = true; 1007 1008 if (!pdu_cb->dlen) 1009 return 0; 1010 1011 return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 1012 } 1013 1014 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 1015 { 1016 struct scatterlist *sg_start; 1017 struct iscsi_conn *conn = csk->conn; 1018 struct iscsi_cmd *cmd = NULL; 1019 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1020 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 1021 u32 data_offset = be32_to_cpu(hdr->offset); 1022 u32 data_len = pdu_cb->dlen; 1023 int rc, sg_nents, sg_off; 1024 bool dcrc_err = false; 1025 1026 if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) { 1027 u32 offset = be32_to_cpu(hdr->offset); 1028 u32 ddp_data_len; 1029 u32 payload_length = ntoh24(hdr->dlength); 1030 bool success = false; 1031 1032 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0); 1033 if (!cmd) 1034 return 0; 1035 1036 ddp_data_len = offset - cmd->write_data_done; 1037 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets); 1038 1039 cmd->write_data_done = offset; 1040 cmd->next_burst_len = ddp_data_len; 1041 cmd->data_sn = be32_to_cpu(hdr->datasn); 1042 1043 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, 1044 cmd, payload_length, &success); 1045 if (rc < 0) 1046 return rc; 1047 else if (!success) 1048 return 0; 1049 } else { 1050 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 1051 if (rc < 0) 1052 return rc; 1053 else if (!cmd) 1054 return 0; 1055 } 1056 1057 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1058 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1059 " DataSN: 0x%08x\n", 1060 hdr->itt, hdr->offset, data_len, 1061 hdr->datasn); 1062 1063 dcrc_err = true; 1064 goto check_payload; 1065 } 1066 1067 pr_debug("DataOut data_len: %u, " 1068 "write_data_done: %u, data_length: %u\n", 1069 data_len, cmd->write_data_done, 1070 cmd->se_cmd.data_length); 1071 1072 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1073 sg_off = data_offset / PAGE_SIZE; 1074 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1075 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); 1076 1077 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); 1078 } 1079 1080 check_payload: 1081 1082 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1083 if (rc < 0) 1084 return rc; 1085 1086 return 0; 1087 } 1088 1089 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1090 { 1091 struct iscsi_conn *conn = csk->conn; 1092 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1093 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1094 unsigned char *ping_data = NULL; 1095 u32 payload_length = pdu_cb->dlen; 1096 int ret; 1097 1098 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1099 if (ret < 0) 1100 return 0; 1101 1102 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1103 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1104 pr_err("Unable to recover from" 1105 " NOPOUT Ping DataCRC failure while in" 1106 " ERL=0.\n"); 1107 ret = -1; 1108 goto out; 1109 } else { 1110 /* 1111 * drop this PDU and let the 1112 * initiator plug the CmdSN gap. 1113 */ 1114 pr_info("Dropping NOPOUT" 1115 " Command CmdSN: 0x%08x due to" 1116 " DataCRC error.\n", hdr->cmdsn); 1117 ret = 0; 1118 goto out; 1119 } 1120 } 1121 1122 /* 1123 * Handle NOP-OUT payload for traditional iSCSI sockets 1124 */ 1125 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1126 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1127 if (!ping_data) { 1128 pr_err("Unable to allocate memory for" 1129 " NOPOUT ping data.\n"); 1130 ret = -1; 1131 goto out; 1132 } 1133 1134 skb_copy_bits(csk->skb, pdu_cb->doffset, 1135 ping_data, payload_length); 1136 1137 ping_data[payload_length] = '\0'; 1138 /* 1139 * Attach ping data to struct iscsi_cmd->buf_ptr. 1140 */ 1141 cmd->buf_ptr = ping_data; 1142 cmd->buf_ptr_size = payload_length; 1143 1144 pr_debug("Got %u bytes of NOPOUT ping" 1145 " data.\n", payload_length); 1146 pr_debug("Ping Data: \"%s\"\n", ping_data); 1147 } 1148 1149 return iscsit_process_nop_out(conn, cmd, hdr); 1150 out: 1151 if (cmd) 1152 iscsit_free_cmd(cmd, false); 1153 return ret; 1154 } 1155 1156 static int 1157 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1158 { 1159 struct iscsi_conn *conn = csk->conn; 1160 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1161 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1162 u32 payload_length = pdu_cb->dlen; 1163 int rc; 1164 unsigned char *text_in = NULL; 1165 1166 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1167 if (rc < 0) 1168 return rc; 1169 1170 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1171 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1172 pr_err("Unable to recover from" 1173 " Text Data digest failure while in" 1174 " ERL=0.\n"); 1175 goto reject; 1176 } else { 1177 /* 1178 * drop this PDU and let the 1179 * initiator plug the CmdSN gap. 1180 */ 1181 pr_info("Dropping Text" 1182 " Command CmdSN: 0x%08x due to" 1183 " DataCRC error.\n", hdr->cmdsn); 1184 return 0; 1185 } 1186 } 1187 1188 if (payload_length) { 1189 text_in = kzalloc(payload_length, GFP_KERNEL); 1190 if (!text_in) { 1191 pr_err("Unable to allocate text_in of payload_length: %u\n", 1192 payload_length); 1193 return -ENOMEM; 1194 } 1195 skb_copy_bits(csk->skb, pdu_cb->doffset, 1196 text_in, payload_length); 1197 1198 text_in[payload_length - 1] = '\0'; 1199 1200 cmd->text_in_ptr = text_in; 1201 } 1202 1203 return iscsit_process_text_cmd(conn, cmd, hdr); 1204 1205 reject: 1206 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1207 pdu_cb->hdr); 1208 } 1209 1210 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1211 { 1212 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1213 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1214 struct iscsi_conn *conn = csk->conn; 1215 struct iscsi_cmd *cmd = NULL; 1216 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1217 int ret = -EINVAL; 1218 1219 switch (opcode) { 1220 case ISCSI_OP_SCSI_CMD: 1221 cmd = cxgbit_allocate_cmd(csk); 1222 if (!cmd) 1223 goto reject; 1224 1225 ret = cxgbit_handle_scsi_cmd(csk, cmd); 1226 break; 1227 case ISCSI_OP_SCSI_DATA_OUT: 1228 ret = cxgbit_handle_iscsi_dataout(csk); 1229 break; 1230 case ISCSI_OP_NOOP_OUT: 1231 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1232 cmd = cxgbit_allocate_cmd(csk); 1233 if (!cmd) 1234 goto reject; 1235 } 1236 1237 ret = cxgbit_handle_nop_out(csk, cmd); 1238 break; 1239 case ISCSI_OP_SCSI_TMFUNC: 1240 cmd = cxgbit_allocate_cmd(csk); 1241 if (!cmd) 1242 goto reject; 1243 1244 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1245 (unsigned char *)hdr); 1246 break; 1247 case ISCSI_OP_TEXT: 1248 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1249 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1250 if (!cmd) 1251 goto reject; 1252 } else { 1253 cmd = cxgbit_allocate_cmd(csk); 1254 if (!cmd) 1255 goto reject; 1256 } 1257 1258 ret = cxgbit_handle_text_cmd(csk, cmd); 1259 break; 1260 case ISCSI_OP_LOGOUT: 1261 cmd = cxgbit_allocate_cmd(csk); 1262 if (!cmd) 1263 goto reject; 1264 1265 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1266 if (ret > 0) 1267 wait_for_completion_timeout(&conn->conn_logout_comp, 1268 SECONDS_FOR_LOGOUT_COMP 1269 * HZ); 1270 break; 1271 case ISCSI_OP_SNACK: 1272 ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1273 break; 1274 default: 1275 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1276 dump_stack(); 1277 break; 1278 } 1279 1280 return ret; 1281 1282 reject: 1283 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1284 (unsigned char *)hdr); 1285 return ret; 1286 } 1287 1288 static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1289 { 1290 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1291 struct iscsi_conn *conn = csk->conn; 1292 struct iscsi_hdr *hdr = pdu_cb->hdr; 1293 u8 opcode; 1294 1295 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1296 atomic_long_inc(&conn->sess->conn_digest_errors); 1297 goto transport_err; 1298 } 1299 1300 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1301 goto transport_err; 1302 1303 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1304 1305 if (conn->sess->sess_ops->SessionType && 1306 ((!(opcode & ISCSI_OP_TEXT)) || 1307 (!(opcode & ISCSI_OP_LOGOUT)))) { 1308 pr_err("Received illegal iSCSI Opcode: 0x%02x" 1309 " while in Discovery Session, rejecting.\n", opcode); 1310 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1311 (unsigned char *)hdr); 1312 goto transport_err; 1313 } 1314 1315 if (cxgbit_target_rx_opcode(csk) < 0) 1316 goto transport_err; 1317 1318 return 0; 1319 1320 transport_err: 1321 return -1; 1322 } 1323 1324 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1325 { 1326 struct iscsi_conn *conn = csk->conn; 1327 struct iscsi_login *login = conn->login; 1328 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1329 struct iscsi_login_req *login_req; 1330 1331 login_req = (struct iscsi_login_req *)login->req; 1332 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1333 1334 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1335 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1336 login_req->flags, login_req->itt, login_req->cmdsn, 1337 login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1338 /* 1339 * Setup the initial iscsi_login values from the leading 1340 * login request PDU. 1341 */ 1342 if (login->first_request) { 1343 login_req = (struct iscsi_login_req *)login->req; 1344 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1345 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1346 login_req->flags); 1347 login->version_min = login_req->min_version; 1348 login->version_max = login_req->max_version; 1349 memcpy(login->isid, login_req->isid, 6); 1350 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1351 login->init_task_tag = login_req->itt; 1352 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1353 login->cid = be16_to_cpu(login_req->cid); 1354 login->tsih = be16_to_cpu(login_req->tsih); 1355 } 1356 1357 if (iscsi_target_check_login_request(conn, login) < 0) 1358 return -1; 1359 1360 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1361 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1362 1363 return 0; 1364 } 1365 1366 static int 1367 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1368 { 1369 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1370 int ret; 1371 1372 cxgbit_rx_pdu_cb(skb) = pdu_cb; 1373 1374 csk->skb = skb; 1375 1376 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1377 ret = cxgbit_rx_login_pdu(csk); 1378 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1379 } else { 1380 ret = cxgbit_rx_opcode(csk); 1381 } 1382 1383 return ret; 1384 } 1385 1386 static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1387 { 1388 struct skb_shared_info *ssi = skb_shinfo(skb); 1389 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1390 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1391 u8 i; 1392 1393 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1394 skb, skb->head, skb->data, skb->len, skb->data_len, 1395 ssi->nr_frags); 1396 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1397 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1398 1399 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1400 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1401 "frags %u.\n", 1402 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1403 pdu_cb->ddigest, pdu_cb->frags); 1404 for (i = 0; i < ssi->nr_frags; i++) 1405 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1406 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 1407 } 1408 1409 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1410 { 1411 struct sk_buff *skb = csk->lro_hskb; 1412 struct skb_shared_info *ssi = skb_shinfo(skb); 1413 u8 i; 1414 1415 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1416 for (i = 0; i < ssi->nr_frags; i++) 1417 put_page(skb_frag_page(&ssi->frags[i])); 1418 ssi->nr_frags = 0; 1419 skb->data_len = 0; 1420 skb->truesize -= skb->len; 1421 skb->len = 0; 1422 } 1423 1424 static void 1425 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1426 { 1427 struct sk_buff *hskb = csk->lro_hskb; 1428 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1429 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1430 struct skb_shared_info *hssi = skb_shinfo(hskb); 1431 struct skb_shared_info *ssi = skb_shinfo(skb); 1432 unsigned int len = 0; 1433 1434 if (pdu_cb->flags & PDUCBF_RX_HDR) { 1435 u8 hfrag_idx = hssi->nr_frags; 1436 1437 hpdu_cb->flags |= pdu_cb->flags; 1438 hpdu_cb->seq = pdu_cb->seq; 1439 hpdu_cb->hdr = pdu_cb->hdr; 1440 hpdu_cb->hlen = pdu_cb->hlen; 1441 1442 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx], 1443 sizeof(skb_frag_t)); 1444 1445 get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1446 hssi->nr_frags++; 1447 hpdu_cb->frags++; 1448 hpdu_cb->hfrag_idx = hfrag_idx; 1449 1450 len = hssi->frags[hfrag_idx].size; 1451 hskb->len += len; 1452 hskb->data_len += len; 1453 hskb->truesize += len; 1454 } 1455 1456 if (pdu_cb->flags & PDUCBF_RX_DATA) { 1457 u8 dfrag_idx = hssi->nr_frags, i; 1458 1459 hpdu_cb->flags |= pdu_cb->flags; 1460 hpdu_cb->dfrag_idx = dfrag_idx; 1461 1462 len = 0; 1463 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) { 1464 memcpy(&hssi->frags[dfrag_idx], 1465 &ssi->frags[pdu_cb->dfrag_idx + i], 1466 sizeof(skb_frag_t)); 1467 1468 get_page(skb_frag_page(&hssi->frags[dfrag_idx])); 1469 1470 len += hssi->frags[dfrag_idx].size; 1471 1472 hssi->nr_frags++; 1473 hpdu_cb->frags++; 1474 } 1475 1476 hpdu_cb->dlen = pdu_cb->dlen; 1477 hpdu_cb->doffset = hpdu_cb->hlen; 1478 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1479 hskb->len += len; 1480 hskb->data_len += len; 1481 hskb->truesize += len; 1482 } 1483 1484 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1485 hpdu_cb->flags |= pdu_cb->flags; 1486 1487 if (hpdu_cb->flags & PDUCBF_RX_DATA) 1488 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1489 1490 hpdu_cb->ddigest = pdu_cb->ddigest; 1491 hpdu_cb->pdulen = pdu_cb->pdulen; 1492 } 1493 } 1494 1495 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1496 { 1497 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1498 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1499 u8 pdu_idx = 0, last_idx = 0; 1500 int ret = 0; 1501 1502 if (!pdu_cb->complete) { 1503 cxgbit_lro_skb_merge(csk, skb, 0); 1504 1505 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1506 struct sk_buff *hskb = csk->lro_hskb; 1507 1508 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1509 1510 cxgbit_lro_hskb_reset(csk); 1511 1512 if (ret < 0) 1513 goto out; 1514 } 1515 1516 pdu_idx = 1; 1517 } 1518 1519 if (lro_cb->pdu_idx) 1520 last_idx = lro_cb->pdu_idx - 1; 1521 1522 for (; pdu_idx <= last_idx; pdu_idx++) { 1523 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1524 if (ret < 0) 1525 goto out; 1526 } 1527 1528 if ((!lro_cb->complete) && lro_cb->pdu_idx) 1529 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1530 1531 out: 1532 return ret; 1533 } 1534 1535 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1536 { 1537 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1538 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1539 int ret = -1; 1540 1541 if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1542 (pdu_cb->seq != csk->rcv_nxt)) { 1543 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1544 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1545 cxgbit_lro_skb_dump(skb); 1546 return ret; 1547 } 1548 1549 csk->rcv_nxt += lro_cb->pdu_totallen; 1550 1551 ret = cxgbit_process_lro_skb(csk, skb); 1552 1553 csk->rx_credits += lro_cb->pdu_totallen; 1554 1555 if (csk->rx_credits >= (csk->rcv_win / 4)) 1556 cxgbit_rx_data_ack(csk); 1557 1558 return ret; 1559 } 1560 1561 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1562 { 1563 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 1564 int ret = -1; 1565 1566 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) { 1567 if (is_t5(lldi->adapter_type)) 1568 ret = cxgbit_rx_lro_skb(csk, skb); 1569 else 1570 ret = cxgbit_process_lro_skb(csk, skb); 1571 } 1572 1573 __kfree_skb(skb); 1574 return ret; 1575 } 1576 1577 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1578 { 1579 spin_lock_bh(&csk->rxq.lock); 1580 if (skb_queue_len(&csk->rxq)) { 1581 skb_queue_splice_init(&csk->rxq, rxq); 1582 spin_unlock_bh(&csk->rxq.lock); 1583 return true; 1584 } 1585 spin_unlock_bh(&csk->rxq.lock); 1586 return false; 1587 } 1588 1589 static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1590 { 1591 struct sk_buff *skb; 1592 struct sk_buff_head rxq; 1593 1594 skb_queue_head_init(&rxq); 1595 1596 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1597 1598 if (signal_pending(current)) 1599 goto out; 1600 1601 while ((skb = __skb_dequeue(&rxq))) { 1602 if (cxgbit_rx_skb(csk, skb)) 1603 goto out; 1604 } 1605 1606 return 0; 1607 out: 1608 __skb_queue_purge(&rxq); 1609 return -1; 1610 } 1611 1612 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1613 { 1614 struct cxgbit_sock *csk = conn->context; 1615 int ret = -1; 1616 1617 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1618 ret = cxgbit_wait_rxq(csk); 1619 if (ret) { 1620 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1621 break; 1622 } 1623 } 1624 1625 return ret; 1626 } 1627 1628 void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1629 { 1630 struct cxgbit_sock *csk = conn->context; 1631 1632 while (!kthread_should_stop()) { 1633 iscsit_thread_check_cpumask(conn, current, 0); 1634 if (cxgbit_wait_rxq(csk)) 1635 return; 1636 } 1637 } 1638