1 /* 2 * Copyright (c) 2016 Chelsio Communications, Inc. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #include <linux/workqueue.h> 10 #include <linux/kthread.h> 11 #include <linux/sched/signal.h> 12 13 #include <asm/unaligned.h> 14 #include <net/tcp.h> 15 #include <target/target_core_base.h> 16 #include <target/target_core_fabric.h> 17 #include "cxgbit.h" 18 19 struct sge_opaque_hdr { 20 void *dev; 21 dma_addr_t addr[MAX_SKB_FRAGS + 1]; 22 }; 23 24 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 25 26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 27 sizeof(struct fw_ofld_tx_data_wr)) 28 29 static struct sk_buff * 30 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 31 { 32 struct sk_buff *skb = NULL; 33 u8 submode = 0; 34 int errcode; 35 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 36 37 if (len) { 38 skb = alloc_skb_with_frags(hdr_len, len, 39 0, &errcode, 40 GFP_KERNEL); 41 if (!skb) 42 return NULL; 43 44 skb_reserve(skb, TX_HDR_LEN); 45 skb_reset_transport_header(skb); 46 __skb_put(skb, ISCSI_HDR_LEN); 47 skb->data_len = len; 48 skb->len += len; 49 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 50 51 } else { 52 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 53 54 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 55 if (!skb) 56 return NULL; 57 58 skb_reserve(skb, TX_HDR_LEN + iso_len); 59 skb_reset_transport_header(skb); 60 __skb_put(skb, ISCSI_HDR_LEN); 61 } 62 63 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 64 cxgbit_skcb_submode(skb) = submode; 65 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 66 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 67 return skb; 68 } 69 70 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 71 { 72 return __cxgbit_alloc_skb(csk, len, false); 73 } 74 75 /* 76 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 77 * @skb: the packet 78 * 79 * Returns true if a packet can be sent as an offload WR with immediate 80 * data. We currently use the same limit as for Ethernet packets. 81 */ 82 static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 83 { 84 int length = skb->len; 85 86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 87 length += sizeof(struct fw_ofld_tx_data_wr); 88 89 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 90 length += sizeof(struct cpl_tx_data_iso); 91 92 #define MAX_IMM_TX_PKT_LEN 256 93 return length <= MAX_IMM_TX_PKT_LEN; 94 } 95 96 /* 97 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 98 * @n: the number of SGL entries 99 * Calculates the number of flits needed for a scatter/gather list that 100 * can hold the given number of entries. 101 */ 102 static inline unsigned int cxgbit_sgl_len(unsigned int n) 103 { 104 n--; 105 return (3 * n) / 2 + (n & 1) + 2; 106 } 107 108 /* 109 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 110 * @skb: the packet 111 * 112 * Returns the number of flits needed for the given offload packet. 113 * These packets are already fully constructed and no additional headers 114 * will be added. 115 */ 116 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 117 { 118 unsigned int flits, cnt; 119 120 if (cxgbit_is_ofld_imm(skb)) 121 return DIV_ROUND_UP(skb->len, 8); 122 flits = skb_transport_offset(skb) / 8; 123 cnt = skb_shinfo(skb)->nr_frags; 124 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 125 cnt++; 126 return flits + cxgbit_sgl_len(cnt); 127 } 128 129 #define CXGBIT_ISO_FSLICE 0x1 130 #define CXGBIT_ISO_LSLICE 0x2 131 static void 132 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 133 { 134 struct cpl_tx_data_iso *cpl; 135 unsigned int submode = cxgbit_skcb_submode(skb); 136 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 137 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 138 139 cpl = __skb_push(skb, sizeof(*cpl)); 140 141 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 142 CPL_TX_DATA_ISO_FIRST_V(fslice) | 143 CPL_TX_DATA_ISO_LAST_V(lslice) | 144 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 145 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 146 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 147 CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 148 CPL_TX_DATA_ISO_SCSI_V(2)); 149 150 cpl->ahs_len = 0; 151 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 152 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 153 cpl->len = htonl(iso_info->len); 154 cpl->reserved2_seglen_offset = htonl(0); 155 cpl->datasn_offset = htonl(0); 156 cpl->buffer_offset = htonl(0); 157 cpl->reserved3 = 0; 158 159 __skb_pull(skb, sizeof(*cpl)); 160 } 161 162 static void 163 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 164 u32 len, u32 credits, u32 compl) 165 { 166 struct fw_ofld_tx_data_wr *req; 167 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 168 u32 submode = cxgbit_skcb_submode(skb); 169 u32 wr_ulp_mode = 0; 170 u32 hdr_size = sizeof(*req); 171 u32 opcode = FW_OFLD_TX_DATA_WR; 172 u32 immlen = 0; 173 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) : 174 T6_TX_FORCE_F; 175 176 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 177 opcode = FW_ISCSI_TX_DATA_WR; 178 immlen += sizeof(struct cpl_tx_data_iso); 179 hdr_size += sizeof(struct cpl_tx_data_iso); 180 submode |= 8; 181 } 182 183 if (cxgbit_is_ofld_imm(skb)) 184 immlen += dlen; 185 186 req = __skb_push(skb, hdr_size); 187 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 188 FW_WR_COMPL_V(compl) | 189 FW_WR_IMMDLEN_V(immlen)); 190 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 191 FW_WR_LEN16_V(credits)); 192 req->plen = htonl(len); 193 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 194 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 195 196 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 197 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 198 } 199 200 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 201 { 202 kfree_skb(skb); 203 } 204 205 void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 206 { 207 struct sk_buff *skb; 208 209 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 210 u32 dlen = skb->len; 211 u32 len = skb->len; 212 u32 credits_needed; 213 u32 compl = 0; 214 u32 flowclen16 = 0; 215 u32 iso_cpl_len = 0; 216 217 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 218 iso_cpl_len = sizeof(struct cpl_tx_data_iso); 219 220 if (cxgbit_is_ofld_imm(skb)) 221 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 222 else 223 credits_needed = DIV_ROUND_UP((8 * 224 cxgbit_calc_tx_flits_ofld(skb)) + 225 iso_cpl_len, 16); 226 227 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 228 credits_needed += DIV_ROUND_UP( 229 sizeof(struct fw_ofld_tx_data_wr), 16); 230 /* 231 * Assumes the initial credits is large enough to support 232 * fw_flowc_wr plus largest possible first payload 233 */ 234 235 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 236 flowclen16 = cxgbit_send_tx_flowc_wr(csk); 237 csk->wr_cred -= flowclen16; 238 csk->wr_una_cred += flowclen16; 239 } 240 241 if (csk->wr_cred < credits_needed) { 242 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 243 csk, skb->len, skb->data_len, 244 credits_needed, csk->wr_cred); 245 break; 246 } 247 __skb_unlink(skb, &csk->txq); 248 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 249 skb->csum = (__force __wsum)(credits_needed + flowclen16); 250 csk->wr_cred -= credits_needed; 251 csk->wr_una_cred += credits_needed; 252 253 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 254 csk, skb->len, skb->data_len, credits_needed, 255 csk->wr_cred, csk->wr_una_cred); 256 257 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 258 len += cxgbit_skcb_tx_extralen(skb); 259 260 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 261 (!before(csk->write_seq, 262 csk->snd_una + csk->snd_win))) { 263 compl = 1; 264 csk->wr_una_cred = 0; 265 } 266 267 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 268 compl); 269 csk->snd_nxt += len; 270 271 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 272 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 273 struct cpl_close_con_req *req = 274 (struct cpl_close_con_req *)skb->data; 275 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 276 csk->wr_una_cred = 0; 277 } 278 279 cxgbit_sock_enqueue_wr(csk, skb); 280 t4_set_arp_err_handler(skb, csk, 281 cxgbit_arp_failure_skb_discard); 282 283 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 284 csk, csk->tid, skb, len); 285 286 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 287 } 288 } 289 290 static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 291 { 292 spin_lock_bh(&csk->lock); 293 294 if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 295 csk->lock_owner = true; 296 297 spin_unlock_bh(&csk->lock); 298 299 return csk->lock_owner; 300 } 301 302 static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 303 { 304 struct sk_buff_head backlogq; 305 struct sk_buff *skb; 306 void (*fn)(struct cxgbit_sock *, struct sk_buff *); 307 308 skb_queue_head_init(&backlogq); 309 310 spin_lock_bh(&csk->lock); 311 while (skb_queue_len(&csk->backlogq)) { 312 skb_queue_splice_init(&csk->backlogq, &backlogq); 313 spin_unlock_bh(&csk->lock); 314 315 while ((skb = __skb_dequeue(&backlogq))) { 316 fn = cxgbit_skcb_rx_backlog_fn(skb); 317 fn(csk, skb); 318 } 319 320 spin_lock_bh(&csk->lock); 321 } 322 323 csk->lock_owner = false; 324 spin_unlock_bh(&csk->lock); 325 } 326 327 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 328 { 329 int ret = 0; 330 331 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 332 333 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 334 signal_pending(current))) { 335 __kfree_skb(skb); 336 __skb_queue_purge(&csk->ppodq); 337 ret = -1; 338 spin_lock_bh(&csk->lock); 339 if (csk->lock_owner) { 340 spin_unlock_bh(&csk->lock); 341 goto unlock; 342 } 343 spin_unlock_bh(&csk->lock); 344 return ret; 345 } 346 347 csk->write_seq += skb->len + 348 cxgbit_skcb_tx_extralen(skb); 349 350 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 351 __skb_queue_tail(&csk->txq, skb); 352 cxgbit_push_tx_frames(csk); 353 354 unlock: 355 cxgbit_unlock_sock(csk); 356 return ret; 357 } 358 359 static int 360 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 361 u32 data_length) 362 { 363 u32 i = 0, nr_frags = MAX_SKB_FRAGS; 364 u32 padding = ((-data_length) & 3); 365 struct scatterlist *sg; 366 struct page *page; 367 unsigned int page_off; 368 369 if (padding) 370 nr_frags--; 371 372 /* 373 * We know each entry in t_data_sg contains a page. 374 */ 375 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 376 page_off = (data_offset % PAGE_SIZE); 377 378 while (data_length && (i < nr_frags)) { 379 u32 cur_len = min_t(u32, data_length, sg->length - page_off); 380 381 page = sg_page(sg); 382 383 get_page(page); 384 skb_fill_page_desc(skb, i, page, sg->offset + page_off, 385 cur_len); 386 skb->data_len += cur_len; 387 skb->len += cur_len; 388 skb->truesize += cur_len; 389 390 data_length -= cur_len; 391 page_off = 0; 392 sg = sg_next(sg); 393 i++; 394 } 395 396 if (data_length) 397 return -1; 398 399 if (padding) { 400 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 401 if (!page) 402 return -1; 403 skb_fill_page_desc(skb, i, page, 0, padding); 404 skb->data_len += padding; 405 skb->len += padding; 406 skb->truesize += padding; 407 } 408 409 return 0; 410 } 411 412 static int 413 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 414 struct iscsi_datain_req *dr) 415 { 416 struct iscsi_conn *conn = csk->conn; 417 struct sk_buff *skb; 418 struct iscsi_datain datain; 419 struct cxgbit_iso_info iso_info; 420 u32 data_length = cmd->se_cmd.data_length; 421 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 422 u32 num_pdu, plen, tx_data = 0; 423 bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 424 SCF_TRANSPORT_TASK_SENSE); 425 bool set_statsn = false; 426 int ret = -1; 427 428 while (data_length) { 429 num_pdu = (data_length + mrdsl - 1) / mrdsl; 430 if (num_pdu > csk->max_iso_npdu) 431 num_pdu = csk->max_iso_npdu; 432 433 plen = num_pdu * mrdsl; 434 if (plen > data_length) 435 plen = data_length; 436 437 skb = __cxgbit_alloc_skb(csk, 0, true); 438 if (unlikely(!skb)) 439 return -ENOMEM; 440 441 memset(skb->data, 0, ISCSI_HDR_LEN); 442 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 443 cxgbit_skcb_submode(skb) |= (csk->submode & 444 CXGBIT_SUBMODE_DCRC); 445 cxgbit_skcb_tx_extralen(skb) = (num_pdu * 446 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 447 ((num_pdu - 1) * ISCSI_HDR_LEN); 448 449 memset(&datain, 0, sizeof(struct iscsi_datain)); 450 memset(&iso_info, 0, sizeof(iso_info)); 451 452 if (!tx_data) 453 iso_info.flags |= CXGBIT_ISO_FSLICE; 454 455 if (!(data_length - plen)) { 456 iso_info.flags |= CXGBIT_ISO_LSLICE; 457 if (!task_sense) { 458 datain.flags = ISCSI_FLAG_DATA_STATUS; 459 iscsit_increment_maxcmdsn(cmd, conn->sess); 460 cmd->stat_sn = conn->stat_sn++; 461 set_statsn = true; 462 } 463 } 464 465 iso_info.burst_len = num_pdu * mrdsl; 466 iso_info.mpdu = mrdsl; 467 iso_info.len = ISCSI_HDR_LEN + plen; 468 469 cxgbit_cpl_tx_data_iso(skb, &iso_info); 470 471 datain.offset = tx_data; 472 datain.data_sn = cmd->data_sn - 1; 473 474 iscsit_build_datain_pdu(cmd, conn, &datain, 475 (struct iscsi_data_rsp *)skb->data, 476 set_statsn); 477 478 ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 479 if (unlikely(ret)) { 480 __kfree_skb(skb); 481 goto out; 482 } 483 484 ret = cxgbit_queue_skb(csk, skb); 485 if (unlikely(ret)) 486 goto out; 487 488 tx_data += plen; 489 data_length -= plen; 490 491 cmd->read_data_done += plen; 492 cmd->data_sn += num_pdu; 493 } 494 495 dr->dr_complete = DATAIN_COMPLETE_NORMAL; 496 497 return 0; 498 499 out: 500 return ret; 501 } 502 503 static int 504 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 505 const struct iscsi_datain *datain) 506 { 507 struct sk_buff *skb; 508 int ret = 0; 509 510 skb = cxgbit_alloc_skb(csk, 0); 511 if (unlikely(!skb)) 512 return -ENOMEM; 513 514 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 515 516 if (datain->length) { 517 cxgbit_skcb_submode(skb) |= (csk->submode & 518 CXGBIT_SUBMODE_DCRC); 519 cxgbit_skcb_tx_extralen(skb) = 520 cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 521 } 522 523 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 524 if (ret < 0) { 525 __kfree_skb(skb); 526 return ret; 527 } 528 529 return cxgbit_queue_skb(csk, skb); 530 } 531 532 static int 533 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 534 struct iscsi_datain_req *dr, 535 const struct iscsi_datain *datain) 536 { 537 struct cxgbit_sock *csk = conn->context; 538 u32 data_length = cmd->se_cmd.data_length; 539 u32 padding = ((-data_length) & 3); 540 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 541 542 if ((data_length > mrdsl) && (!dr->recovery) && 543 (!padding) && (!datain->offset) && csk->max_iso_npdu) { 544 atomic_long_add(data_length - datain->length, 545 &conn->sess->tx_data_octets); 546 return cxgbit_tx_datain_iso(csk, cmd, dr); 547 } 548 549 return cxgbit_tx_datain(csk, cmd, datain); 550 } 551 552 static int 553 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 554 const void *data_buf, u32 data_buf_len) 555 { 556 struct cxgbit_sock *csk = conn->context; 557 struct sk_buff *skb; 558 u32 padding = ((-data_buf_len) & 3); 559 560 skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 561 if (unlikely(!skb)) 562 return -ENOMEM; 563 564 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 565 566 if (data_buf_len) { 567 u32 pad_bytes = 0; 568 569 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 570 571 if (padding) 572 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 573 &pad_bytes, padding); 574 } 575 576 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 577 cxgbit_skcb_submode(skb)]; 578 579 return cxgbit_queue_skb(csk, skb); 580 } 581 582 int 583 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 584 struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 585 { 586 if (dr) 587 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 588 else 589 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 590 } 591 592 int cxgbit_validate_params(struct iscsi_conn *conn) 593 { 594 struct cxgbit_sock *csk = conn->context; 595 struct cxgbit_device *cdev = csk->com.cdev; 596 struct iscsi_param *param; 597 u32 max_xmitdsl; 598 599 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 600 conn->param_list); 601 if (!param) 602 return -1; 603 604 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 605 return -1; 606 607 if (max_xmitdsl > cdev->mdsl) { 608 if (iscsi_change_param_sprintf( 609 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 610 return -1; 611 } 612 613 return 0; 614 } 615 616 static int cxgbit_set_digest(struct cxgbit_sock *csk) 617 { 618 struct iscsi_conn *conn = csk->conn; 619 struct iscsi_param *param; 620 621 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 622 if (!param) { 623 pr_err("param not found key %s\n", HEADERDIGEST); 624 return -1; 625 } 626 627 if (!strcmp(param->value, CRC32C)) 628 csk->submode |= CXGBIT_SUBMODE_HCRC; 629 630 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 631 if (!param) { 632 csk->submode = 0; 633 pr_err("param not found key %s\n", DATADIGEST); 634 return -1; 635 } 636 637 if (!strcmp(param->value, CRC32C)) 638 csk->submode |= CXGBIT_SUBMODE_DCRC; 639 640 if (cxgbit_setup_conn_digest(csk)) { 641 csk->submode = 0; 642 return -1; 643 } 644 645 return 0; 646 } 647 648 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 649 { 650 struct iscsi_conn *conn = csk->conn; 651 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 652 struct iscsi_param *param; 653 u32 mrdsl, mbl; 654 u32 max_npdu, max_iso_npdu; 655 u32 max_iso_payload; 656 657 if (conn->login->leading_connection) { 658 param = iscsi_find_param_from_key(MAXBURSTLENGTH, 659 conn->param_list); 660 if (!param) { 661 pr_err("param not found key %s\n", MAXBURSTLENGTH); 662 return -1; 663 } 664 665 if (kstrtou32(param->value, 0, &mbl) < 0) 666 return -1; 667 } else { 668 mbl = conn->sess->sess_ops->MaxBurstLength; 669 } 670 671 mrdsl = conn_ops->MaxRecvDataSegmentLength; 672 max_npdu = mbl / mrdsl; 673 674 max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss); 675 676 max_iso_npdu = max_iso_payload / 677 (ISCSI_HDR_LEN + mrdsl + 678 cxgbit_digest_len[csk->submode]); 679 680 csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 681 682 if (csk->max_iso_npdu <= 1) 683 csk->max_iso_npdu = 0; 684 685 return 0; 686 } 687 688 /* 689 * cxgbit_seq_pdu_inorder() 690 * @csk: pointer to cxgbit socket structure 691 * 692 * This function checks whether data sequence and data 693 * pdu are in order. 694 * 695 * Return: returns -1 on error, 0 if data sequence and 696 * data pdu are in order, 1 if data sequence or data pdu 697 * is not in order. 698 */ 699 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk) 700 { 701 struct iscsi_conn *conn = csk->conn; 702 struct iscsi_param *param; 703 704 if (conn->login->leading_connection) { 705 param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 706 conn->param_list); 707 if (!param) { 708 pr_err("param not found key %s\n", DATASEQUENCEINORDER); 709 return -1; 710 } 711 712 if (strcmp(param->value, YES)) 713 return 1; 714 715 param = iscsi_find_param_from_key(DATAPDUINORDER, 716 conn->param_list); 717 if (!param) { 718 pr_err("param not found key %s\n", DATAPDUINORDER); 719 return -1; 720 } 721 722 if (strcmp(param->value, YES)) 723 return 1; 724 725 } else { 726 if (!conn->sess->sess_ops->DataSequenceInOrder) 727 return 1; 728 if (!conn->sess->sess_ops->DataPDUInOrder) 729 return 1; 730 } 731 732 return 0; 733 } 734 735 static int cxgbit_set_params(struct iscsi_conn *conn) 736 { 737 struct cxgbit_sock *csk = conn->context; 738 struct cxgbit_device *cdev = csk->com.cdev; 739 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 740 struct iscsi_conn_ops *conn_ops = conn->conn_ops; 741 struct iscsi_param *param; 742 u8 erl; 743 744 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 745 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 746 747 if (cxgbit_set_digest(csk)) 748 return -1; 749 750 if (conn->login->leading_connection) { 751 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 752 conn->param_list); 753 if (!param) { 754 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 755 return -1; 756 } 757 if (kstrtou8(param->value, 0, &erl) < 0) 758 return -1; 759 } else { 760 erl = conn->sess->sess_ops->ErrorRecoveryLevel; 761 } 762 763 if (!erl) { 764 int ret; 765 766 ret = cxgbit_seq_pdu_inorder(csk); 767 if (ret < 0) { 768 return -1; 769 } else if (ret > 0) { 770 if (is_t5(cdev->lldi.adapter_type)) 771 goto enable_ddp; 772 else 773 return 0; 774 } 775 776 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 777 if (cxgbit_set_iso_npdu(csk)) 778 return -1; 779 } 780 781 enable_ddp: 782 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 783 if (cxgbit_setup_conn_pgidx(csk, 784 ppm->tformat.pgsz_idx_dflt)) 785 return -1; 786 set_bit(CSK_DDP_ENABLE, &csk->com.flags); 787 } 788 } 789 790 return 0; 791 } 792 793 int 794 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 795 u32 length) 796 { 797 struct cxgbit_sock *csk = conn->context; 798 struct sk_buff *skb; 799 u32 padding_buf = 0; 800 u8 padding = ((-length) & 3); 801 802 skb = cxgbit_alloc_skb(csk, length + padding); 803 if (!skb) 804 return -ENOMEM; 805 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 806 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 807 808 if (padding) 809 skb_store_bits(skb, ISCSI_HDR_LEN + length, 810 &padding_buf, padding); 811 812 if (login->login_complete) { 813 if (cxgbit_set_params(conn)) { 814 kfree_skb(skb); 815 return -1; 816 } 817 818 set_bit(CSK_LOGIN_DONE, &csk->com.flags); 819 } 820 821 if (cxgbit_queue_skb(csk, skb)) 822 return -1; 823 824 if ((!login->login_complete) && (!login->login_failed)) 825 schedule_delayed_work(&conn->login_work, 0); 826 827 return 0; 828 } 829 830 static void 831 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 832 unsigned int nents, u32 skip) 833 { 834 struct skb_seq_state st; 835 const u8 *buf; 836 unsigned int consumed = 0, buf_len; 837 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 838 839 skb_prepare_seq_read(skb, pdu_cb->doffset, 840 pdu_cb->doffset + pdu_cb->dlen, 841 &st); 842 843 while (true) { 844 buf_len = skb_seq_read(consumed, &buf, &st); 845 if (!buf_len) { 846 skb_abort_seq_read(&st); 847 break; 848 } 849 850 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 851 buf_len, skip + consumed); 852 } 853 } 854 855 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 856 { 857 struct iscsi_conn *conn = csk->conn; 858 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 859 struct cxgbit_cmd *ccmd; 860 struct iscsi_cmd *cmd; 861 862 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 863 if (!cmd) { 864 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 865 return NULL; 866 } 867 868 ccmd = iscsit_priv_cmd(cmd); 869 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 870 ccmd->setup_ddp = true; 871 872 return cmd; 873 } 874 875 static int 876 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 877 u32 length) 878 { 879 struct iscsi_conn *conn = cmd->conn; 880 struct cxgbit_sock *csk = conn->context; 881 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 882 883 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 884 pr_err("ImmediateData CRC32C DataDigest error\n"); 885 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 886 pr_err("Unable to recover from" 887 " Immediate Data digest failure while" 888 " in ERL=0.\n"); 889 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 890 (unsigned char *)hdr); 891 return IMMEDIATE_DATA_CANNOT_RECOVER; 892 } 893 894 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 895 (unsigned char *)hdr); 896 return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 897 } 898 899 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 900 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 901 struct skb_shared_info *ssi = skb_shinfo(csk->skb); 902 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 903 904 sg_init_table(&ccmd->sg, 1); 905 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 906 dfrag->page_offset); 907 get_page(dfrag->page.p); 908 909 cmd->se_cmd.t_data_sg = &ccmd->sg; 910 cmd->se_cmd.t_data_nents = 1; 911 912 ccmd->release = true; 913 } else { 914 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 915 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 916 917 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0); 918 } 919 920 cmd->write_data_done += pdu_cb->dlen; 921 922 if (cmd->write_data_done == cmd->se_cmd.data_length) { 923 spin_lock_bh(&cmd->istate_lock); 924 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 925 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 926 spin_unlock_bh(&cmd->istate_lock); 927 } 928 929 return IMMEDIATE_DATA_NORMAL_OPERATION; 930 } 931 932 static int 933 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 934 bool dump_payload) 935 { 936 struct iscsi_conn *conn = cmd->conn; 937 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 938 /* 939 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 940 */ 941 if (dump_payload) 942 goto after_immediate_data; 943 944 immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 945 cmd->first_burst_len); 946 after_immediate_data: 947 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 948 /* 949 * A PDU/CmdSN carrying Immediate Data passed 950 * DataCRC, check against ExpCmdSN/MaxCmdSN if 951 * Immediate Bit is not set. 952 */ 953 cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 954 (unsigned char *)hdr, 955 hdr->cmdsn); 956 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 957 return -1; 958 959 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 960 target_put_sess_cmd(&cmd->se_cmd); 961 return 0; 962 } else if (cmd->unsolicited_data) { 963 iscsit_set_unsolicited_dataout(cmd); 964 } 965 966 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 967 /* 968 * Immediate Data failed DataCRC and ERL>=1, 969 * silently drop this PDU and let the initiator 970 * plug the CmdSN gap. 971 * 972 * FIXME: Send Unsolicited NOPIN with reserved 973 * TTT here to help the initiator figure out 974 * the missing CmdSN, although they should be 975 * intelligent enough to determine the missing 976 * CmdSN and issue a retry to plug the sequence. 977 */ 978 cmd->i_state = ISTATE_REMOVE; 979 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 980 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 981 return -1; 982 983 return 0; 984 } 985 986 static int 987 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 988 { 989 struct iscsi_conn *conn = csk->conn; 990 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 991 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 992 int rc; 993 bool dump_payload = false; 994 995 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 996 if (rc < 0) 997 return rc; 998 999 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 1000 (pdu_cb->nr_dfrags == 1)) 1001 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1002 1003 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1004 if (rc < 0) 1005 return 0; 1006 else if (rc > 0) 1007 dump_payload = true; 1008 1009 if (!pdu_cb->dlen) 1010 return 0; 1011 1012 return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 1013 } 1014 1015 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 1016 { 1017 struct scatterlist *sg_start; 1018 struct iscsi_conn *conn = csk->conn; 1019 struct iscsi_cmd *cmd = NULL; 1020 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1021 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 1022 u32 data_offset = be32_to_cpu(hdr->offset); 1023 u32 data_len = pdu_cb->dlen; 1024 int rc, sg_nents, sg_off; 1025 bool dcrc_err = false; 1026 1027 if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) { 1028 u32 offset = be32_to_cpu(hdr->offset); 1029 u32 ddp_data_len; 1030 u32 payload_length = ntoh24(hdr->dlength); 1031 bool success = false; 1032 1033 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0); 1034 if (!cmd) 1035 return 0; 1036 1037 ddp_data_len = offset - cmd->write_data_done; 1038 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets); 1039 1040 cmd->write_data_done = offset; 1041 cmd->next_burst_len = ddp_data_len; 1042 cmd->data_sn = be32_to_cpu(hdr->datasn); 1043 1044 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, 1045 cmd, payload_length, &success); 1046 if (rc < 0) 1047 return rc; 1048 else if (!success) 1049 return 0; 1050 } else { 1051 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 1052 if (rc < 0) 1053 return rc; 1054 else if (!cmd) 1055 return 0; 1056 } 1057 1058 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1059 pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 1060 " DataSN: 0x%08x\n", 1061 hdr->itt, hdr->offset, data_len, 1062 hdr->datasn); 1063 1064 dcrc_err = true; 1065 goto check_payload; 1066 } 1067 1068 pr_debug("DataOut data_len: %u, " 1069 "write_data_done: %u, data_length: %u\n", 1070 data_len, cmd->write_data_done, 1071 cmd->se_cmd.data_length); 1072 1073 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 1074 u32 skip = data_offset % PAGE_SIZE; 1075 1076 sg_off = data_offset / PAGE_SIZE; 1077 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1078 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE)); 1079 1080 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip); 1081 } 1082 1083 check_payload: 1084 1085 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 1086 if (rc < 0) 1087 return rc; 1088 1089 return 0; 1090 } 1091 1092 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1093 { 1094 struct iscsi_conn *conn = csk->conn; 1095 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1096 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 1097 unsigned char *ping_data = NULL; 1098 u32 payload_length = pdu_cb->dlen; 1099 int ret; 1100 1101 ret = iscsit_setup_nop_out(conn, cmd, hdr); 1102 if (ret < 0) 1103 return 0; 1104 1105 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1106 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1107 pr_err("Unable to recover from" 1108 " NOPOUT Ping DataCRC failure while in" 1109 " ERL=0.\n"); 1110 ret = -1; 1111 goto out; 1112 } else { 1113 /* 1114 * drop this PDU and let the 1115 * initiator plug the CmdSN gap. 1116 */ 1117 pr_info("Dropping NOPOUT" 1118 " Command CmdSN: 0x%08x due to" 1119 " DataCRC error.\n", hdr->cmdsn); 1120 ret = 0; 1121 goto out; 1122 } 1123 } 1124 1125 /* 1126 * Handle NOP-OUT payload for traditional iSCSI sockets 1127 */ 1128 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1129 ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 1130 if (!ping_data) { 1131 pr_err("Unable to allocate memory for" 1132 " NOPOUT ping data.\n"); 1133 ret = -1; 1134 goto out; 1135 } 1136 1137 skb_copy_bits(csk->skb, pdu_cb->doffset, 1138 ping_data, payload_length); 1139 1140 ping_data[payload_length] = '\0'; 1141 /* 1142 * Attach ping data to struct iscsi_cmd->buf_ptr. 1143 */ 1144 cmd->buf_ptr = ping_data; 1145 cmd->buf_ptr_size = payload_length; 1146 1147 pr_debug("Got %u bytes of NOPOUT ping" 1148 " data.\n", payload_length); 1149 pr_debug("Ping Data: \"%s\"\n", ping_data); 1150 } 1151 1152 return iscsit_process_nop_out(conn, cmd, hdr); 1153 out: 1154 if (cmd) 1155 iscsit_free_cmd(cmd, false); 1156 return ret; 1157 } 1158 1159 static int 1160 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 1161 { 1162 struct iscsi_conn *conn = csk->conn; 1163 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1164 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 1165 u32 payload_length = pdu_cb->dlen; 1166 int rc; 1167 unsigned char *text_in = NULL; 1168 1169 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1170 if (rc < 0) 1171 return rc; 1172 1173 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 1174 if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 1175 pr_err("Unable to recover from" 1176 " Text Data digest failure while in" 1177 " ERL=0.\n"); 1178 goto reject; 1179 } else { 1180 /* 1181 * drop this PDU and let the 1182 * initiator plug the CmdSN gap. 1183 */ 1184 pr_info("Dropping Text" 1185 " Command CmdSN: 0x%08x due to" 1186 " DataCRC error.\n", hdr->cmdsn); 1187 return 0; 1188 } 1189 } 1190 1191 if (payload_length) { 1192 text_in = kzalloc(payload_length, GFP_KERNEL); 1193 if (!text_in) { 1194 pr_err("Unable to allocate text_in of payload_length: %u\n", 1195 payload_length); 1196 return -ENOMEM; 1197 } 1198 skb_copy_bits(csk->skb, pdu_cb->doffset, 1199 text_in, payload_length); 1200 1201 text_in[payload_length - 1] = '\0'; 1202 1203 cmd->text_in_ptr = text_in; 1204 } 1205 1206 return iscsit_process_text_cmd(conn, cmd, hdr); 1207 1208 reject: 1209 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 1210 pdu_cb->hdr); 1211 } 1212 1213 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 1214 { 1215 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1216 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 1217 struct iscsi_conn *conn = csk->conn; 1218 struct iscsi_cmd *cmd = NULL; 1219 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1220 int ret = -EINVAL; 1221 1222 switch (opcode) { 1223 case ISCSI_OP_SCSI_CMD: 1224 cmd = cxgbit_allocate_cmd(csk); 1225 if (!cmd) 1226 goto reject; 1227 1228 ret = cxgbit_handle_scsi_cmd(csk, cmd); 1229 break; 1230 case ISCSI_OP_SCSI_DATA_OUT: 1231 ret = cxgbit_handle_iscsi_dataout(csk); 1232 break; 1233 case ISCSI_OP_NOOP_OUT: 1234 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 1235 cmd = cxgbit_allocate_cmd(csk); 1236 if (!cmd) 1237 goto reject; 1238 } 1239 1240 ret = cxgbit_handle_nop_out(csk, cmd); 1241 break; 1242 case ISCSI_OP_SCSI_TMFUNC: 1243 cmd = cxgbit_allocate_cmd(csk); 1244 if (!cmd) 1245 goto reject; 1246 1247 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1248 (unsigned char *)hdr); 1249 break; 1250 case ISCSI_OP_TEXT: 1251 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 1252 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1253 if (!cmd) 1254 goto reject; 1255 } else { 1256 cmd = cxgbit_allocate_cmd(csk); 1257 if (!cmd) 1258 goto reject; 1259 } 1260 1261 ret = cxgbit_handle_text_cmd(csk, cmd); 1262 break; 1263 case ISCSI_OP_LOGOUT: 1264 cmd = cxgbit_allocate_cmd(csk); 1265 if (!cmd) 1266 goto reject; 1267 1268 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1269 if (ret > 0) 1270 wait_for_completion_timeout(&conn->conn_logout_comp, 1271 SECONDS_FOR_LOGOUT_COMP 1272 * HZ); 1273 break; 1274 case ISCSI_OP_SNACK: 1275 ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 1276 break; 1277 default: 1278 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1279 dump_stack(); 1280 break; 1281 } 1282 1283 return ret; 1284 1285 reject: 1286 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1287 (unsigned char *)hdr); 1288 return ret; 1289 } 1290 1291 static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 1292 { 1293 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1294 struct iscsi_conn *conn = csk->conn; 1295 struct iscsi_hdr *hdr = pdu_cb->hdr; 1296 u8 opcode; 1297 1298 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 1299 atomic_long_inc(&conn->sess->conn_digest_errors); 1300 goto transport_err; 1301 } 1302 1303 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 1304 goto transport_err; 1305 1306 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1307 1308 if (conn->sess->sess_ops->SessionType && 1309 ((!(opcode & ISCSI_OP_TEXT)) || 1310 (!(opcode & ISCSI_OP_LOGOUT)))) { 1311 pr_err("Received illegal iSCSI Opcode: 0x%02x" 1312 " while in Discovery Session, rejecting.\n", opcode); 1313 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 1314 (unsigned char *)hdr); 1315 goto transport_err; 1316 } 1317 1318 if (cxgbit_target_rx_opcode(csk) < 0) 1319 goto transport_err; 1320 1321 return 0; 1322 1323 transport_err: 1324 return -1; 1325 } 1326 1327 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 1328 { 1329 struct iscsi_conn *conn = csk->conn; 1330 struct iscsi_login *login = conn->login; 1331 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 1332 struct iscsi_login_req *login_req; 1333 1334 login_req = (struct iscsi_login_req *)login->req; 1335 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 1336 1337 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 1338 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 1339 login_req->flags, login_req->itt, login_req->cmdsn, 1340 login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 1341 /* 1342 * Setup the initial iscsi_login values from the leading 1343 * login request PDU. 1344 */ 1345 if (login->first_request) { 1346 login_req = (struct iscsi_login_req *)login->req; 1347 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1348 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 1349 login_req->flags); 1350 login->version_min = login_req->min_version; 1351 login->version_max = login_req->max_version; 1352 memcpy(login->isid, login_req->isid, 6); 1353 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1354 login->init_task_tag = login_req->itt; 1355 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1356 login->cid = be16_to_cpu(login_req->cid); 1357 login->tsih = be16_to_cpu(login_req->tsih); 1358 } 1359 1360 if (iscsi_target_check_login_request(conn, login) < 0) 1361 return -1; 1362 1363 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 1364 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 1365 1366 return 0; 1367 } 1368 1369 static int 1370 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 1371 { 1372 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 1373 int ret; 1374 1375 cxgbit_rx_pdu_cb(skb) = pdu_cb; 1376 1377 csk->skb = skb; 1378 1379 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 1380 ret = cxgbit_rx_login_pdu(csk); 1381 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1382 } else { 1383 ret = cxgbit_rx_opcode(csk); 1384 } 1385 1386 return ret; 1387 } 1388 1389 static void cxgbit_lro_skb_dump(struct sk_buff *skb) 1390 { 1391 struct skb_shared_info *ssi = skb_shinfo(skb); 1392 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1393 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1394 u8 i; 1395 1396 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 1397 skb, skb->head, skb->data, skb->len, skb->data_len, 1398 ssi->nr_frags); 1399 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 1400 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 1401 1402 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 1403 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 1404 "frags %u.\n", 1405 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 1406 pdu_cb->ddigest, pdu_cb->frags); 1407 for (i = 0; i < ssi->nr_frags; i++) 1408 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 1409 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 1410 } 1411 1412 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 1413 { 1414 struct sk_buff *skb = csk->lro_hskb; 1415 struct skb_shared_info *ssi = skb_shinfo(skb); 1416 u8 i; 1417 1418 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 1419 for (i = 0; i < ssi->nr_frags; i++) 1420 put_page(skb_frag_page(&ssi->frags[i])); 1421 ssi->nr_frags = 0; 1422 skb->data_len = 0; 1423 skb->truesize -= skb->len; 1424 skb->len = 0; 1425 } 1426 1427 static void 1428 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 1429 { 1430 struct sk_buff *hskb = csk->lro_hskb; 1431 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 1432 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 1433 struct skb_shared_info *hssi = skb_shinfo(hskb); 1434 struct skb_shared_info *ssi = skb_shinfo(skb); 1435 unsigned int len = 0; 1436 1437 if (pdu_cb->flags & PDUCBF_RX_HDR) { 1438 u8 hfrag_idx = hssi->nr_frags; 1439 1440 hpdu_cb->flags |= pdu_cb->flags; 1441 hpdu_cb->seq = pdu_cb->seq; 1442 hpdu_cb->hdr = pdu_cb->hdr; 1443 hpdu_cb->hlen = pdu_cb->hlen; 1444 1445 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx], 1446 sizeof(skb_frag_t)); 1447 1448 get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 1449 hssi->nr_frags++; 1450 hpdu_cb->frags++; 1451 hpdu_cb->hfrag_idx = hfrag_idx; 1452 1453 len = hssi->frags[hfrag_idx].size; 1454 hskb->len += len; 1455 hskb->data_len += len; 1456 hskb->truesize += len; 1457 } 1458 1459 if (pdu_cb->flags & PDUCBF_RX_DATA) { 1460 u8 dfrag_idx = hssi->nr_frags, i; 1461 1462 hpdu_cb->flags |= pdu_cb->flags; 1463 hpdu_cb->dfrag_idx = dfrag_idx; 1464 1465 len = 0; 1466 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) { 1467 memcpy(&hssi->frags[dfrag_idx], 1468 &ssi->frags[pdu_cb->dfrag_idx + i], 1469 sizeof(skb_frag_t)); 1470 1471 get_page(skb_frag_page(&hssi->frags[dfrag_idx])); 1472 1473 len += hssi->frags[dfrag_idx].size; 1474 1475 hssi->nr_frags++; 1476 hpdu_cb->frags++; 1477 } 1478 1479 hpdu_cb->dlen = pdu_cb->dlen; 1480 hpdu_cb->doffset = hpdu_cb->hlen; 1481 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 1482 hskb->len += len; 1483 hskb->data_len += len; 1484 hskb->truesize += len; 1485 } 1486 1487 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1488 hpdu_cb->flags |= pdu_cb->flags; 1489 1490 if (hpdu_cb->flags & PDUCBF_RX_DATA) 1491 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 1492 1493 hpdu_cb->ddigest = pdu_cb->ddigest; 1494 hpdu_cb->pdulen = pdu_cb->pdulen; 1495 } 1496 } 1497 1498 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1499 { 1500 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1501 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1502 u8 pdu_idx = 0, last_idx = 0; 1503 int ret = 0; 1504 1505 if (!pdu_cb->complete) { 1506 cxgbit_lro_skb_merge(csk, skb, 0); 1507 1508 if (pdu_cb->flags & PDUCBF_RX_STATUS) { 1509 struct sk_buff *hskb = csk->lro_hskb; 1510 1511 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 1512 1513 cxgbit_lro_hskb_reset(csk); 1514 1515 if (ret < 0) 1516 goto out; 1517 } 1518 1519 pdu_idx = 1; 1520 } 1521 1522 if (lro_cb->pdu_idx) 1523 last_idx = lro_cb->pdu_idx - 1; 1524 1525 for (; pdu_idx <= last_idx; pdu_idx++) { 1526 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 1527 if (ret < 0) 1528 goto out; 1529 } 1530 1531 if ((!lro_cb->complete) && lro_cb->pdu_idx) 1532 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 1533 1534 out: 1535 return ret; 1536 } 1537 1538 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1539 { 1540 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 1541 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 1542 int ret = -1; 1543 1544 if ((pdu_cb->flags & PDUCBF_RX_HDR) && 1545 (pdu_cb->seq != csk->rcv_nxt)) { 1546 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 1547 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 1548 cxgbit_lro_skb_dump(skb); 1549 return ret; 1550 } 1551 1552 csk->rcv_nxt += lro_cb->pdu_totallen; 1553 1554 ret = cxgbit_process_lro_skb(csk, skb); 1555 1556 csk->rx_credits += lro_cb->pdu_totallen; 1557 1558 if (csk->rx_credits >= (csk->rcv_win / 4)) 1559 cxgbit_rx_data_ack(csk); 1560 1561 return ret; 1562 } 1563 1564 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 1565 { 1566 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi; 1567 int ret = -1; 1568 1569 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) { 1570 if (is_t5(lldi->adapter_type)) 1571 ret = cxgbit_rx_lro_skb(csk, skb); 1572 else 1573 ret = cxgbit_process_lro_skb(csk, skb); 1574 } 1575 1576 __kfree_skb(skb); 1577 return ret; 1578 } 1579 1580 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 1581 { 1582 spin_lock_bh(&csk->rxq.lock); 1583 if (skb_queue_len(&csk->rxq)) { 1584 skb_queue_splice_init(&csk->rxq, rxq); 1585 spin_unlock_bh(&csk->rxq.lock); 1586 return true; 1587 } 1588 spin_unlock_bh(&csk->rxq.lock); 1589 return false; 1590 } 1591 1592 static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 1593 { 1594 struct sk_buff *skb; 1595 struct sk_buff_head rxq; 1596 1597 skb_queue_head_init(&rxq); 1598 1599 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 1600 1601 if (signal_pending(current)) 1602 goto out; 1603 1604 while ((skb = __skb_dequeue(&rxq))) { 1605 if (cxgbit_rx_skb(csk, skb)) 1606 goto out; 1607 } 1608 1609 return 0; 1610 out: 1611 __skb_queue_purge(&rxq); 1612 return -1; 1613 } 1614 1615 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 1616 { 1617 struct cxgbit_sock *csk = conn->context; 1618 int ret = -1; 1619 1620 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 1621 ret = cxgbit_wait_rxq(csk); 1622 if (ret) { 1623 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 1624 break; 1625 } 1626 } 1627 1628 return ret; 1629 } 1630 1631 void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 1632 { 1633 struct cxgbit_sock *csk = conn->context; 1634 1635 while (!kthread_should_stop()) { 1636 iscsit_thread_check_cpumask(conn, current, 0); 1637 if (cxgbit_wait_rxq(csk)) 1638 return; 1639 } 1640 } 1641