19730ffcbSVarun Prakash /* 29730ffcbSVarun Prakash * Copyright (c) 2016 Chelsio Communications, Inc. 39730ffcbSVarun Prakash * 49730ffcbSVarun Prakash * This program is free software; you can redistribute it and/or modify 59730ffcbSVarun Prakash * it under the terms of the GNU General Public License version 2 as 69730ffcbSVarun Prakash * published by the Free Software Foundation. 79730ffcbSVarun Prakash */ 89730ffcbSVarun Prakash 99730ffcbSVarun Prakash #include <linux/workqueue.h> 109730ffcbSVarun Prakash #include <linux/kthread.h> 11174cd4b1SIngo Molnar #include <linux/sched/signal.h> 12174cd4b1SIngo Molnar 139730ffcbSVarun Prakash #include <asm/unaligned.h> 143bc71e1fSBart Van Assche #include <net/tcp.h> 159730ffcbSVarun Prakash #include <target/target_core_base.h> 169730ffcbSVarun Prakash #include <target/target_core_fabric.h> 179730ffcbSVarun Prakash #include "cxgbit.h" 189730ffcbSVarun Prakash 199730ffcbSVarun Prakash struct sge_opaque_hdr { 209730ffcbSVarun Prakash void *dev; 219730ffcbSVarun Prakash dma_addr_t addr[MAX_SKB_FRAGS + 1]; 229730ffcbSVarun Prakash }; 239730ffcbSVarun Prakash 249730ffcbSVarun Prakash static const u8 cxgbit_digest_len[] = {0, 4, 4, 8}; 259730ffcbSVarun Prakash 269730ffcbSVarun Prakash #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \ 279730ffcbSVarun Prakash sizeof(struct fw_ofld_tx_data_wr)) 289730ffcbSVarun Prakash 299730ffcbSVarun Prakash static struct sk_buff * 309730ffcbSVarun Prakash __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso) 319730ffcbSVarun Prakash { 329730ffcbSVarun Prakash struct sk_buff *skb = NULL; 339730ffcbSVarun Prakash u8 submode = 0; 349730ffcbSVarun Prakash int errcode; 359730ffcbSVarun Prakash static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN; 369730ffcbSVarun Prakash 379730ffcbSVarun Prakash if (len) { 389730ffcbSVarun Prakash skb = alloc_skb_with_frags(hdr_len, len, 399730ffcbSVarun Prakash 0, &errcode, 409730ffcbSVarun Prakash GFP_KERNEL); 419730ffcbSVarun Prakash if (!skb) 429730ffcbSVarun Prakash return NULL; 439730ffcbSVarun Prakash 449730ffcbSVarun Prakash skb_reserve(skb, TX_HDR_LEN); 459730ffcbSVarun Prakash skb_reset_transport_header(skb); 469730ffcbSVarun Prakash __skb_put(skb, ISCSI_HDR_LEN); 479730ffcbSVarun Prakash skb->data_len = len; 489730ffcbSVarun Prakash skb->len += len; 499730ffcbSVarun Prakash submode |= (csk->submode & CXGBIT_SUBMODE_DCRC); 509730ffcbSVarun Prakash 519730ffcbSVarun Prakash } else { 529730ffcbSVarun Prakash u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0; 539730ffcbSVarun Prakash 549730ffcbSVarun Prakash skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL); 559730ffcbSVarun Prakash if (!skb) 569730ffcbSVarun Prakash return NULL; 579730ffcbSVarun Prakash 589730ffcbSVarun Prakash skb_reserve(skb, TX_HDR_LEN + iso_len); 599730ffcbSVarun Prakash skb_reset_transport_header(skb); 609730ffcbSVarun Prakash __skb_put(skb, ISCSI_HDR_LEN); 619730ffcbSVarun Prakash } 629730ffcbSVarun Prakash 639730ffcbSVarun Prakash submode |= (csk->submode & CXGBIT_SUBMODE_HCRC); 649730ffcbSVarun Prakash cxgbit_skcb_submode(skb) = submode; 659730ffcbSVarun Prakash cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode]; 669730ffcbSVarun Prakash cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR; 679730ffcbSVarun Prakash return skb; 689730ffcbSVarun Prakash } 699730ffcbSVarun Prakash 709730ffcbSVarun Prakash static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len) 719730ffcbSVarun Prakash { 729730ffcbSVarun Prakash return __cxgbit_alloc_skb(csk, len, false); 739730ffcbSVarun Prakash } 749730ffcbSVarun Prakash 759730ffcbSVarun Prakash /* 769730ffcbSVarun Prakash * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data 779730ffcbSVarun Prakash * @skb: the packet 789730ffcbSVarun Prakash * 799730ffcbSVarun Prakash * Returns true if a packet can be sent as an offload WR with immediate 809730ffcbSVarun Prakash * data. We currently use the same limit as for Ethernet packets. 819730ffcbSVarun Prakash */ 829730ffcbSVarun Prakash static int cxgbit_is_ofld_imm(const struct sk_buff *skb) 839730ffcbSVarun Prakash { 849730ffcbSVarun Prakash int length = skb->len; 859730ffcbSVarun Prakash 869730ffcbSVarun Prakash if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 879730ffcbSVarun Prakash length += sizeof(struct fw_ofld_tx_data_wr); 889730ffcbSVarun Prakash 899730ffcbSVarun Prakash if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)) 909730ffcbSVarun Prakash length += sizeof(struct cpl_tx_data_iso); 919730ffcbSVarun Prakash 929730ffcbSVarun Prakash #define MAX_IMM_TX_PKT_LEN 256 939730ffcbSVarun Prakash return length <= MAX_IMM_TX_PKT_LEN; 949730ffcbSVarun Prakash } 959730ffcbSVarun Prakash 969730ffcbSVarun Prakash /* 979730ffcbSVarun Prakash * cxgbit_sgl_len - calculates the size of an SGL of the given capacity 989730ffcbSVarun Prakash * @n: the number of SGL entries 999730ffcbSVarun Prakash * Calculates the number of flits needed for a scatter/gather list that 1009730ffcbSVarun Prakash * can hold the given number of entries. 1019730ffcbSVarun Prakash */ 1029730ffcbSVarun Prakash static inline unsigned int cxgbit_sgl_len(unsigned int n) 1039730ffcbSVarun Prakash { 1049730ffcbSVarun Prakash n--; 1059730ffcbSVarun Prakash return (3 * n) / 2 + (n & 1) + 2; 1069730ffcbSVarun Prakash } 1079730ffcbSVarun Prakash 1089730ffcbSVarun Prakash /* 1099730ffcbSVarun Prakash * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet 1109730ffcbSVarun Prakash * @skb: the packet 1119730ffcbSVarun Prakash * 1129730ffcbSVarun Prakash * Returns the number of flits needed for the given offload packet. 1139730ffcbSVarun Prakash * These packets are already fully constructed and no additional headers 1149730ffcbSVarun Prakash * will be added. 1159730ffcbSVarun Prakash */ 1169730ffcbSVarun Prakash static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb) 1179730ffcbSVarun Prakash { 1189730ffcbSVarun Prakash unsigned int flits, cnt; 1199730ffcbSVarun Prakash 1209730ffcbSVarun Prakash if (cxgbit_is_ofld_imm(skb)) 1219730ffcbSVarun Prakash return DIV_ROUND_UP(skb->len, 8); 1229730ffcbSVarun Prakash flits = skb_transport_offset(skb) / 8; 1239730ffcbSVarun Prakash cnt = skb_shinfo(skb)->nr_frags; 1249730ffcbSVarun Prakash if (skb_tail_pointer(skb) != skb_transport_header(skb)) 1259730ffcbSVarun Prakash cnt++; 1269730ffcbSVarun Prakash return flits + cxgbit_sgl_len(cnt); 1279730ffcbSVarun Prakash } 1289730ffcbSVarun Prakash 1299730ffcbSVarun Prakash #define CXGBIT_ISO_FSLICE 0x1 1309730ffcbSVarun Prakash #define CXGBIT_ISO_LSLICE 0x2 1319730ffcbSVarun Prakash static void 1329730ffcbSVarun Prakash cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info) 1339730ffcbSVarun Prakash { 1349730ffcbSVarun Prakash struct cpl_tx_data_iso *cpl; 1359730ffcbSVarun Prakash unsigned int submode = cxgbit_skcb_submode(skb); 1369730ffcbSVarun Prakash unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE); 1379730ffcbSVarun Prakash unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE); 1389730ffcbSVarun Prakash 1399730ffcbSVarun Prakash cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl)); 1409730ffcbSVarun Prakash 1419730ffcbSVarun Prakash cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) | 1429730ffcbSVarun Prakash CPL_TX_DATA_ISO_FIRST_V(fslice) | 1439730ffcbSVarun Prakash CPL_TX_DATA_ISO_LAST_V(lslice) | 1449730ffcbSVarun Prakash CPL_TX_DATA_ISO_CPLHDRLEN_V(0) | 1459730ffcbSVarun Prakash CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) | 1469730ffcbSVarun Prakash CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) | 1479730ffcbSVarun Prakash CPL_TX_DATA_ISO_IMMEDIATE_V(0) | 1489730ffcbSVarun Prakash CPL_TX_DATA_ISO_SCSI_V(2)); 1499730ffcbSVarun Prakash 1509730ffcbSVarun Prakash cpl->ahs_len = 0; 1519730ffcbSVarun Prakash cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4)); 1529730ffcbSVarun Prakash cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4)); 1539730ffcbSVarun Prakash cpl->len = htonl(iso_info->len); 1549730ffcbSVarun Prakash cpl->reserved2_seglen_offset = htonl(0); 1559730ffcbSVarun Prakash cpl->datasn_offset = htonl(0); 1569730ffcbSVarun Prakash cpl->buffer_offset = htonl(0); 1579730ffcbSVarun Prakash cpl->reserved3 = 0; 1589730ffcbSVarun Prakash 1599730ffcbSVarun Prakash __skb_pull(skb, sizeof(*cpl)); 1609730ffcbSVarun Prakash } 1619730ffcbSVarun Prakash 1629730ffcbSVarun Prakash static void 1639730ffcbSVarun Prakash cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen, 1649730ffcbSVarun Prakash u32 len, u32 credits, u32 compl) 1659730ffcbSVarun Prakash { 1669730ffcbSVarun Prakash struct fw_ofld_tx_data_wr *req; 1679730ffcbSVarun Prakash u32 submode = cxgbit_skcb_submode(skb); 1689730ffcbSVarun Prakash u32 wr_ulp_mode = 0; 1699730ffcbSVarun Prakash u32 hdr_size = sizeof(*req); 1709730ffcbSVarun Prakash u32 opcode = FW_OFLD_TX_DATA_WR; 1719730ffcbSVarun Prakash u32 immlen = 0; 1729730ffcbSVarun Prakash u32 force = TX_FORCE_V(!submode); 1739730ffcbSVarun Prakash 1749730ffcbSVarun Prakash if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) { 1759730ffcbSVarun Prakash opcode = FW_ISCSI_TX_DATA_WR; 1769730ffcbSVarun Prakash immlen += sizeof(struct cpl_tx_data_iso); 1779730ffcbSVarun Prakash hdr_size += sizeof(struct cpl_tx_data_iso); 1789730ffcbSVarun Prakash submode |= 8; 1799730ffcbSVarun Prakash } 1809730ffcbSVarun Prakash 1819730ffcbSVarun Prakash if (cxgbit_is_ofld_imm(skb)) 1829730ffcbSVarun Prakash immlen += dlen; 1839730ffcbSVarun Prakash 1849730ffcbSVarun Prakash req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, 1859730ffcbSVarun Prakash hdr_size); 1869730ffcbSVarun Prakash req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) | 1879730ffcbSVarun Prakash FW_WR_COMPL_V(compl) | 1889730ffcbSVarun Prakash FW_WR_IMMDLEN_V(immlen)); 1899730ffcbSVarun Prakash req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 1909730ffcbSVarun Prakash FW_WR_LEN16_V(credits)); 1919730ffcbSVarun Prakash req->plen = htonl(len); 1929730ffcbSVarun Prakash wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) | 1939730ffcbSVarun Prakash FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 1949730ffcbSVarun Prakash 1959730ffcbSVarun Prakash req->tunnel_to_proxy = htonl((wr_ulp_mode) | force | 1969730ffcbSVarun Prakash FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1)); 1979730ffcbSVarun Prakash } 1989730ffcbSVarun Prakash 1999730ffcbSVarun Prakash static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb) 2009730ffcbSVarun Prakash { 2019730ffcbSVarun Prakash kfree_skb(skb); 2029730ffcbSVarun Prakash } 2039730ffcbSVarun Prakash 2049730ffcbSVarun Prakash void cxgbit_push_tx_frames(struct cxgbit_sock *csk) 2059730ffcbSVarun Prakash { 2069730ffcbSVarun Prakash struct sk_buff *skb; 2079730ffcbSVarun Prakash 2089730ffcbSVarun Prakash while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) { 2099730ffcbSVarun Prakash u32 dlen = skb->len; 2109730ffcbSVarun Prakash u32 len = skb->len; 2119730ffcbSVarun Prakash u32 credits_needed; 2129730ffcbSVarun Prakash u32 compl = 0; 2139730ffcbSVarun Prakash u32 flowclen16 = 0; 2149730ffcbSVarun Prakash u32 iso_cpl_len = 0; 2159730ffcbSVarun Prakash 2169730ffcbSVarun Prakash if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) 2179730ffcbSVarun Prakash iso_cpl_len = sizeof(struct cpl_tx_data_iso); 2189730ffcbSVarun Prakash 2199730ffcbSVarun Prakash if (cxgbit_is_ofld_imm(skb)) 2209730ffcbSVarun Prakash credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16); 2219730ffcbSVarun Prakash else 2229730ffcbSVarun Prakash credits_needed = DIV_ROUND_UP((8 * 2239730ffcbSVarun Prakash cxgbit_calc_tx_flits_ofld(skb)) + 2249730ffcbSVarun Prakash iso_cpl_len, 16); 2259730ffcbSVarun Prakash 2269730ffcbSVarun Prakash if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) 2279730ffcbSVarun Prakash credits_needed += DIV_ROUND_UP( 2289730ffcbSVarun Prakash sizeof(struct fw_ofld_tx_data_wr), 16); 2299730ffcbSVarun Prakash /* 2309730ffcbSVarun Prakash * Assumes the initial credits is large enough to support 2319730ffcbSVarun Prakash * fw_flowc_wr plus largest possible first payload 2329730ffcbSVarun Prakash */ 2339730ffcbSVarun Prakash 2349730ffcbSVarun Prakash if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) { 2359730ffcbSVarun Prakash flowclen16 = cxgbit_send_tx_flowc_wr(csk); 2369730ffcbSVarun Prakash csk->wr_cred -= flowclen16; 2379730ffcbSVarun Prakash csk->wr_una_cred += flowclen16; 2389730ffcbSVarun Prakash } 2399730ffcbSVarun Prakash 2409730ffcbSVarun Prakash if (csk->wr_cred < credits_needed) { 2419730ffcbSVarun Prakash pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n", 2429730ffcbSVarun Prakash csk, skb->len, skb->data_len, 2439730ffcbSVarun Prakash credits_needed, csk->wr_cred); 2449730ffcbSVarun Prakash break; 2459730ffcbSVarun Prakash } 2469730ffcbSVarun Prakash __skb_unlink(skb, &csk->txq); 2479730ffcbSVarun Prakash set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx); 2489730ffcbSVarun Prakash skb->csum = credits_needed + flowclen16; 2499730ffcbSVarun Prakash csk->wr_cred -= credits_needed; 2509730ffcbSVarun Prakash csk->wr_una_cred += credits_needed; 2519730ffcbSVarun Prakash 2529730ffcbSVarun Prakash pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 2539730ffcbSVarun Prakash csk, skb->len, skb->data_len, credits_needed, 2549730ffcbSVarun Prakash csk->wr_cred, csk->wr_una_cred); 2559730ffcbSVarun Prakash 2569730ffcbSVarun Prakash if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) { 2579730ffcbSVarun Prakash len += cxgbit_skcb_tx_extralen(skb); 2589730ffcbSVarun Prakash 2599730ffcbSVarun Prakash if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) || 2609730ffcbSVarun Prakash (!before(csk->write_seq, 2619730ffcbSVarun Prakash csk->snd_una + csk->snd_win))) { 2629730ffcbSVarun Prakash compl = 1; 2639730ffcbSVarun Prakash csk->wr_una_cred = 0; 2649730ffcbSVarun Prakash } 2659730ffcbSVarun Prakash 2669730ffcbSVarun Prakash cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed, 2679730ffcbSVarun Prakash compl); 2689730ffcbSVarun Prakash csk->snd_nxt += len; 2699730ffcbSVarun Prakash 2709730ffcbSVarun Prakash } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) || 2719730ffcbSVarun Prakash (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 2729730ffcbSVarun Prakash struct cpl_close_con_req *req = 2739730ffcbSVarun Prakash (struct cpl_close_con_req *)skb->data; 2749730ffcbSVarun Prakash req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 2759730ffcbSVarun Prakash csk->wr_una_cred = 0; 2769730ffcbSVarun Prakash } 2779730ffcbSVarun Prakash 2789730ffcbSVarun Prakash cxgbit_sock_enqueue_wr(csk, skb); 2799730ffcbSVarun Prakash t4_set_arp_err_handler(skb, csk, 2809730ffcbSVarun Prakash cxgbit_arp_failure_skb_discard); 2819730ffcbSVarun Prakash 2829730ffcbSVarun Prakash pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n", 2839730ffcbSVarun Prakash csk, csk->tid, skb, len); 2849730ffcbSVarun Prakash 2859730ffcbSVarun Prakash cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t); 2869730ffcbSVarun Prakash } 2879730ffcbSVarun Prakash } 2889730ffcbSVarun Prakash 2899730ffcbSVarun Prakash static bool cxgbit_lock_sock(struct cxgbit_sock *csk) 2909730ffcbSVarun Prakash { 2919730ffcbSVarun Prakash spin_lock_bh(&csk->lock); 2929730ffcbSVarun Prakash 2939730ffcbSVarun Prakash if (before(csk->write_seq, csk->snd_una + csk->snd_win)) 2949730ffcbSVarun Prakash csk->lock_owner = true; 2959730ffcbSVarun Prakash 2969730ffcbSVarun Prakash spin_unlock_bh(&csk->lock); 2979730ffcbSVarun Prakash 2989730ffcbSVarun Prakash return csk->lock_owner; 2999730ffcbSVarun Prakash } 3009730ffcbSVarun Prakash 3019730ffcbSVarun Prakash static void cxgbit_unlock_sock(struct cxgbit_sock *csk) 3029730ffcbSVarun Prakash { 3039730ffcbSVarun Prakash struct sk_buff_head backlogq; 3049730ffcbSVarun Prakash struct sk_buff *skb; 3059730ffcbSVarun Prakash void (*fn)(struct cxgbit_sock *, struct sk_buff *); 3069730ffcbSVarun Prakash 3079730ffcbSVarun Prakash skb_queue_head_init(&backlogq); 3089730ffcbSVarun Prakash 3099730ffcbSVarun Prakash spin_lock_bh(&csk->lock); 3109730ffcbSVarun Prakash while (skb_queue_len(&csk->backlogq)) { 3119730ffcbSVarun Prakash skb_queue_splice_init(&csk->backlogq, &backlogq); 3129730ffcbSVarun Prakash spin_unlock_bh(&csk->lock); 3139730ffcbSVarun Prakash 3149730ffcbSVarun Prakash while ((skb = __skb_dequeue(&backlogq))) { 3159730ffcbSVarun Prakash fn = cxgbit_skcb_rx_backlog_fn(skb); 3169730ffcbSVarun Prakash fn(csk, skb); 3179730ffcbSVarun Prakash } 3189730ffcbSVarun Prakash 3199730ffcbSVarun Prakash spin_lock_bh(&csk->lock); 3209730ffcbSVarun Prakash } 3219730ffcbSVarun Prakash 3229730ffcbSVarun Prakash csk->lock_owner = false; 3239730ffcbSVarun Prakash spin_unlock_bh(&csk->lock); 3249730ffcbSVarun Prakash } 3259730ffcbSVarun Prakash 3269730ffcbSVarun Prakash static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 3279730ffcbSVarun Prakash { 3289730ffcbSVarun Prakash int ret = 0; 3299730ffcbSVarun Prakash 3309730ffcbSVarun Prakash wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk)); 3319730ffcbSVarun Prakash 3329730ffcbSVarun Prakash if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) || 3339730ffcbSVarun Prakash signal_pending(current))) { 3349730ffcbSVarun Prakash __kfree_skb(skb); 3359730ffcbSVarun Prakash __skb_queue_purge(&csk->ppodq); 3369730ffcbSVarun Prakash ret = -1; 3379730ffcbSVarun Prakash spin_lock_bh(&csk->lock); 3389730ffcbSVarun Prakash if (csk->lock_owner) { 3399730ffcbSVarun Prakash spin_unlock_bh(&csk->lock); 3409730ffcbSVarun Prakash goto unlock; 3419730ffcbSVarun Prakash } 3429730ffcbSVarun Prakash spin_unlock_bh(&csk->lock); 3439730ffcbSVarun Prakash return ret; 3449730ffcbSVarun Prakash } 3459730ffcbSVarun Prakash 3469730ffcbSVarun Prakash csk->write_seq += skb->len + 3479730ffcbSVarun Prakash cxgbit_skcb_tx_extralen(skb); 3489730ffcbSVarun Prakash 3499730ffcbSVarun Prakash skb_queue_splice_tail_init(&csk->ppodq, &csk->txq); 3509730ffcbSVarun Prakash __skb_queue_tail(&csk->txq, skb); 3519730ffcbSVarun Prakash cxgbit_push_tx_frames(csk); 3529730ffcbSVarun Prakash 3539730ffcbSVarun Prakash unlock: 3549730ffcbSVarun Prakash cxgbit_unlock_sock(csk); 3559730ffcbSVarun Prakash return ret; 3569730ffcbSVarun Prakash } 3579730ffcbSVarun Prakash 3589730ffcbSVarun Prakash static int 3599730ffcbSVarun Prakash cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset, 3609730ffcbSVarun Prakash u32 data_length) 3619730ffcbSVarun Prakash { 3629730ffcbSVarun Prakash u32 i = 0, nr_frags = MAX_SKB_FRAGS; 3639730ffcbSVarun Prakash u32 padding = ((-data_length) & 3); 3649730ffcbSVarun Prakash struct scatterlist *sg; 3659730ffcbSVarun Prakash struct page *page; 3669730ffcbSVarun Prakash unsigned int page_off; 3679730ffcbSVarun Prakash 3689730ffcbSVarun Prakash if (padding) 3699730ffcbSVarun Prakash nr_frags--; 3709730ffcbSVarun Prakash 3719730ffcbSVarun Prakash /* 3729730ffcbSVarun Prakash * We know each entry in t_data_sg contains a page. 3739730ffcbSVarun Prakash */ 3749730ffcbSVarun Prakash sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE]; 3759730ffcbSVarun Prakash page_off = (data_offset % PAGE_SIZE); 3769730ffcbSVarun Prakash 3779730ffcbSVarun Prakash while (data_length && (i < nr_frags)) { 3789730ffcbSVarun Prakash u32 cur_len = min_t(u32, data_length, sg->length - page_off); 3799730ffcbSVarun Prakash 3809730ffcbSVarun Prakash page = sg_page(sg); 3819730ffcbSVarun Prakash 3829730ffcbSVarun Prakash get_page(page); 3839730ffcbSVarun Prakash skb_fill_page_desc(skb, i, page, sg->offset + page_off, 3849730ffcbSVarun Prakash cur_len); 3859730ffcbSVarun Prakash skb->data_len += cur_len; 3869730ffcbSVarun Prakash skb->len += cur_len; 3879730ffcbSVarun Prakash skb->truesize += cur_len; 3889730ffcbSVarun Prakash 3899730ffcbSVarun Prakash data_length -= cur_len; 3909730ffcbSVarun Prakash page_off = 0; 3919730ffcbSVarun Prakash sg = sg_next(sg); 3929730ffcbSVarun Prakash i++; 3939730ffcbSVarun Prakash } 3949730ffcbSVarun Prakash 3959730ffcbSVarun Prakash if (data_length) 3969730ffcbSVarun Prakash return -1; 3979730ffcbSVarun Prakash 3989730ffcbSVarun Prakash if (padding) { 3999730ffcbSVarun Prakash page = alloc_page(GFP_KERNEL | __GFP_ZERO); 4009730ffcbSVarun Prakash if (!page) 4019730ffcbSVarun Prakash return -1; 4029730ffcbSVarun Prakash skb_fill_page_desc(skb, i, page, 0, padding); 4039730ffcbSVarun Prakash skb->data_len += padding; 4049730ffcbSVarun Prakash skb->len += padding; 4059730ffcbSVarun Prakash skb->truesize += padding; 4069730ffcbSVarun Prakash } 4079730ffcbSVarun Prakash 4089730ffcbSVarun Prakash return 0; 4099730ffcbSVarun Prakash } 4109730ffcbSVarun Prakash 4119730ffcbSVarun Prakash static int 4129730ffcbSVarun Prakash cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 4139730ffcbSVarun Prakash struct iscsi_datain_req *dr) 4149730ffcbSVarun Prakash { 4159730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 4169730ffcbSVarun Prakash struct sk_buff *skb; 4179730ffcbSVarun Prakash struct iscsi_datain datain; 4189730ffcbSVarun Prakash struct cxgbit_iso_info iso_info; 4199730ffcbSVarun Prakash u32 data_length = cmd->se_cmd.data_length; 4209730ffcbSVarun Prakash u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 4219730ffcbSVarun Prakash u32 num_pdu, plen, tx_data = 0; 4229730ffcbSVarun Prakash bool task_sense = !!(cmd->se_cmd.se_cmd_flags & 4239730ffcbSVarun Prakash SCF_TRANSPORT_TASK_SENSE); 4249730ffcbSVarun Prakash bool set_statsn = false; 4259730ffcbSVarun Prakash int ret = -1; 4269730ffcbSVarun Prakash 4279730ffcbSVarun Prakash while (data_length) { 4289730ffcbSVarun Prakash num_pdu = (data_length + mrdsl - 1) / mrdsl; 4299730ffcbSVarun Prakash if (num_pdu > csk->max_iso_npdu) 4309730ffcbSVarun Prakash num_pdu = csk->max_iso_npdu; 4319730ffcbSVarun Prakash 4329730ffcbSVarun Prakash plen = num_pdu * mrdsl; 4339730ffcbSVarun Prakash if (plen > data_length) 4349730ffcbSVarun Prakash plen = data_length; 4359730ffcbSVarun Prakash 4369730ffcbSVarun Prakash skb = __cxgbit_alloc_skb(csk, 0, true); 4379730ffcbSVarun Prakash if (unlikely(!skb)) 4389730ffcbSVarun Prakash return -ENOMEM; 4399730ffcbSVarun Prakash 4409730ffcbSVarun Prakash memset(skb->data, 0, ISCSI_HDR_LEN); 4419730ffcbSVarun Prakash cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO; 4429730ffcbSVarun Prakash cxgbit_skcb_submode(skb) |= (csk->submode & 4439730ffcbSVarun Prakash CXGBIT_SUBMODE_DCRC); 4449730ffcbSVarun Prakash cxgbit_skcb_tx_extralen(skb) = (num_pdu * 4459730ffcbSVarun Prakash cxgbit_digest_len[cxgbit_skcb_submode(skb)]) + 4469730ffcbSVarun Prakash ((num_pdu - 1) * ISCSI_HDR_LEN); 4479730ffcbSVarun Prakash 4489730ffcbSVarun Prakash memset(&datain, 0, sizeof(struct iscsi_datain)); 4499730ffcbSVarun Prakash memset(&iso_info, 0, sizeof(iso_info)); 4509730ffcbSVarun Prakash 4519730ffcbSVarun Prakash if (!tx_data) 4529730ffcbSVarun Prakash iso_info.flags |= CXGBIT_ISO_FSLICE; 4539730ffcbSVarun Prakash 4549730ffcbSVarun Prakash if (!(data_length - plen)) { 4559730ffcbSVarun Prakash iso_info.flags |= CXGBIT_ISO_LSLICE; 4569730ffcbSVarun Prakash if (!task_sense) { 4579730ffcbSVarun Prakash datain.flags = ISCSI_FLAG_DATA_STATUS; 4589730ffcbSVarun Prakash iscsit_increment_maxcmdsn(cmd, conn->sess); 4599730ffcbSVarun Prakash cmd->stat_sn = conn->stat_sn++; 4609730ffcbSVarun Prakash set_statsn = true; 4619730ffcbSVarun Prakash } 4629730ffcbSVarun Prakash } 4639730ffcbSVarun Prakash 4649730ffcbSVarun Prakash iso_info.burst_len = num_pdu * mrdsl; 4659730ffcbSVarun Prakash iso_info.mpdu = mrdsl; 4669730ffcbSVarun Prakash iso_info.len = ISCSI_HDR_LEN + plen; 4679730ffcbSVarun Prakash 4689730ffcbSVarun Prakash cxgbit_cpl_tx_data_iso(skb, &iso_info); 4699730ffcbSVarun Prakash 4709730ffcbSVarun Prakash datain.offset = tx_data; 4719730ffcbSVarun Prakash datain.data_sn = cmd->data_sn - 1; 4729730ffcbSVarun Prakash 4739730ffcbSVarun Prakash iscsit_build_datain_pdu(cmd, conn, &datain, 4749730ffcbSVarun Prakash (struct iscsi_data_rsp *)skb->data, 4759730ffcbSVarun Prakash set_statsn); 4769730ffcbSVarun Prakash 4779730ffcbSVarun Prakash ret = cxgbit_map_skb(cmd, skb, tx_data, plen); 4789730ffcbSVarun Prakash if (unlikely(ret)) { 4799730ffcbSVarun Prakash __kfree_skb(skb); 4809730ffcbSVarun Prakash goto out; 4819730ffcbSVarun Prakash } 4829730ffcbSVarun Prakash 4839730ffcbSVarun Prakash ret = cxgbit_queue_skb(csk, skb); 4849730ffcbSVarun Prakash if (unlikely(ret)) 4859730ffcbSVarun Prakash goto out; 4869730ffcbSVarun Prakash 4879730ffcbSVarun Prakash tx_data += plen; 4889730ffcbSVarun Prakash data_length -= plen; 4899730ffcbSVarun Prakash 4909730ffcbSVarun Prakash cmd->read_data_done += plen; 4919730ffcbSVarun Prakash cmd->data_sn += num_pdu; 4929730ffcbSVarun Prakash } 4939730ffcbSVarun Prakash 4949730ffcbSVarun Prakash dr->dr_complete = DATAIN_COMPLETE_NORMAL; 4959730ffcbSVarun Prakash 4969730ffcbSVarun Prakash return 0; 4979730ffcbSVarun Prakash 4989730ffcbSVarun Prakash out: 4999730ffcbSVarun Prakash return ret; 5009730ffcbSVarun Prakash } 5019730ffcbSVarun Prakash 5029730ffcbSVarun Prakash static int 5039730ffcbSVarun Prakash cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd, 5049730ffcbSVarun Prakash const struct iscsi_datain *datain) 5059730ffcbSVarun Prakash { 5069730ffcbSVarun Prakash struct sk_buff *skb; 5079730ffcbSVarun Prakash int ret = 0; 5089730ffcbSVarun Prakash 5099730ffcbSVarun Prakash skb = cxgbit_alloc_skb(csk, 0); 5109730ffcbSVarun Prakash if (unlikely(!skb)) 5119730ffcbSVarun Prakash return -ENOMEM; 5129730ffcbSVarun Prakash 5139730ffcbSVarun Prakash memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 5149730ffcbSVarun Prakash 5159730ffcbSVarun Prakash if (datain->length) { 5169730ffcbSVarun Prakash cxgbit_skcb_submode(skb) |= (csk->submode & 5179730ffcbSVarun Prakash CXGBIT_SUBMODE_DCRC); 5189730ffcbSVarun Prakash cxgbit_skcb_tx_extralen(skb) = 5199730ffcbSVarun Prakash cxgbit_digest_len[cxgbit_skcb_submode(skb)]; 5209730ffcbSVarun Prakash } 5219730ffcbSVarun Prakash 5229730ffcbSVarun Prakash ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length); 5239730ffcbSVarun Prakash if (ret < 0) { 5249730ffcbSVarun Prakash __kfree_skb(skb); 5259730ffcbSVarun Prakash return ret; 5269730ffcbSVarun Prakash } 5279730ffcbSVarun Prakash 5289730ffcbSVarun Prakash return cxgbit_queue_skb(csk, skb); 5299730ffcbSVarun Prakash } 5309730ffcbSVarun Prakash 5319730ffcbSVarun Prakash static int 5329730ffcbSVarun Prakash cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5339730ffcbSVarun Prakash struct iscsi_datain_req *dr, 5349730ffcbSVarun Prakash const struct iscsi_datain *datain) 5359730ffcbSVarun Prakash { 5369730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 5379730ffcbSVarun Prakash u32 data_length = cmd->se_cmd.data_length; 5389730ffcbSVarun Prakash u32 padding = ((-data_length) & 3); 5399730ffcbSVarun Prakash u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength; 5409730ffcbSVarun Prakash 5419730ffcbSVarun Prakash if ((data_length > mrdsl) && (!dr->recovery) && 5429730ffcbSVarun Prakash (!padding) && (!datain->offset) && csk->max_iso_npdu) { 5439730ffcbSVarun Prakash atomic_long_add(data_length - datain->length, 5449730ffcbSVarun Prakash &conn->sess->tx_data_octets); 5459730ffcbSVarun Prakash return cxgbit_tx_datain_iso(csk, cmd, dr); 5469730ffcbSVarun Prakash } 5479730ffcbSVarun Prakash 5489730ffcbSVarun Prakash return cxgbit_tx_datain(csk, cmd, datain); 5499730ffcbSVarun Prakash } 5509730ffcbSVarun Prakash 5519730ffcbSVarun Prakash static int 5529730ffcbSVarun Prakash cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5539730ffcbSVarun Prakash const void *data_buf, u32 data_buf_len) 5549730ffcbSVarun Prakash { 5559730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 5569730ffcbSVarun Prakash struct sk_buff *skb; 5579730ffcbSVarun Prakash u32 padding = ((-data_buf_len) & 3); 5589730ffcbSVarun Prakash 5599730ffcbSVarun Prakash skb = cxgbit_alloc_skb(csk, data_buf_len + padding); 5609730ffcbSVarun Prakash if (unlikely(!skb)) 5619730ffcbSVarun Prakash return -ENOMEM; 5629730ffcbSVarun Prakash 5639730ffcbSVarun Prakash memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN); 5649730ffcbSVarun Prakash 5659730ffcbSVarun Prakash if (data_buf_len) { 5669730ffcbSVarun Prakash u32 pad_bytes = 0; 5679730ffcbSVarun Prakash 5689730ffcbSVarun Prakash skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len); 5699730ffcbSVarun Prakash 5709730ffcbSVarun Prakash if (padding) 5719730ffcbSVarun Prakash skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len, 5729730ffcbSVarun Prakash &pad_bytes, padding); 5739730ffcbSVarun Prakash } 5749730ffcbSVarun Prakash 5759730ffcbSVarun Prakash cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[ 5769730ffcbSVarun Prakash cxgbit_skcb_submode(skb)]; 5779730ffcbSVarun Prakash 5789730ffcbSVarun Prakash return cxgbit_queue_skb(csk, skb); 5799730ffcbSVarun Prakash } 5809730ffcbSVarun Prakash 5819730ffcbSVarun Prakash int 5829730ffcbSVarun Prakash cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 5839730ffcbSVarun Prakash struct iscsi_datain_req *dr, const void *buf, u32 buf_len) 5849730ffcbSVarun Prakash { 5859730ffcbSVarun Prakash if (dr) 5869730ffcbSVarun Prakash return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf); 5879730ffcbSVarun Prakash else 5889730ffcbSVarun Prakash return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len); 5899730ffcbSVarun Prakash } 5909730ffcbSVarun Prakash 5919730ffcbSVarun Prakash int cxgbit_validate_params(struct iscsi_conn *conn) 5929730ffcbSVarun Prakash { 5939730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 5949730ffcbSVarun Prakash struct cxgbit_device *cdev = csk->com.cdev; 5959730ffcbSVarun Prakash struct iscsi_param *param; 5969730ffcbSVarun Prakash u32 max_xmitdsl; 5979730ffcbSVarun Prakash 5989730ffcbSVarun Prakash param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH, 5999730ffcbSVarun Prakash conn->param_list); 6009730ffcbSVarun Prakash if (!param) 6019730ffcbSVarun Prakash return -1; 6029730ffcbSVarun Prakash 6039730ffcbSVarun Prakash if (kstrtou32(param->value, 0, &max_xmitdsl) < 0) 6049730ffcbSVarun Prakash return -1; 6059730ffcbSVarun Prakash 6069730ffcbSVarun Prakash if (max_xmitdsl > cdev->mdsl) { 6079730ffcbSVarun Prakash if (iscsi_change_param_sprintf( 6089730ffcbSVarun Prakash conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl)) 6099730ffcbSVarun Prakash return -1; 6109730ffcbSVarun Prakash } 6119730ffcbSVarun Prakash 6129730ffcbSVarun Prakash return 0; 6139730ffcbSVarun Prakash } 6149730ffcbSVarun Prakash 6159730ffcbSVarun Prakash static int cxgbit_set_digest(struct cxgbit_sock *csk) 6169730ffcbSVarun Prakash { 6179730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 6189730ffcbSVarun Prakash struct iscsi_param *param; 6199730ffcbSVarun Prakash 6209730ffcbSVarun Prakash param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list); 6219730ffcbSVarun Prakash if (!param) { 6229730ffcbSVarun Prakash pr_err("param not found key %s\n", HEADERDIGEST); 6239730ffcbSVarun Prakash return -1; 6249730ffcbSVarun Prakash } 6259730ffcbSVarun Prakash 6269730ffcbSVarun Prakash if (!strcmp(param->value, CRC32C)) 6279730ffcbSVarun Prakash csk->submode |= CXGBIT_SUBMODE_HCRC; 6289730ffcbSVarun Prakash 6299730ffcbSVarun Prakash param = iscsi_find_param_from_key(DATADIGEST, conn->param_list); 6309730ffcbSVarun Prakash if (!param) { 6319730ffcbSVarun Prakash csk->submode = 0; 6329730ffcbSVarun Prakash pr_err("param not found key %s\n", DATADIGEST); 6339730ffcbSVarun Prakash return -1; 6349730ffcbSVarun Prakash } 6359730ffcbSVarun Prakash 6369730ffcbSVarun Prakash if (!strcmp(param->value, CRC32C)) 6379730ffcbSVarun Prakash csk->submode |= CXGBIT_SUBMODE_DCRC; 6389730ffcbSVarun Prakash 6399730ffcbSVarun Prakash if (cxgbit_setup_conn_digest(csk)) { 6409730ffcbSVarun Prakash csk->submode = 0; 6419730ffcbSVarun Prakash return -1; 6429730ffcbSVarun Prakash } 6439730ffcbSVarun Prakash 6449730ffcbSVarun Prakash return 0; 6459730ffcbSVarun Prakash } 6469730ffcbSVarun Prakash 6479730ffcbSVarun Prakash static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk) 6489730ffcbSVarun Prakash { 6499730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 6509730ffcbSVarun Prakash struct iscsi_conn_ops *conn_ops = conn->conn_ops; 6519730ffcbSVarun Prakash struct iscsi_param *param; 6529730ffcbSVarun Prakash u32 mrdsl, mbl; 6539730ffcbSVarun Prakash u32 max_npdu, max_iso_npdu; 6549730ffcbSVarun Prakash 6559730ffcbSVarun Prakash if (conn->login->leading_connection) { 6569730ffcbSVarun Prakash param = iscsi_find_param_from_key(DATASEQUENCEINORDER, 6579730ffcbSVarun Prakash conn->param_list); 6589730ffcbSVarun Prakash if (!param) { 6599730ffcbSVarun Prakash pr_err("param not found key %s\n", DATASEQUENCEINORDER); 6609730ffcbSVarun Prakash return -1; 6619730ffcbSVarun Prakash } 6629730ffcbSVarun Prakash 6639730ffcbSVarun Prakash if (strcmp(param->value, YES)) 6649730ffcbSVarun Prakash return 0; 6659730ffcbSVarun Prakash 6669730ffcbSVarun Prakash param = iscsi_find_param_from_key(DATAPDUINORDER, 6679730ffcbSVarun Prakash conn->param_list); 6689730ffcbSVarun Prakash if (!param) { 6699730ffcbSVarun Prakash pr_err("param not found key %s\n", DATAPDUINORDER); 6709730ffcbSVarun Prakash return -1; 6719730ffcbSVarun Prakash } 6729730ffcbSVarun Prakash 6739730ffcbSVarun Prakash if (strcmp(param->value, YES)) 6749730ffcbSVarun Prakash return 0; 6759730ffcbSVarun Prakash 6769730ffcbSVarun Prakash param = iscsi_find_param_from_key(MAXBURSTLENGTH, 6779730ffcbSVarun Prakash conn->param_list); 6789730ffcbSVarun Prakash if (!param) { 6799730ffcbSVarun Prakash pr_err("param not found key %s\n", MAXBURSTLENGTH); 6809730ffcbSVarun Prakash return -1; 6819730ffcbSVarun Prakash } 6829730ffcbSVarun Prakash 6839730ffcbSVarun Prakash if (kstrtou32(param->value, 0, &mbl) < 0) 6849730ffcbSVarun Prakash return -1; 6859730ffcbSVarun Prakash } else { 6869730ffcbSVarun Prakash if (!conn->sess->sess_ops->DataSequenceInOrder) 6879730ffcbSVarun Prakash return 0; 6889730ffcbSVarun Prakash if (!conn->sess->sess_ops->DataPDUInOrder) 6899730ffcbSVarun Prakash return 0; 6909730ffcbSVarun Prakash 6919730ffcbSVarun Prakash mbl = conn->sess->sess_ops->MaxBurstLength; 6929730ffcbSVarun Prakash } 6939730ffcbSVarun Prakash 6949730ffcbSVarun Prakash mrdsl = conn_ops->MaxRecvDataSegmentLength; 6959730ffcbSVarun Prakash max_npdu = mbl / mrdsl; 6969730ffcbSVarun Prakash 6979730ffcbSVarun Prakash max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD / 6989730ffcbSVarun Prakash (ISCSI_HDR_LEN + mrdsl + 6999730ffcbSVarun Prakash cxgbit_digest_len[csk->submode]); 7009730ffcbSVarun Prakash 7019730ffcbSVarun Prakash csk->max_iso_npdu = min(max_npdu, max_iso_npdu); 7029730ffcbSVarun Prakash 7039730ffcbSVarun Prakash if (csk->max_iso_npdu <= 1) 7049730ffcbSVarun Prakash csk->max_iso_npdu = 0; 7059730ffcbSVarun Prakash 7069730ffcbSVarun Prakash return 0; 7079730ffcbSVarun Prakash } 7089730ffcbSVarun Prakash 7099730ffcbSVarun Prakash static int cxgbit_set_params(struct iscsi_conn *conn) 7109730ffcbSVarun Prakash { 7119730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 7129730ffcbSVarun Prakash struct cxgbit_device *cdev = csk->com.cdev; 7139730ffcbSVarun Prakash struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm; 7149730ffcbSVarun Prakash struct iscsi_conn_ops *conn_ops = conn->conn_ops; 7159730ffcbSVarun Prakash struct iscsi_param *param; 7169730ffcbSVarun Prakash u8 erl; 7179730ffcbSVarun Prakash 7189730ffcbSVarun Prakash if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl) 7199730ffcbSVarun Prakash conn_ops->MaxRecvDataSegmentLength = cdev->mdsl; 7209730ffcbSVarun Prakash 7219730ffcbSVarun Prakash if (conn->login->leading_connection) { 7229730ffcbSVarun Prakash param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL, 7239730ffcbSVarun Prakash conn->param_list); 7249730ffcbSVarun Prakash if (!param) { 7259730ffcbSVarun Prakash pr_err("param not found key %s\n", ERRORRECOVERYLEVEL); 7269730ffcbSVarun Prakash return -1; 7279730ffcbSVarun Prakash } 7289730ffcbSVarun Prakash if (kstrtou8(param->value, 0, &erl) < 0) 7299730ffcbSVarun Prakash return -1; 7309730ffcbSVarun Prakash } else { 7319730ffcbSVarun Prakash erl = conn->sess->sess_ops->ErrorRecoveryLevel; 7329730ffcbSVarun Prakash } 7339730ffcbSVarun Prakash 7349730ffcbSVarun Prakash if (!erl) { 7359730ffcbSVarun Prakash if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) { 7369730ffcbSVarun Prakash if (cxgbit_set_iso_npdu(csk)) 7379730ffcbSVarun Prakash return -1; 7389730ffcbSVarun Prakash } 7399730ffcbSVarun Prakash 7409730ffcbSVarun Prakash if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) { 7419730ffcbSVarun Prakash if (cxgbit_setup_conn_pgidx(csk, 7429730ffcbSVarun Prakash ppm->tformat.pgsz_idx_dflt)) 7439730ffcbSVarun Prakash return -1; 7449730ffcbSVarun Prakash set_bit(CSK_DDP_ENABLE, &csk->com.flags); 7459730ffcbSVarun Prakash } 7469730ffcbSVarun Prakash } 7479730ffcbSVarun Prakash 7489730ffcbSVarun Prakash if (cxgbit_set_digest(csk)) 7499730ffcbSVarun Prakash return -1; 7509730ffcbSVarun Prakash 7519730ffcbSVarun Prakash return 0; 7529730ffcbSVarun Prakash } 7539730ffcbSVarun Prakash 7549730ffcbSVarun Prakash int 7559730ffcbSVarun Prakash cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 7569730ffcbSVarun Prakash u32 length) 7579730ffcbSVarun Prakash { 7589730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 7599730ffcbSVarun Prakash struct sk_buff *skb; 7609730ffcbSVarun Prakash u32 padding_buf = 0; 7619730ffcbSVarun Prakash u8 padding = ((-length) & 3); 7629730ffcbSVarun Prakash 7639730ffcbSVarun Prakash skb = cxgbit_alloc_skb(csk, length + padding); 7649730ffcbSVarun Prakash if (!skb) 7659730ffcbSVarun Prakash return -ENOMEM; 7669730ffcbSVarun Prakash skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN); 7679730ffcbSVarun Prakash skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length); 7689730ffcbSVarun Prakash 7699730ffcbSVarun Prakash if (padding) 7709730ffcbSVarun Prakash skb_store_bits(skb, ISCSI_HDR_LEN + length, 7719730ffcbSVarun Prakash &padding_buf, padding); 7729730ffcbSVarun Prakash 7739730ffcbSVarun Prakash if (login->login_complete) { 7749730ffcbSVarun Prakash if (cxgbit_set_params(conn)) { 7759730ffcbSVarun Prakash kfree_skb(skb); 7769730ffcbSVarun Prakash return -1; 7779730ffcbSVarun Prakash } 7789730ffcbSVarun Prakash 7799730ffcbSVarun Prakash set_bit(CSK_LOGIN_DONE, &csk->com.flags); 7809730ffcbSVarun Prakash } 7819730ffcbSVarun Prakash 7829730ffcbSVarun Prakash if (cxgbit_queue_skb(csk, skb)) 7839730ffcbSVarun Prakash return -1; 7849730ffcbSVarun Prakash 7859730ffcbSVarun Prakash if ((!login->login_complete) && (!login->login_failed)) 7869730ffcbSVarun Prakash schedule_delayed_work(&conn->login_work, 0); 7879730ffcbSVarun Prakash 7889730ffcbSVarun Prakash return 0; 7899730ffcbSVarun Prakash } 7909730ffcbSVarun Prakash 7919730ffcbSVarun Prakash static void 7929730ffcbSVarun Prakash cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg, 7939730ffcbSVarun Prakash unsigned int nents) 7949730ffcbSVarun Prakash { 7959730ffcbSVarun Prakash struct skb_seq_state st; 7969730ffcbSVarun Prakash const u8 *buf; 7979730ffcbSVarun Prakash unsigned int consumed = 0, buf_len; 7989730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb); 7999730ffcbSVarun Prakash 8009730ffcbSVarun Prakash skb_prepare_seq_read(skb, pdu_cb->doffset, 8019730ffcbSVarun Prakash pdu_cb->doffset + pdu_cb->dlen, 8029730ffcbSVarun Prakash &st); 8039730ffcbSVarun Prakash 8049730ffcbSVarun Prakash while (true) { 8059730ffcbSVarun Prakash buf_len = skb_seq_read(consumed, &buf, &st); 8069730ffcbSVarun Prakash if (!buf_len) { 8079730ffcbSVarun Prakash skb_abort_seq_read(&st); 8089730ffcbSVarun Prakash break; 8099730ffcbSVarun Prakash } 8109730ffcbSVarun Prakash 8119730ffcbSVarun Prakash consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf, 8129730ffcbSVarun Prakash buf_len, consumed); 8139730ffcbSVarun Prakash } 8149730ffcbSVarun Prakash } 8159730ffcbSVarun Prakash 8169730ffcbSVarun Prakash static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk) 8179730ffcbSVarun Prakash { 8189730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 8199730ffcbSVarun Prakash struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev); 8209730ffcbSVarun Prakash struct cxgbit_cmd *ccmd; 8219730ffcbSVarun Prakash struct iscsi_cmd *cmd; 8229730ffcbSVarun Prakash 8239730ffcbSVarun Prakash cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 8249730ffcbSVarun Prakash if (!cmd) { 8259730ffcbSVarun Prakash pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n"); 8269730ffcbSVarun Prakash return NULL; 8279730ffcbSVarun Prakash } 8289730ffcbSVarun Prakash 8299730ffcbSVarun Prakash ccmd = iscsit_priv_cmd(cmd); 8309730ffcbSVarun Prakash ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask; 8319730ffcbSVarun Prakash ccmd->setup_ddp = true; 8329730ffcbSVarun Prakash 8339730ffcbSVarun Prakash return cmd; 8349730ffcbSVarun Prakash } 8359730ffcbSVarun Prakash 8369730ffcbSVarun Prakash static int 8379730ffcbSVarun Prakash cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 8389730ffcbSVarun Prakash u32 length) 8399730ffcbSVarun Prakash { 8409730ffcbSVarun Prakash struct iscsi_conn *conn = cmd->conn; 8419730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 8429730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 8439730ffcbSVarun Prakash 8449730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 8459730ffcbSVarun Prakash pr_err("ImmediateData CRC32C DataDigest error\n"); 8469730ffcbSVarun Prakash if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 8479730ffcbSVarun Prakash pr_err("Unable to recover from" 8489730ffcbSVarun Prakash " Immediate Data digest failure while" 8499730ffcbSVarun Prakash " in ERL=0.\n"); 8509730ffcbSVarun Prakash iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 8519730ffcbSVarun Prakash (unsigned char *)hdr); 8529730ffcbSVarun Prakash return IMMEDIATE_DATA_CANNOT_RECOVER; 8539730ffcbSVarun Prakash } 8549730ffcbSVarun Prakash 8559730ffcbSVarun Prakash iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, 8569730ffcbSVarun Prakash (unsigned char *)hdr); 8579730ffcbSVarun Prakash return IMMEDIATE_DATA_ERL1_CRC_FAILURE; 8589730ffcbSVarun Prakash } 8599730ffcbSVarun Prakash 8609730ffcbSVarun Prakash if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) { 8619730ffcbSVarun Prakash struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd); 8629730ffcbSVarun Prakash struct skb_shared_info *ssi = skb_shinfo(csk->skb); 8639730ffcbSVarun Prakash skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx]; 8649730ffcbSVarun Prakash 8659730ffcbSVarun Prakash sg_init_table(&ccmd->sg, 1); 8669730ffcbSVarun Prakash sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag), 8679730ffcbSVarun Prakash dfrag->page_offset); 8689730ffcbSVarun Prakash get_page(dfrag->page.p); 8699730ffcbSVarun Prakash 8709730ffcbSVarun Prakash cmd->se_cmd.t_data_sg = &ccmd->sg; 8719730ffcbSVarun Prakash cmd->se_cmd.t_data_nents = 1; 8729730ffcbSVarun Prakash 8739730ffcbSVarun Prakash ccmd->release = true; 8749730ffcbSVarun Prakash } else { 8759730ffcbSVarun Prakash struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0]; 8769730ffcbSVarun Prakash u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE)); 8779730ffcbSVarun Prakash 8789730ffcbSVarun Prakash cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents); 8799730ffcbSVarun Prakash } 8809730ffcbSVarun Prakash 8819730ffcbSVarun Prakash cmd->write_data_done += pdu_cb->dlen; 8829730ffcbSVarun Prakash 8839730ffcbSVarun Prakash if (cmd->write_data_done == cmd->se_cmd.data_length) { 8849730ffcbSVarun Prakash spin_lock_bh(&cmd->istate_lock); 8859730ffcbSVarun Prakash cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 8869730ffcbSVarun Prakash cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 8879730ffcbSVarun Prakash spin_unlock_bh(&cmd->istate_lock); 8889730ffcbSVarun Prakash } 8899730ffcbSVarun Prakash 8909730ffcbSVarun Prakash return IMMEDIATE_DATA_NORMAL_OPERATION; 8919730ffcbSVarun Prakash } 8929730ffcbSVarun Prakash 8939730ffcbSVarun Prakash static int 8949730ffcbSVarun Prakash cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, 8959730ffcbSVarun Prakash bool dump_payload) 8969730ffcbSVarun Prakash { 8979730ffcbSVarun Prakash struct iscsi_conn *conn = cmd->conn; 8989730ffcbSVarun Prakash int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; 8999730ffcbSVarun Prakash /* 9009730ffcbSVarun Prakash * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. 9019730ffcbSVarun Prakash */ 9029730ffcbSVarun Prakash if (dump_payload) 9039730ffcbSVarun Prakash goto after_immediate_data; 9049730ffcbSVarun Prakash 9059730ffcbSVarun Prakash immed_ret = cxgbit_handle_immediate_data(cmd, hdr, 9069730ffcbSVarun Prakash cmd->first_burst_len); 9079730ffcbSVarun Prakash after_immediate_data: 9089730ffcbSVarun Prakash if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { 9099730ffcbSVarun Prakash /* 9109730ffcbSVarun Prakash * A PDU/CmdSN carrying Immediate Data passed 9119730ffcbSVarun Prakash * DataCRC, check against ExpCmdSN/MaxCmdSN if 9129730ffcbSVarun Prakash * Immediate Bit is not set. 9139730ffcbSVarun Prakash */ 9149730ffcbSVarun Prakash cmdsn_ret = iscsit_sequence_cmd(conn, cmd, 9159730ffcbSVarun Prakash (unsigned char *)hdr, 9169730ffcbSVarun Prakash hdr->cmdsn); 9179730ffcbSVarun Prakash if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) 9189730ffcbSVarun Prakash return -1; 9199730ffcbSVarun Prakash 9209730ffcbSVarun Prakash if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) { 9219730ffcbSVarun Prakash target_put_sess_cmd(&cmd->se_cmd); 9229730ffcbSVarun Prakash return 0; 9239730ffcbSVarun Prakash } else if (cmd->unsolicited_data) { 9249730ffcbSVarun Prakash iscsit_set_unsoliticed_dataout(cmd); 9259730ffcbSVarun Prakash } 9269730ffcbSVarun Prakash 9279730ffcbSVarun Prakash } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { 9289730ffcbSVarun Prakash /* 9299730ffcbSVarun Prakash * Immediate Data failed DataCRC and ERL>=1, 9309730ffcbSVarun Prakash * silently drop this PDU and let the initiator 9319730ffcbSVarun Prakash * plug the CmdSN gap. 9329730ffcbSVarun Prakash * 9339730ffcbSVarun Prakash * FIXME: Send Unsolicited NOPIN with reserved 9349730ffcbSVarun Prakash * TTT here to help the initiator figure out 9359730ffcbSVarun Prakash * the missing CmdSN, although they should be 9369730ffcbSVarun Prakash * intelligent enough to determine the missing 9379730ffcbSVarun Prakash * CmdSN and issue a retry to plug the sequence. 9389730ffcbSVarun Prakash */ 9399730ffcbSVarun Prakash cmd->i_state = ISTATE_REMOVE; 9409730ffcbSVarun Prakash iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); 9419730ffcbSVarun Prakash } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ 9429730ffcbSVarun Prakash return -1; 9439730ffcbSVarun Prakash 9449730ffcbSVarun Prakash return 0; 9459730ffcbSVarun Prakash } 9469730ffcbSVarun Prakash 9479730ffcbSVarun Prakash static int 9489730ffcbSVarun Prakash cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 9499730ffcbSVarun Prakash { 9509730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 9519730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 9529730ffcbSVarun Prakash struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr; 9539730ffcbSVarun Prakash int rc; 9549730ffcbSVarun Prakash bool dump_payload = false; 9559730ffcbSVarun Prakash 9569730ffcbSVarun Prakash rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr); 9579730ffcbSVarun Prakash if (rc < 0) 9589730ffcbSVarun Prakash return rc; 9599730ffcbSVarun Prakash 9609730ffcbSVarun Prakash if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) && 9619730ffcbSVarun Prakash (pdu_cb->nr_dfrags == 1)) 9629730ffcbSVarun Prakash cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 9639730ffcbSVarun Prakash 9649730ffcbSVarun Prakash rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 9659730ffcbSVarun Prakash if (rc < 0) 9669730ffcbSVarun Prakash return 0; 9679730ffcbSVarun Prakash else if (rc > 0) 9689730ffcbSVarun Prakash dump_payload = true; 9699730ffcbSVarun Prakash 9709730ffcbSVarun Prakash if (!pdu_cb->dlen) 9719730ffcbSVarun Prakash return 0; 9729730ffcbSVarun Prakash 9739730ffcbSVarun Prakash return cxgbit_get_immediate_data(cmd, hdr, dump_payload); 9749730ffcbSVarun Prakash } 9759730ffcbSVarun Prakash 9769730ffcbSVarun Prakash static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk) 9779730ffcbSVarun Prakash { 9789730ffcbSVarun Prakash struct scatterlist *sg_start; 9799730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 9809730ffcbSVarun Prakash struct iscsi_cmd *cmd = NULL; 9819730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 9829730ffcbSVarun Prakash struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr; 9839730ffcbSVarun Prakash u32 data_offset = be32_to_cpu(hdr->offset); 9849730ffcbSVarun Prakash u32 data_len = pdu_cb->dlen; 9859730ffcbSVarun Prakash int rc, sg_nents, sg_off; 9869730ffcbSVarun Prakash bool dcrc_err = false; 9879730ffcbSVarun Prakash 9889730ffcbSVarun Prakash rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd); 9899730ffcbSVarun Prakash if (rc < 0) 9909730ffcbSVarun Prakash return rc; 9919730ffcbSVarun Prakash else if (!cmd) 9929730ffcbSVarun Prakash return 0; 9939730ffcbSVarun Prakash 9949730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 9959730ffcbSVarun Prakash pr_err("ITT: 0x%08x, Offset: %u, Length: %u," 9969730ffcbSVarun Prakash " DataSN: 0x%08x\n", 9979730ffcbSVarun Prakash hdr->itt, hdr->offset, data_len, 9989730ffcbSVarun Prakash hdr->datasn); 9999730ffcbSVarun Prakash 10009730ffcbSVarun Prakash dcrc_err = true; 10019730ffcbSVarun Prakash goto check_payload; 10029730ffcbSVarun Prakash } 10039730ffcbSVarun Prakash 10049730ffcbSVarun Prakash pr_debug("DataOut data_len: %u, " 10059730ffcbSVarun Prakash "write_data_done: %u, data_length: %u\n", 10069730ffcbSVarun Prakash data_len, cmd->write_data_done, 10079730ffcbSVarun Prakash cmd->se_cmd.data_length); 10089730ffcbSVarun Prakash 10099730ffcbSVarun Prakash if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) { 10109730ffcbSVarun Prakash sg_off = data_offset / PAGE_SIZE; 10119730ffcbSVarun Prakash sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 10129730ffcbSVarun Prakash sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE)); 10139730ffcbSVarun Prakash 10149730ffcbSVarun Prakash cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents); 10159730ffcbSVarun Prakash } 10169730ffcbSVarun Prakash 10179730ffcbSVarun Prakash check_payload: 10189730ffcbSVarun Prakash 10199730ffcbSVarun Prakash rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err); 10209730ffcbSVarun Prakash if (rc < 0) 10219730ffcbSVarun Prakash return rc; 10229730ffcbSVarun Prakash 10239730ffcbSVarun Prakash return 0; 10249730ffcbSVarun Prakash } 10259730ffcbSVarun Prakash 10269730ffcbSVarun Prakash static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 10279730ffcbSVarun Prakash { 10289730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 10299730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 10309730ffcbSVarun Prakash struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr; 10319730ffcbSVarun Prakash unsigned char *ping_data = NULL; 10329730ffcbSVarun Prakash u32 payload_length = pdu_cb->dlen; 10339730ffcbSVarun Prakash int ret; 10349730ffcbSVarun Prakash 10359730ffcbSVarun Prakash ret = iscsit_setup_nop_out(conn, cmd, hdr); 10369730ffcbSVarun Prakash if (ret < 0) 10379730ffcbSVarun Prakash return 0; 10389730ffcbSVarun Prakash 10399730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 10409730ffcbSVarun Prakash if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 10419730ffcbSVarun Prakash pr_err("Unable to recover from" 10429730ffcbSVarun Prakash " NOPOUT Ping DataCRC failure while in" 10439730ffcbSVarun Prakash " ERL=0.\n"); 10449730ffcbSVarun Prakash ret = -1; 10459730ffcbSVarun Prakash goto out; 10469730ffcbSVarun Prakash } else { 10479730ffcbSVarun Prakash /* 10489730ffcbSVarun Prakash * drop this PDU and let the 10499730ffcbSVarun Prakash * initiator plug the CmdSN gap. 10509730ffcbSVarun Prakash */ 10519730ffcbSVarun Prakash pr_info("Dropping NOPOUT" 10529730ffcbSVarun Prakash " Command CmdSN: 0x%08x due to" 10539730ffcbSVarun Prakash " DataCRC error.\n", hdr->cmdsn); 10549730ffcbSVarun Prakash ret = 0; 10559730ffcbSVarun Prakash goto out; 10569730ffcbSVarun Prakash } 10579730ffcbSVarun Prakash } 10589730ffcbSVarun Prakash 10599730ffcbSVarun Prakash /* 10609730ffcbSVarun Prakash * Handle NOP-OUT payload for traditional iSCSI sockets 10619730ffcbSVarun Prakash */ 10629730ffcbSVarun Prakash if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 10639730ffcbSVarun Prakash ping_data = kzalloc(payload_length + 1, GFP_KERNEL); 10649730ffcbSVarun Prakash if (!ping_data) { 10659730ffcbSVarun Prakash pr_err("Unable to allocate memory for" 10669730ffcbSVarun Prakash " NOPOUT ping data.\n"); 10679730ffcbSVarun Prakash ret = -1; 10689730ffcbSVarun Prakash goto out; 10699730ffcbSVarun Prakash } 10709730ffcbSVarun Prakash 10719730ffcbSVarun Prakash skb_copy_bits(csk->skb, pdu_cb->doffset, 10729730ffcbSVarun Prakash ping_data, payload_length); 10739730ffcbSVarun Prakash 10749730ffcbSVarun Prakash ping_data[payload_length] = '\0'; 10759730ffcbSVarun Prakash /* 10769730ffcbSVarun Prakash * Attach ping data to struct iscsi_cmd->buf_ptr. 10779730ffcbSVarun Prakash */ 10789730ffcbSVarun Prakash cmd->buf_ptr = ping_data; 10799730ffcbSVarun Prakash cmd->buf_ptr_size = payload_length; 10809730ffcbSVarun Prakash 10819730ffcbSVarun Prakash pr_debug("Got %u bytes of NOPOUT ping" 10829730ffcbSVarun Prakash " data.\n", payload_length); 10839730ffcbSVarun Prakash pr_debug("Ping Data: \"%s\"\n", ping_data); 10849730ffcbSVarun Prakash } 10859730ffcbSVarun Prakash 10869730ffcbSVarun Prakash return iscsit_process_nop_out(conn, cmd, hdr); 10879730ffcbSVarun Prakash out: 10889730ffcbSVarun Prakash if (cmd) 10899730ffcbSVarun Prakash iscsit_free_cmd(cmd, false); 10909730ffcbSVarun Prakash return ret; 10919730ffcbSVarun Prakash } 10929730ffcbSVarun Prakash 10939730ffcbSVarun Prakash static int 10949730ffcbSVarun Prakash cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd) 10959730ffcbSVarun Prakash { 10969730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 10979730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 10989730ffcbSVarun Prakash struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr; 10999730ffcbSVarun Prakash u32 payload_length = pdu_cb->dlen; 11009730ffcbSVarun Prakash int rc; 11019730ffcbSVarun Prakash unsigned char *text_in = NULL; 11029730ffcbSVarun Prakash 11039730ffcbSVarun Prakash rc = iscsit_setup_text_cmd(conn, cmd, hdr); 11049730ffcbSVarun Prakash if (rc < 0) 11059730ffcbSVarun Prakash return rc; 11069730ffcbSVarun Prakash 11079730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) { 11089730ffcbSVarun Prakash if (!conn->sess->sess_ops->ErrorRecoveryLevel) { 11099730ffcbSVarun Prakash pr_err("Unable to recover from" 11109730ffcbSVarun Prakash " Text Data digest failure while in" 11119730ffcbSVarun Prakash " ERL=0.\n"); 11129730ffcbSVarun Prakash goto reject; 11139730ffcbSVarun Prakash } else { 11149730ffcbSVarun Prakash /* 11159730ffcbSVarun Prakash * drop this PDU and let the 11169730ffcbSVarun Prakash * initiator plug the CmdSN gap. 11179730ffcbSVarun Prakash */ 11189730ffcbSVarun Prakash pr_info("Dropping Text" 11199730ffcbSVarun Prakash " Command CmdSN: 0x%08x due to" 11209730ffcbSVarun Prakash " DataCRC error.\n", hdr->cmdsn); 11219730ffcbSVarun Prakash return 0; 11229730ffcbSVarun Prakash } 11239730ffcbSVarun Prakash } 11249730ffcbSVarun Prakash 11259730ffcbSVarun Prakash if (payload_length) { 11269730ffcbSVarun Prakash text_in = kzalloc(payload_length, GFP_KERNEL); 11279730ffcbSVarun Prakash if (!text_in) { 11289730ffcbSVarun Prakash pr_err("Unable to allocate text_in of payload_length: %u\n", 11299730ffcbSVarun Prakash payload_length); 11309730ffcbSVarun Prakash return -ENOMEM; 11319730ffcbSVarun Prakash } 11329730ffcbSVarun Prakash skb_copy_bits(csk->skb, pdu_cb->doffset, 11339730ffcbSVarun Prakash text_in, payload_length); 11349730ffcbSVarun Prakash 11359730ffcbSVarun Prakash text_in[payload_length - 1] = '\0'; 11369730ffcbSVarun Prakash 11379730ffcbSVarun Prakash cmd->text_in_ptr = text_in; 11389730ffcbSVarun Prakash } 11399730ffcbSVarun Prakash 11409730ffcbSVarun Prakash return iscsit_process_text_cmd(conn, cmd, hdr); 11419730ffcbSVarun Prakash 11429730ffcbSVarun Prakash reject: 11439730ffcbSVarun Prakash return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, 11449730ffcbSVarun Prakash pdu_cb->hdr); 11459730ffcbSVarun Prakash } 11469730ffcbSVarun Prakash 11479730ffcbSVarun Prakash static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk) 11489730ffcbSVarun Prakash { 11499730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 11509730ffcbSVarun Prakash struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr; 11519730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 11529730ffcbSVarun Prakash struct iscsi_cmd *cmd = NULL; 11539730ffcbSVarun Prakash u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 11549730ffcbSVarun Prakash int ret = -EINVAL; 11559730ffcbSVarun Prakash 11569730ffcbSVarun Prakash switch (opcode) { 11579730ffcbSVarun Prakash case ISCSI_OP_SCSI_CMD: 11589730ffcbSVarun Prakash cmd = cxgbit_allocate_cmd(csk); 11599730ffcbSVarun Prakash if (!cmd) 11609730ffcbSVarun Prakash goto reject; 11619730ffcbSVarun Prakash 11629730ffcbSVarun Prakash ret = cxgbit_handle_scsi_cmd(csk, cmd); 11639730ffcbSVarun Prakash break; 11649730ffcbSVarun Prakash case ISCSI_OP_SCSI_DATA_OUT: 11659730ffcbSVarun Prakash ret = cxgbit_handle_iscsi_dataout(csk); 11669730ffcbSVarun Prakash break; 11679730ffcbSVarun Prakash case ISCSI_OP_NOOP_OUT: 11689730ffcbSVarun Prakash if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { 11699730ffcbSVarun Prakash cmd = cxgbit_allocate_cmd(csk); 11709730ffcbSVarun Prakash if (!cmd) 11719730ffcbSVarun Prakash goto reject; 11729730ffcbSVarun Prakash } 11739730ffcbSVarun Prakash 11749730ffcbSVarun Prakash ret = cxgbit_handle_nop_out(csk, cmd); 11759730ffcbSVarun Prakash break; 11769730ffcbSVarun Prakash case ISCSI_OP_SCSI_TMFUNC: 11779730ffcbSVarun Prakash cmd = cxgbit_allocate_cmd(csk); 11789730ffcbSVarun Prakash if (!cmd) 11799730ffcbSVarun Prakash goto reject; 11809730ffcbSVarun Prakash 11819730ffcbSVarun Prakash ret = iscsit_handle_task_mgt_cmd(conn, cmd, 11829730ffcbSVarun Prakash (unsigned char *)hdr); 11839730ffcbSVarun Prakash break; 11849730ffcbSVarun Prakash case ISCSI_OP_TEXT: 11859730ffcbSVarun Prakash if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) { 11869730ffcbSVarun Prakash cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 11879730ffcbSVarun Prakash if (!cmd) 11889730ffcbSVarun Prakash goto reject; 11899730ffcbSVarun Prakash } else { 11909730ffcbSVarun Prakash cmd = cxgbit_allocate_cmd(csk); 11919730ffcbSVarun Prakash if (!cmd) 11929730ffcbSVarun Prakash goto reject; 11939730ffcbSVarun Prakash } 11949730ffcbSVarun Prakash 11959730ffcbSVarun Prakash ret = cxgbit_handle_text_cmd(csk, cmd); 11969730ffcbSVarun Prakash break; 11979730ffcbSVarun Prakash case ISCSI_OP_LOGOUT: 11989730ffcbSVarun Prakash cmd = cxgbit_allocate_cmd(csk); 11999730ffcbSVarun Prakash if (!cmd) 12009730ffcbSVarun Prakash goto reject; 12019730ffcbSVarun Prakash 12029730ffcbSVarun Prakash ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 12039730ffcbSVarun Prakash if (ret > 0) 12049730ffcbSVarun Prakash wait_for_completion_timeout(&conn->conn_logout_comp, 12059730ffcbSVarun Prakash SECONDS_FOR_LOGOUT_COMP 12069730ffcbSVarun Prakash * HZ); 12079730ffcbSVarun Prakash break; 12089730ffcbSVarun Prakash case ISCSI_OP_SNACK: 12099730ffcbSVarun Prakash ret = iscsit_handle_snack(conn, (unsigned char *)hdr); 12109730ffcbSVarun Prakash break; 12119730ffcbSVarun Prakash default: 12129730ffcbSVarun Prakash pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 12139730ffcbSVarun Prakash dump_stack(); 12149730ffcbSVarun Prakash break; 12159730ffcbSVarun Prakash } 12169730ffcbSVarun Prakash 12179730ffcbSVarun Prakash return ret; 12189730ffcbSVarun Prakash 12199730ffcbSVarun Prakash reject: 12209730ffcbSVarun Prakash return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, 12219730ffcbSVarun Prakash (unsigned char *)hdr); 12229730ffcbSVarun Prakash return ret; 12239730ffcbSVarun Prakash } 12249730ffcbSVarun Prakash 12259730ffcbSVarun Prakash static int cxgbit_rx_opcode(struct cxgbit_sock *csk) 12269730ffcbSVarun Prakash { 12279730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 12289730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 12299730ffcbSVarun Prakash struct iscsi_hdr *hdr = pdu_cb->hdr; 12309730ffcbSVarun Prakash u8 opcode; 12319730ffcbSVarun Prakash 12329730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) { 12339730ffcbSVarun Prakash atomic_long_inc(&conn->sess->conn_digest_errors); 12349730ffcbSVarun Prakash goto transport_err; 12359730ffcbSVarun Prakash } 12369730ffcbSVarun Prakash 12379730ffcbSVarun Prakash if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) 12389730ffcbSVarun Prakash goto transport_err; 12399730ffcbSVarun Prakash 12409730ffcbSVarun Prakash opcode = hdr->opcode & ISCSI_OPCODE_MASK; 12419730ffcbSVarun Prakash 12429730ffcbSVarun Prakash if (conn->sess->sess_ops->SessionType && 12439730ffcbSVarun Prakash ((!(opcode & ISCSI_OP_TEXT)) || 12449730ffcbSVarun Prakash (!(opcode & ISCSI_OP_LOGOUT)))) { 12459730ffcbSVarun Prakash pr_err("Received illegal iSCSI Opcode: 0x%02x" 12469730ffcbSVarun Prakash " while in Discovery Session, rejecting.\n", opcode); 12479730ffcbSVarun Prakash iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, 12489730ffcbSVarun Prakash (unsigned char *)hdr); 12499730ffcbSVarun Prakash goto transport_err; 12509730ffcbSVarun Prakash } 12519730ffcbSVarun Prakash 12529730ffcbSVarun Prakash if (cxgbit_target_rx_opcode(csk) < 0) 12539730ffcbSVarun Prakash goto transport_err; 12549730ffcbSVarun Prakash 12559730ffcbSVarun Prakash return 0; 12569730ffcbSVarun Prakash 12579730ffcbSVarun Prakash transport_err: 12589730ffcbSVarun Prakash return -1; 12599730ffcbSVarun Prakash } 12609730ffcbSVarun Prakash 12619730ffcbSVarun Prakash static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk) 12629730ffcbSVarun Prakash { 12639730ffcbSVarun Prakash struct iscsi_conn *conn = csk->conn; 12649730ffcbSVarun Prakash struct iscsi_login *login = conn->login; 12659730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb); 12669730ffcbSVarun Prakash struct iscsi_login_req *login_req; 12679730ffcbSVarun Prakash 12689730ffcbSVarun Prakash login_req = (struct iscsi_login_req *)login->req; 12699730ffcbSVarun Prakash memcpy(login_req, pdu_cb->hdr, sizeof(*login_req)); 12709730ffcbSVarun Prakash 12719730ffcbSVarun Prakash pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," 12729730ffcbSVarun Prakash " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", 12739730ffcbSVarun Prakash login_req->flags, login_req->itt, login_req->cmdsn, 12749730ffcbSVarun Prakash login_req->exp_statsn, login_req->cid, pdu_cb->dlen); 12759730ffcbSVarun Prakash /* 12769730ffcbSVarun Prakash * Setup the initial iscsi_login values from the leading 12779730ffcbSVarun Prakash * login request PDU. 12789730ffcbSVarun Prakash */ 12799730ffcbSVarun Prakash if (login->first_request) { 12809730ffcbSVarun Prakash login_req = (struct iscsi_login_req *)login->req; 12819730ffcbSVarun Prakash login->leading_connection = (!login_req->tsih) ? 1 : 0; 12829730ffcbSVarun Prakash login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 12839730ffcbSVarun Prakash login_req->flags); 12849730ffcbSVarun Prakash login->version_min = login_req->min_version; 12859730ffcbSVarun Prakash login->version_max = login_req->max_version; 12869730ffcbSVarun Prakash memcpy(login->isid, login_req->isid, 6); 12879730ffcbSVarun Prakash login->cmd_sn = be32_to_cpu(login_req->cmdsn); 12889730ffcbSVarun Prakash login->init_task_tag = login_req->itt; 12899730ffcbSVarun Prakash login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 12909730ffcbSVarun Prakash login->cid = be16_to_cpu(login_req->cid); 12919730ffcbSVarun Prakash login->tsih = be16_to_cpu(login_req->tsih); 12929730ffcbSVarun Prakash } 12939730ffcbSVarun Prakash 12949730ffcbSVarun Prakash if (iscsi_target_check_login_request(conn, login) < 0) 12959730ffcbSVarun Prakash return -1; 12969730ffcbSVarun Prakash 12979730ffcbSVarun Prakash memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); 12989730ffcbSVarun Prakash skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen); 12999730ffcbSVarun Prakash 13009730ffcbSVarun Prakash return 0; 13019730ffcbSVarun Prakash } 13029730ffcbSVarun Prakash 13039730ffcbSVarun Prakash static int 13049730ffcbSVarun Prakash cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx) 13059730ffcbSVarun Prakash { 13069730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx); 13079730ffcbSVarun Prakash int ret; 13089730ffcbSVarun Prakash 13099730ffcbSVarun Prakash cxgbit_rx_pdu_cb(skb) = pdu_cb; 13109730ffcbSVarun Prakash 13119730ffcbSVarun Prakash csk->skb = skb; 13129730ffcbSVarun Prakash 13139730ffcbSVarun Prakash if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) { 13149730ffcbSVarun Prakash ret = cxgbit_rx_login_pdu(csk); 13159730ffcbSVarun Prakash set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 13169730ffcbSVarun Prakash } else { 13179730ffcbSVarun Prakash ret = cxgbit_rx_opcode(csk); 13189730ffcbSVarun Prakash } 13199730ffcbSVarun Prakash 13209730ffcbSVarun Prakash return ret; 13219730ffcbSVarun Prakash } 13229730ffcbSVarun Prakash 13239730ffcbSVarun Prakash static void cxgbit_lro_skb_dump(struct sk_buff *skb) 13249730ffcbSVarun Prakash { 13259730ffcbSVarun Prakash struct skb_shared_info *ssi = skb_shinfo(skb); 13269730ffcbSVarun Prakash struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 13279730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 13289730ffcbSVarun Prakash u8 i; 13299730ffcbSVarun Prakash 13309730ffcbSVarun Prakash pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n", 13319730ffcbSVarun Prakash skb, skb->head, skb->data, skb->len, skb->data_len, 13329730ffcbSVarun Prakash ssi->nr_frags); 13339730ffcbSVarun Prakash pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n", 13349730ffcbSVarun Prakash skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen); 13359730ffcbSVarun Prakash 13369730ffcbSVarun Prakash for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++) 13379730ffcbSVarun Prakash pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, " 13389730ffcbSVarun Prakash "frags %u.\n", 13399730ffcbSVarun Prakash skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq, 13409730ffcbSVarun Prakash pdu_cb->ddigest, pdu_cb->frags); 13419730ffcbSVarun Prakash for (i = 0; i < ssi->nr_frags; i++) 13429730ffcbSVarun Prakash pr_info("skb 0x%p, frag %d, off %u, sz %u.\n", 13439730ffcbSVarun Prakash skb, i, ssi->frags[i].page_offset, ssi->frags[i].size); 13449730ffcbSVarun Prakash } 13459730ffcbSVarun Prakash 13469730ffcbSVarun Prakash static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk) 13479730ffcbSVarun Prakash { 13489730ffcbSVarun Prakash struct sk_buff *skb = csk->lro_hskb; 13499730ffcbSVarun Prakash struct skb_shared_info *ssi = skb_shinfo(skb); 13509730ffcbSVarun Prakash u8 i; 13519730ffcbSVarun Prakash 13529730ffcbSVarun Prakash memset(skb->data, 0, LRO_SKB_MIN_HEADROOM); 13539730ffcbSVarun Prakash for (i = 0; i < ssi->nr_frags; i++) 13549730ffcbSVarun Prakash put_page(skb_frag_page(&ssi->frags[i])); 13559730ffcbSVarun Prakash ssi->nr_frags = 0; 13569730ffcbSVarun Prakash } 13579730ffcbSVarun Prakash 13589730ffcbSVarun Prakash static void 13599730ffcbSVarun Prakash cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx) 13609730ffcbSVarun Prakash { 13619730ffcbSVarun Prakash struct sk_buff *hskb = csk->lro_hskb; 13629730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0); 13639730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx); 13649730ffcbSVarun Prakash struct skb_shared_info *hssi = skb_shinfo(hskb); 13659730ffcbSVarun Prakash struct skb_shared_info *ssi = skb_shinfo(skb); 13669730ffcbSVarun Prakash unsigned int len = 0; 13679730ffcbSVarun Prakash 13689730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_HDR) { 13699730ffcbSVarun Prakash hpdu_cb->flags = pdu_cb->flags; 13709730ffcbSVarun Prakash hpdu_cb->seq = pdu_cb->seq; 13719730ffcbSVarun Prakash hpdu_cb->hdr = pdu_cb->hdr; 13729730ffcbSVarun Prakash hpdu_cb->hlen = pdu_cb->hlen; 13739730ffcbSVarun Prakash 13749730ffcbSVarun Prakash memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx], 13759730ffcbSVarun Prakash sizeof(skb_frag_t)); 13769730ffcbSVarun Prakash 13779730ffcbSVarun Prakash get_page(skb_frag_page(&hssi->frags[0])); 13789730ffcbSVarun Prakash hssi->nr_frags = 1; 13799730ffcbSVarun Prakash hpdu_cb->frags = 1; 13809730ffcbSVarun Prakash hpdu_cb->hfrag_idx = 0; 13819730ffcbSVarun Prakash 13829730ffcbSVarun Prakash len = hssi->frags[0].size; 13839730ffcbSVarun Prakash hskb->len = len; 13849730ffcbSVarun Prakash hskb->data_len = len; 13859730ffcbSVarun Prakash hskb->truesize = len; 13869730ffcbSVarun Prakash } 13879730ffcbSVarun Prakash 13889730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_DATA) { 13899730ffcbSVarun Prakash u8 hfrag_idx = 1, i; 13909730ffcbSVarun Prakash 13919730ffcbSVarun Prakash hpdu_cb->flags |= pdu_cb->flags; 13929730ffcbSVarun Prakash 13939730ffcbSVarun Prakash len = 0; 13949730ffcbSVarun Prakash for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) { 13959730ffcbSVarun Prakash memcpy(&hssi->frags[hfrag_idx], 13969730ffcbSVarun Prakash &ssi->frags[pdu_cb->dfrag_idx + i], 13979730ffcbSVarun Prakash sizeof(skb_frag_t)); 13989730ffcbSVarun Prakash 13999730ffcbSVarun Prakash get_page(skb_frag_page(&hssi->frags[hfrag_idx])); 14009730ffcbSVarun Prakash 14019730ffcbSVarun Prakash len += hssi->frags[hfrag_idx].size; 14029730ffcbSVarun Prakash 14039730ffcbSVarun Prakash hssi->nr_frags++; 14049730ffcbSVarun Prakash hpdu_cb->frags++; 14059730ffcbSVarun Prakash } 14069730ffcbSVarun Prakash 14079730ffcbSVarun Prakash hpdu_cb->dlen = pdu_cb->dlen; 14089730ffcbSVarun Prakash hpdu_cb->doffset = hpdu_cb->hlen; 14099730ffcbSVarun Prakash hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags; 14109730ffcbSVarun Prakash hpdu_cb->dfrag_idx = 1; 14119730ffcbSVarun Prakash hskb->len += len; 14129730ffcbSVarun Prakash hskb->data_len += len; 14139730ffcbSVarun Prakash hskb->truesize += len; 14149730ffcbSVarun Prakash } 14159730ffcbSVarun Prakash 14169730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_STATUS) { 14179730ffcbSVarun Prakash hpdu_cb->flags |= pdu_cb->flags; 14189730ffcbSVarun Prakash 14199730ffcbSVarun Prakash if (hpdu_cb->flags & PDUCBF_RX_DATA) 14209730ffcbSVarun Prakash hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD; 14219730ffcbSVarun Prakash 14229730ffcbSVarun Prakash hpdu_cb->ddigest = pdu_cb->ddigest; 14239730ffcbSVarun Prakash hpdu_cb->pdulen = pdu_cb->pdulen; 14249730ffcbSVarun Prakash } 14259730ffcbSVarun Prakash } 14269730ffcbSVarun Prakash 14279730ffcbSVarun Prakash static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 14289730ffcbSVarun Prakash { 14299730ffcbSVarun Prakash struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 14309730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 14319730ffcbSVarun Prakash u8 pdu_idx = 0, last_idx = 0; 14329730ffcbSVarun Prakash int ret = 0; 14339730ffcbSVarun Prakash 14349730ffcbSVarun Prakash if (!pdu_cb->complete) { 14359730ffcbSVarun Prakash cxgbit_lro_skb_merge(csk, skb, 0); 14369730ffcbSVarun Prakash 14379730ffcbSVarun Prakash if (pdu_cb->flags & PDUCBF_RX_STATUS) { 14389730ffcbSVarun Prakash struct sk_buff *hskb = csk->lro_hskb; 14399730ffcbSVarun Prakash 14409730ffcbSVarun Prakash ret = cxgbit_process_iscsi_pdu(csk, hskb, 0); 14419730ffcbSVarun Prakash 14429730ffcbSVarun Prakash cxgbit_lro_hskb_reset(csk); 14439730ffcbSVarun Prakash 14449730ffcbSVarun Prakash if (ret < 0) 14459730ffcbSVarun Prakash goto out; 14469730ffcbSVarun Prakash } 14479730ffcbSVarun Prakash 14489730ffcbSVarun Prakash pdu_idx = 1; 14499730ffcbSVarun Prakash } 14509730ffcbSVarun Prakash 14519730ffcbSVarun Prakash if (lro_cb->pdu_idx) 14529730ffcbSVarun Prakash last_idx = lro_cb->pdu_idx - 1; 14539730ffcbSVarun Prakash 14549730ffcbSVarun Prakash for (; pdu_idx <= last_idx; pdu_idx++) { 14559730ffcbSVarun Prakash ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx); 14569730ffcbSVarun Prakash if (ret < 0) 14579730ffcbSVarun Prakash goto out; 14589730ffcbSVarun Prakash } 14599730ffcbSVarun Prakash 14609730ffcbSVarun Prakash if ((!lro_cb->complete) && lro_cb->pdu_idx) 14619730ffcbSVarun Prakash cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx); 14629730ffcbSVarun Prakash 14639730ffcbSVarun Prakash out: 14649730ffcbSVarun Prakash return ret; 14659730ffcbSVarun Prakash } 14669730ffcbSVarun Prakash 14679730ffcbSVarun Prakash static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 14689730ffcbSVarun Prakash { 14699730ffcbSVarun Prakash struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb); 14709730ffcbSVarun Prakash struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0); 14719730ffcbSVarun Prakash int ret = -1; 14729730ffcbSVarun Prakash 14739730ffcbSVarun Prakash if ((pdu_cb->flags & PDUCBF_RX_HDR) && 14749730ffcbSVarun Prakash (pdu_cb->seq != csk->rcv_nxt)) { 14759730ffcbSVarun Prakash pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n", 14769730ffcbSVarun Prakash csk, csk->tid, pdu_cb->seq, csk->rcv_nxt); 14779730ffcbSVarun Prakash cxgbit_lro_skb_dump(skb); 14789730ffcbSVarun Prakash return ret; 14799730ffcbSVarun Prakash } 14809730ffcbSVarun Prakash 14819730ffcbSVarun Prakash csk->rcv_nxt += lro_cb->pdu_totallen; 14829730ffcbSVarun Prakash 14839730ffcbSVarun Prakash ret = cxgbit_process_lro_skb(csk, skb); 14849730ffcbSVarun Prakash 14859730ffcbSVarun Prakash csk->rx_credits += lro_cb->pdu_totallen; 14869730ffcbSVarun Prakash 14879730ffcbSVarun Prakash if (csk->rx_credits >= (csk->rcv_win / 4)) 14889730ffcbSVarun Prakash cxgbit_rx_data_ack(csk); 14899730ffcbSVarun Prakash 14909730ffcbSVarun Prakash return ret; 14919730ffcbSVarun Prakash } 14929730ffcbSVarun Prakash 14939730ffcbSVarun Prakash static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb) 14949730ffcbSVarun Prakash { 14959730ffcbSVarun Prakash int ret = -1; 14969730ffcbSVarun Prakash 14979730ffcbSVarun Prakash if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) 14989730ffcbSVarun Prakash ret = cxgbit_rx_lro_skb(csk, skb); 14999730ffcbSVarun Prakash 15009730ffcbSVarun Prakash __kfree_skb(skb); 15019730ffcbSVarun Prakash return ret; 15029730ffcbSVarun Prakash } 15039730ffcbSVarun Prakash 15049730ffcbSVarun Prakash static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq) 15059730ffcbSVarun Prakash { 15069730ffcbSVarun Prakash spin_lock_bh(&csk->rxq.lock); 15079730ffcbSVarun Prakash if (skb_queue_len(&csk->rxq)) { 15089730ffcbSVarun Prakash skb_queue_splice_init(&csk->rxq, rxq); 15099730ffcbSVarun Prakash spin_unlock_bh(&csk->rxq.lock); 15109730ffcbSVarun Prakash return true; 15119730ffcbSVarun Prakash } 15129730ffcbSVarun Prakash spin_unlock_bh(&csk->rxq.lock); 15139730ffcbSVarun Prakash return false; 15149730ffcbSVarun Prakash } 15159730ffcbSVarun Prakash 15169730ffcbSVarun Prakash static int cxgbit_wait_rxq(struct cxgbit_sock *csk) 15179730ffcbSVarun Prakash { 15189730ffcbSVarun Prakash struct sk_buff *skb; 15199730ffcbSVarun Prakash struct sk_buff_head rxq; 15209730ffcbSVarun Prakash 15219730ffcbSVarun Prakash skb_queue_head_init(&rxq); 15229730ffcbSVarun Prakash 15239730ffcbSVarun Prakash wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq)); 15249730ffcbSVarun Prakash 15259730ffcbSVarun Prakash if (signal_pending(current)) 15269730ffcbSVarun Prakash goto out; 15279730ffcbSVarun Prakash 15289730ffcbSVarun Prakash while ((skb = __skb_dequeue(&rxq))) { 15299730ffcbSVarun Prakash if (cxgbit_rx_skb(csk, skb)) 15309730ffcbSVarun Prakash goto out; 15319730ffcbSVarun Prakash } 15329730ffcbSVarun Prakash 15339730ffcbSVarun Prakash return 0; 15349730ffcbSVarun Prakash out: 15359730ffcbSVarun Prakash __skb_queue_purge(&rxq); 15369730ffcbSVarun Prakash return -1; 15379730ffcbSVarun Prakash } 15389730ffcbSVarun Prakash 15399730ffcbSVarun Prakash int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 15409730ffcbSVarun Prakash { 15419730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 15429730ffcbSVarun Prakash int ret = -1; 15439730ffcbSVarun Prakash 15449730ffcbSVarun Prakash while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) { 15459730ffcbSVarun Prakash ret = cxgbit_wait_rxq(csk); 15469730ffcbSVarun Prakash if (ret) { 15479730ffcbSVarun Prakash clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags); 15489730ffcbSVarun Prakash break; 15499730ffcbSVarun Prakash } 15509730ffcbSVarun Prakash } 15519730ffcbSVarun Prakash 15529730ffcbSVarun Prakash return ret; 15539730ffcbSVarun Prakash } 15549730ffcbSVarun Prakash 15559730ffcbSVarun Prakash void cxgbit_get_rx_pdu(struct iscsi_conn *conn) 15569730ffcbSVarun Prakash { 15579730ffcbSVarun Prakash struct cxgbit_sock *csk = conn->context; 15589730ffcbSVarun Prakash 15599730ffcbSVarun Prakash while (!kthread_should_stop()) { 15609730ffcbSVarun Prakash iscsit_thread_check_cpumask(conn, current, 0); 15619730ffcbSVarun Prakash if (cxgbit_wait_rxq(csk)) 15629730ffcbSVarun Prakash return; 15639730ffcbSVarun Prakash } 15649730ffcbSVarun Prakash } 1565