1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2016 Chelsio Communications, Inc.
4  */
5 
6 #include <linux/workqueue.h>
7 #include <linux/kthread.h>
8 #include <linux/sched/signal.h>
9 
10 #include <asm/unaligned.h>
11 #include <net/tcp.h>
12 #include <target/target_core_base.h>
13 #include <target/target_core_fabric.h>
14 #include "cxgbit.h"
15 
16 struct sge_opaque_hdr {
17 	void *dev;
18 	dma_addr_t addr[MAX_SKB_FRAGS + 1];
19 };
20 
21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
22 
23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
24 		    sizeof(struct fw_ofld_tx_data_wr))
25 
26 static struct sk_buff *
27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
28 {
29 	struct sk_buff *skb = NULL;
30 	u8 submode = 0;
31 	int errcode;
32 	static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
33 
34 	if (len) {
35 		skb = alloc_skb_with_frags(hdr_len, len,
36 					   0, &errcode,
37 					   GFP_KERNEL);
38 		if (!skb)
39 			return NULL;
40 
41 		skb_reserve(skb, TX_HDR_LEN);
42 		skb_reset_transport_header(skb);
43 		__skb_put(skb, ISCSI_HDR_LEN);
44 		skb->data_len = len;
45 		skb->len += len;
46 		submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
47 
48 	} else {
49 		u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
50 
51 		skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
52 		if (!skb)
53 			return NULL;
54 
55 		skb_reserve(skb, TX_HDR_LEN + iso_len);
56 		skb_reset_transport_header(skb);
57 		__skb_put(skb, ISCSI_HDR_LEN);
58 	}
59 
60 	submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
61 	cxgbit_skcb_submode(skb) = submode;
62 	cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
63 	cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
64 	return skb;
65 }
66 
67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
68 {
69 	return __cxgbit_alloc_skb(csk, len, false);
70 }
71 
72 /*
73  * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
74  * @skb: the packet
75  *
76  * Returns true if a packet can be sent as an offload WR with immediate
77  * data.  We currently use the same limit as for Ethernet packets.
78  */
79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
80 {
81 	int length = skb->len;
82 
83 	if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
84 		length += sizeof(struct fw_ofld_tx_data_wr);
85 
86 	if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
87 		length += sizeof(struct cpl_tx_data_iso);
88 
89 	return length <= MAX_IMM_OFLD_TX_DATA_WR_LEN;
90 }
91 
92 /*
93  * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
94  * @n: the number of SGL entries
95  * Calculates the number of flits needed for a scatter/gather list that
96  * can hold the given number of entries.
97  */
98 static inline unsigned int cxgbit_sgl_len(unsigned int n)
99 {
100 	n--;
101 	return (3 * n) / 2 + (n & 1) + 2;
102 }
103 
104 /*
105  * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
106  * @skb: the packet
107  *
108  * Returns the number of flits needed for the given offload packet.
109  * These packets are already fully constructed and no additional headers
110  * will be added.
111  */
112 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
113 {
114 	unsigned int flits, cnt;
115 
116 	if (cxgbit_is_ofld_imm(skb))
117 		return DIV_ROUND_UP(skb->len, 8);
118 	flits = skb_transport_offset(skb) / 8;
119 	cnt = skb_shinfo(skb)->nr_frags;
120 	if (skb_tail_pointer(skb) != skb_transport_header(skb))
121 		cnt++;
122 	return flits + cxgbit_sgl_len(cnt);
123 }
124 
125 #define CXGBIT_ISO_FSLICE 0x1
126 #define CXGBIT_ISO_LSLICE 0x2
127 static void
128 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
129 {
130 	struct cpl_tx_data_iso *cpl;
131 	unsigned int submode = cxgbit_skcb_submode(skb);
132 	unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
133 	unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
134 
135 	cpl = __skb_push(skb, sizeof(*cpl));
136 
137 	cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
138 			CPL_TX_DATA_ISO_FIRST_V(fslice) |
139 			CPL_TX_DATA_ISO_LAST_V(lslice) |
140 			CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
141 			CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
142 			CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
143 			CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
144 			CPL_TX_DATA_ISO_SCSI_V(2));
145 
146 	cpl->ahs_len = 0;
147 	cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
148 	cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
149 	cpl->len = htonl(iso_info->len);
150 	cpl->reserved2_seglen_offset = htonl(0);
151 	cpl->datasn_offset = htonl(0);
152 	cpl->buffer_offset = htonl(0);
153 	cpl->reserved3 = 0;
154 
155 	__skb_pull(skb, sizeof(*cpl));
156 }
157 
158 static void
159 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
160 		  u32 len, u32 credits, u32 compl)
161 {
162 	struct fw_ofld_tx_data_wr *req;
163 	const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
164 	u32 submode = cxgbit_skcb_submode(skb);
165 	u32 wr_ulp_mode = 0;
166 	u32 hdr_size = sizeof(*req);
167 	u32 opcode = FW_OFLD_TX_DATA_WR;
168 	u32 immlen = 0;
169 	u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
170 		    T6_TX_FORCE_F;
171 
172 	if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
173 		opcode = FW_ISCSI_TX_DATA_WR;
174 		immlen += sizeof(struct cpl_tx_data_iso);
175 		hdr_size += sizeof(struct cpl_tx_data_iso);
176 		submode |= 8;
177 	}
178 
179 	if (cxgbit_is_ofld_imm(skb))
180 		immlen += dlen;
181 
182 	req = __skb_push(skb, hdr_size);
183 	req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
184 					FW_WR_COMPL_V(compl) |
185 					FW_WR_IMMDLEN_V(immlen));
186 	req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
187 					FW_WR_LEN16_V(credits));
188 	req->plen = htonl(len);
189 	wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
190 				FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
191 
192 	req->tunnel_to_proxy = htonl(wr_ulp_mode | force |
193 				     FW_OFLD_TX_DATA_WR_SHOVE_F);
194 }
195 
196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
197 {
198 	kfree_skb(skb);
199 }
200 
201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
202 {
203 	struct sk_buff *skb;
204 
205 	while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
206 		u32 dlen = skb->len;
207 		u32 len = skb->len;
208 		u32 credits_needed;
209 		u32 compl = 0;
210 		u32 flowclen16 = 0;
211 		u32 iso_cpl_len = 0;
212 
213 		if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
214 			iso_cpl_len = sizeof(struct cpl_tx_data_iso);
215 
216 		if (cxgbit_is_ofld_imm(skb))
217 			credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
218 		else
219 			credits_needed = DIV_ROUND_UP((8 *
220 					cxgbit_calc_tx_flits_ofld(skb)) +
221 					iso_cpl_len, 16);
222 
223 		if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
224 			credits_needed += DIV_ROUND_UP(
225 				sizeof(struct fw_ofld_tx_data_wr), 16);
226 		/*
227 		 * Assumes the initial credits is large enough to support
228 		 * fw_flowc_wr plus largest possible first payload
229 		 */
230 
231 		if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
232 			flowclen16 = cxgbit_send_tx_flowc_wr(csk);
233 			csk->wr_cred -= flowclen16;
234 			csk->wr_una_cred += flowclen16;
235 		}
236 
237 		if (csk->wr_cred < credits_needed) {
238 			pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
239 				 csk, skb->len, skb->data_len,
240 				 credits_needed, csk->wr_cred);
241 			break;
242 		}
243 		__skb_unlink(skb, &csk->txq);
244 		set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
245 		skb->csum = (__force __wsum)(credits_needed + flowclen16);
246 		csk->wr_cred -= credits_needed;
247 		csk->wr_una_cred += credits_needed;
248 
249 		pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
250 			 csk, skb->len, skb->data_len, credits_needed,
251 			 csk->wr_cred, csk->wr_una_cred);
252 
253 		if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
254 			len += cxgbit_skcb_tx_extralen(skb);
255 
256 			if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
257 			    (!before(csk->write_seq,
258 				     csk->snd_una + csk->snd_win))) {
259 				compl = 1;
260 				csk->wr_una_cred = 0;
261 			}
262 
263 			cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
264 					  compl);
265 			csk->snd_nxt += len;
266 
267 		} else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
268 			   (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
269 			struct cpl_close_con_req *req =
270 				(struct cpl_close_con_req *)skb->data;
271 			req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
272 			csk->wr_una_cred = 0;
273 		}
274 
275 		cxgbit_sock_enqueue_wr(csk, skb);
276 		t4_set_arp_err_handler(skb, csk,
277 				       cxgbit_arp_failure_skb_discard);
278 
279 		pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
280 			 csk, csk->tid, skb, len);
281 
282 		cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
283 	}
284 }
285 
286 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
287 {
288 	struct sk_buff_head backlogq;
289 	struct sk_buff *skb;
290 	void (*fn)(struct cxgbit_sock *, struct sk_buff *);
291 
292 	skb_queue_head_init(&backlogq);
293 
294 	spin_lock_bh(&csk->lock);
295 	while (skb_queue_len(&csk->backlogq)) {
296 		skb_queue_splice_init(&csk->backlogq, &backlogq);
297 		spin_unlock_bh(&csk->lock);
298 
299 		while ((skb = __skb_dequeue(&backlogq))) {
300 			fn = cxgbit_skcb_rx_backlog_fn(skb);
301 			fn(csk, skb);
302 		}
303 
304 		spin_lock_bh(&csk->lock);
305 	}
306 
307 	csk->lock_owner = false;
308 	spin_unlock_bh(&csk->lock);
309 }
310 
311 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
312 {
313 	int ret = 0;
314 
315 	spin_lock_bh(&csk->lock);
316 	csk->lock_owner = true;
317 	spin_unlock_bh(&csk->lock);
318 
319 	if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
320 		     signal_pending(current))) {
321 		__kfree_skb(skb);
322 		__skb_queue_purge(&csk->ppodq);
323 		ret = -1;
324 		goto unlock;
325 	}
326 
327 	csk->write_seq += skb->len +
328 			  cxgbit_skcb_tx_extralen(skb);
329 
330 	skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
331 	__skb_queue_tail(&csk->txq, skb);
332 	cxgbit_push_tx_frames(csk);
333 
334 unlock:
335 	cxgbit_unlock_sock(csk);
336 	return ret;
337 }
338 
339 static int
340 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
341 	       u32 data_length)
342 {
343 	u32 i = 0, nr_frags = MAX_SKB_FRAGS;
344 	u32 padding = ((-data_length) & 3);
345 	struct scatterlist *sg;
346 	struct page *page;
347 	unsigned int page_off;
348 
349 	if (padding)
350 		nr_frags--;
351 
352 	/*
353 	 * We know each entry in t_data_sg contains a page.
354 	 */
355 	sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
356 	page_off = (data_offset % PAGE_SIZE);
357 
358 	while (data_length && (i < nr_frags)) {
359 		u32 cur_len = min_t(u32, data_length, sg->length - page_off);
360 
361 		page = sg_page(sg);
362 
363 		get_page(page);
364 		skb_fill_page_desc(skb, i, page, sg->offset + page_off,
365 				   cur_len);
366 		skb->data_len += cur_len;
367 		skb->len += cur_len;
368 		skb->truesize += cur_len;
369 
370 		data_length -= cur_len;
371 		page_off = 0;
372 		sg = sg_next(sg);
373 		i++;
374 	}
375 
376 	if (data_length)
377 		return -1;
378 
379 	if (padding) {
380 		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
381 		if (!page)
382 			return -1;
383 		skb_fill_page_desc(skb, i, page, 0, padding);
384 		skb->data_len += padding;
385 		skb->len += padding;
386 		skb->truesize += padding;
387 	}
388 
389 	return 0;
390 }
391 
392 static int
393 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
394 		     struct iscsi_datain_req *dr)
395 {
396 	struct iscsi_conn *conn = csk->conn;
397 	struct sk_buff *skb;
398 	struct iscsi_datain datain;
399 	struct cxgbit_iso_info iso_info;
400 	u32 data_length = cmd->se_cmd.data_length;
401 	u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
402 	u32 num_pdu, plen, tx_data = 0;
403 	bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
404 		SCF_TRANSPORT_TASK_SENSE);
405 	bool set_statsn = false;
406 	int ret = -1;
407 
408 	while (data_length) {
409 		num_pdu = (data_length + mrdsl - 1) / mrdsl;
410 		if (num_pdu > csk->max_iso_npdu)
411 			num_pdu = csk->max_iso_npdu;
412 
413 		plen = num_pdu * mrdsl;
414 		if (plen > data_length)
415 			plen = data_length;
416 
417 		skb = __cxgbit_alloc_skb(csk, 0, true);
418 		if (unlikely(!skb))
419 			return -ENOMEM;
420 
421 		memset(skb->data, 0, ISCSI_HDR_LEN);
422 		cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
423 		cxgbit_skcb_submode(skb) |= (csk->submode &
424 				CXGBIT_SUBMODE_DCRC);
425 		cxgbit_skcb_tx_extralen(skb) = (num_pdu *
426 				cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
427 						((num_pdu - 1) * ISCSI_HDR_LEN);
428 
429 		memset(&datain, 0, sizeof(struct iscsi_datain));
430 		memset(&iso_info, 0, sizeof(iso_info));
431 
432 		if (!tx_data)
433 			iso_info.flags |= CXGBIT_ISO_FSLICE;
434 
435 		if (!(data_length - plen)) {
436 			iso_info.flags |= CXGBIT_ISO_LSLICE;
437 			if (!task_sense) {
438 				datain.flags = ISCSI_FLAG_DATA_STATUS;
439 				iscsit_increment_maxcmdsn(cmd, conn->sess);
440 				cmd->stat_sn = conn->stat_sn++;
441 				set_statsn = true;
442 			}
443 		}
444 
445 		iso_info.burst_len = num_pdu * mrdsl;
446 		iso_info.mpdu = mrdsl;
447 		iso_info.len = ISCSI_HDR_LEN + plen;
448 
449 		cxgbit_cpl_tx_data_iso(skb, &iso_info);
450 
451 		datain.offset = tx_data;
452 		datain.data_sn = cmd->data_sn - 1;
453 
454 		iscsit_build_datain_pdu(cmd, conn, &datain,
455 					(struct iscsi_data_rsp *)skb->data,
456 					set_statsn);
457 
458 		ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
459 		if (unlikely(ret)) {
460 			__kfree_skb(skb);
461 			goto out;
462 		}
463 
464 		ret = cxgbit_queue_skb(csk, skb);
465 		if (unlikely(ret))
466 			goto out;
467 
468 		tx_data += plen;
469 		data_length -= plen;
470 
471 		cmd->read_data_done += plen;
472 		cmd->data_sn += num_pdu;
473 	}
474 
475 	dr->dr_complete = DATAIN_COMPLETE_NORMAL;
476 
477 	return 0;
478 
479 out:
480 	return ret;
481 }
482 
483 static int
484 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
485 		 const struct iscsi_datain *datain)
486 {
487 	struct sk_buff *skb;
488 	int ret = 0;
489 
490 	skb = cxgbit_alloc_skb(csk, 0);
491 	if (unlikely(!skb))
492 		return -ENOMEM;
493 
494 	memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
495 
496 	if (datain->length) {
497 		cxgbit_skcb_submode(skb) |= (csk->submode &
498 				CXGBIT_SUBMODE_DCRC);
499 		cxgbit_skcb_tx_extralen(skb) =
500 				cxgbit_digest_len[cxgbit_skcb_submode(skb)];
501 	}
502 
503 	ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
504 	if (ret < 0) {
505 		__kfree_skb(skb);
506 		return ret;
507 	}
508 
509 	return cxgbit_queue_skb(csk, skb);
510 }
511 
512 static int
513 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
514 		       struct iscsi_datain_req *dr,
515 		       const struct iscsi_datain *datain)
516 {
517 	struct cxgbit_sock *csk = conn->context;
518 	u32 data_length = cmd->se_cmd.data_length;
519 	u32 padding = ((-data_length) & 3);
520 	u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
521 
522 	if ((data_length > mrdsl) && (!dr->recovery) &&
523 	    (!padding) && (!datain->offset) && csk->max_iso_npdu) {
524 		atomic_long_add(data_length - datain->length,
525 				&conn->sess->tx_data_octets);
526 		return cxgbit_tx_datain_iso(csk, cmd, dr);
527 	}
528 
529 	return cxgbit_tx_datain(csk, cmd, datain);
530 }
531 
532 static int
533 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
534 			  const void *data_buf, u32 data_buf_len)
535 {
536 	struct cxgbit_sock *csk = conn->context;
537 	struct sk_buff *skb;
538 	u32 padding = ((-data_buf_len) & 3);
539 
540 	skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
541 	if (unlikely(!skb))
542 		return -ENOMEM;
543 
544 	memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
545 
546 	if (data_buf_len) {
547 		u32 pad_bytes = 0;
548 
549 		skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
550 
551 		if (padding)
552 			skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
553 				       &pad_bytes, padding);
554 	}
555 
556 	cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
557 				       cxgbit_skcb_submode(skb)];
558 
559 	return cxgbit_queue_skb(csk, skb);
560 }
561 
562 int
563 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
564 		struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
565 {
566 	if (dr)
567 		return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
568 	else
569 		return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
570 }
571 
572 int cxgbit_validate_params(struct iscsi_conn *conn)
573 {
574 	struct cxgbit_sock *csk = conn->context;
575 	struct cxgbit_device *cdev = csk->com.cdev;
576 	struct iscsi_param *param;
577 	u32 max_xmitdsl;
578 
579 	param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
580 					  conn->param_list);
581 	if (!param)
582 		return -1;
583 
584 	if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
585 		return -1;
586 
587 	if (max_xmitdsl > cdev->mdsl) {
588 		if (iscsi_change_param_sprintf(
589 			conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
590 			return -1;
591 	}
592 
593 	return 0;
594 }
595 
596 static int cxgbit_set_digest(struct cxgbit_sock *csk)
597 {
598 	struct iscsi_conn *conn = csk->conn;
599 	struct iscsi_param *param;
600 
601 	param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
602 	if (!param) {
603 		pr_err("param not found key %s\n", HEADERDIGEST);
604 		return -1;
605 	}
606 
607 	if (!strcmp(param->value, CRC32C))
608 		csk->submode |= CXGBIT_SUBMODE_HCRC;
609 
610 	param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
611 	if (!param) {
612 		csk->submode = 0;
613 		pr_err("param not found key %s\n", DATADIGEST);
614 		return -1;
615 	}
616 
617 	if (!strcmp(param->value, CRC32C))
618 		csk->submode |= CXGBIT_SUBMODE_DCRC;
619 
620 	if (cxgbit_setup_conn_digest(csk)) {
621 		csk->submode = 0;
622 		return -1;
623 	}
624 
625 	return 0;
626 }
627 
628 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
629 {
630 	struct iscsi_conn *conn = csk->conn;
631 	struct iscsi_conn_ops *conn_ops = conn->conn_ops;
632 	struct iscsi_param *param;
633 	u32 mrdsl, mbl;
634 	u32 max_npdu, max_iso_npdu;
635 	u32 max_iso_payload;
636 
637 	if (conn->login->leading_connection) {
638 		param = iscsi_find_param_from_key(MAXBURSTLENGTH,
639 						  conn->param_list);
640 		if (!param) {
641 			pr_err("param not found key %s\n", MAXBURSTLENGTH);
642 			return -1;
643 		}
644 
645 		if (kstrtou32(param->value, 0, &mbl) < 0)
646 			return -1;
647 	} else {
648 		mbl = conn->sess->sess_ops->MaxBurstLength;
649 	}
650 
651 	mrdsl = conn_ops->MaxRecvDataSegmentLength;
652 	max_npdu = mbl / mrdsl;
653 
654 	max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
655 
656 	max_iso_npdu = max_iso_payload /
657 		       (ISCSI_HDR_LEN + mrdsl +
658 			cxgbit_digest_len[csk->submode]);
659 
660 	csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
661 
662 	if (csk->max_iso_npdu <= 1)
663 		csk->max_iso_npdu = 0;
664 
665 	return 0;
666 }
667 
668 /*
669  * cxgbit_seq_pdu_inorder()
670  * @csk: pointer to cxgbit socket structure
671  *
672  * This function checks whether data sequence and data
673  * pdu are in order.
674  *
675  * Return: returns -1 on error, 0 if data sequence and
676  * data pdu are in order, 1 if data sequence or data pdu
677  * is not in order.
678  */
679 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
680 {
681 	struct iscsi_conn *conn = csk->conn;
682 	struct iscsi_param *param;
683 
684 	if (conn->login->leading_connection) {
685 		param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
686 						  conn->param_list);
687 		if (!param) {
688 			pr_err("param not found key %s\n", DATASEQUENCEINORDER);
689 			return -1;
690 		}
691 
692 		if (strcmp(param->value, YES))
693 			return 1;
694 
695 		param = iscsi_find_param_from_key(DATAPDUINORDER,
696 						  conn->param_list);
697 		if (!param) {
698 			pr_err("param not found key %s\n", DATAPDUINORDER);
699 			return -1;
700 		}
701 
702 		if (strcmp(param->value, YES))
703 			return 1;
704 
705 	} else {
706 		if (!conn->sess->sess_ops->DataSequenceInOrder)
707 			return 1;
708 		if (!conn->sess->sess_ops->DataPDUInOrder)
709 			return 1;
710 	}
711 
712 	return 0;
713 }
714 
715 static int cxgbit_set_params(struct iscsi_conn *conn)
716 {
717 	struct cxgbit_sock *csk = conn->context;
718 	struct cxgbit_device *cdev = csk->com.cdev;
719 	struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
720 	struct iscsi_conn_ops *conn_ops = conn->conn_ops;
721 	struct iscsi_param *param;
722 	u8 erl;
723 
724 	if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
725 		conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
726 
727 	if (cxgbit_set_digest(csk))
728 		return -1;
729 
730 	if (conn->login->leading_connection) {
731 		param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
732 						  conn->param_list);
733 		if (!param) {
734 			pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
735 			return -1;
736 		}
737 		if (kstrtou8(param->value, 0, &erl) < 0)
738 			return -1;
739 	} else {
740 		erl = conn->sess->sess_ops->ErrorRecoveryLevel;
741 	}
742 
743 	if (!erl) {
744 		int ret;
745 
746 		ret = cxgbit_seq_pdu_inorder(csk);
747 		if (ret < 0) {
748 			return -1;
749 		} else if (ret > 0) {
750 			if (is_t5(cdev->lldi.adapter_type))
751 				goto enable_ddp;
752 			else
753 				return 0;
754 		}
755 
756 		if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
757 			if (cxgbit_set_iso_npdu(csk))
758 				return -1;
759 		}
760 
761 enable_ddp:
762 		if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
763 			if (cxgbit_setup_conn_pgidx(csk,
764 						    ppm->tformat.pgsz_idx_dflt))
765 				return -1;
766 			set_bit(CSK_DDP_ENABLE, &csk->com.flags);
767 		}
768 	}
769 
770 	return 0;
771 }
772 
773 int
774 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
775 		    u32 length)
776 {
777 	struct cxgbit_sock *csk = conn->context;
778 	struct sk_buff *skb;
779 	u32 padding_buf = 0;
780 	u8 padding = ((-length) & 3);
781 
782 	skb = cxgbit_alloc_skb(csk, length + padding);
783 	if (!skb)
784 		return -ENOMEM;
785 	skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
786 	skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
787 
788 	if (padding)
789 		skb_store_bits(skb, ISCSI_HDR_LEN + length,
790 			       &padding_buf, padding);
791 
792 	if (login->login_complete) {
793 		if (cxgbit_set_params(conn)) {
794 			kfree_skb(skb);
795 			return -1;
796 		}
797 
798 		set_bit(CSK_LOGIN_DONE, &csk->com.flags);
799 	}
800 
801 	if (cxgbit_queue_skb(csk, skb))
802 		return -1;
803 
804 	if ((!login->login_complete) && (!login->login_failed))
805 		schedule_delayed_work(&conn->login_work, 0);
806 
807 	return 0;
808 }
809 
810 static void
811 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
812 		      unsigned int nents, u32 skip)
813 {
814 	struct skb_seq_state st;
815 	const u8 *buf;
816 	unsigned int consumed = 0, buf_len;
817 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
818 
819 	skb_prepare_seq_read(skb, pdu_cb->doffset,
820 			     pdu_cb->doffset + pdu_cb->dlen,
821 			     &st);
822 
823 	while (true) {
824 		buf_len = skb_seq_read(consumed, &buf, &st);
825 		if (!buf_len) {
826 			skb_abort_seq_read(&st);
827 			break;
828 		}
829 
830 		consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
831 						 buf_len, skip + consumed);
832 	}
833 }
834 
835 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
836 {
837 	struct iscsi_conn *conn = csk->conn;
838 	struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
839 	struct cxgbit_cmd *ccmd;
840 	struct iscsi_cmd *cmd;
841 
842 	cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
843 	if (!cmd) {
844 		pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
845 		return NULL;
846 	}
847 
848 	ccmd = iscsit_priv_cmd(cmd);
849 	ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
850 	ccmd->setup_ddp = true;
851 
852 	return cmd;
853 }
854 
855 static int
856 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
857 			     u32 length)
858 {
859 	struct iscsi_conn *conn = cmd->conn;
860 	struct cxgbit_sock *csk = conn->context;
861 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
862 
863 	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
864 		pr_err("ImmediateData CRC32C DataDigest error\n");
865 		if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
866 			pr_err("Unable to recover from"
867 			       " Immediate Data digest failure while"
868 			       " in ERL=0.\n");
869 			iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
870 					  (unsigned char *)hdr);
871 			return IMMEDIATE_DATA_CANNOT_RECOVER;
872 		}
873 
874 		iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
875 				  (unsigned char *)hdr);
876 		return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
877 	}
878 
879 	if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
880 		struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
881 		struct skb_shared_info *ssi = skb_shinfo(csk->skb);
882 		skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
883 
884 		sg_init_table(&ccmd->sg, 1);
885 		sg_set_page(&ccmd->sg, skb_frag_page(dfrag),
886 				skb_frag_size(dfrag), skb_frag_off(dfrag));
887 		get_page(skb_frag_page(dfrag));
888 
889 		cmd->se_cmd.t_data_sg = &ccmd->sg;
890 		cmd->se_cmd.t_data_nents = 1;
891 
892 		ccmd->release = true;
893 	} else {
894 		struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
895 		u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
896 
897 		cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
898 	}
899 
900 	cmd->write_data_done += pdu_cb->dlen;
901 
902 	if (cmd->write_data_done == cmd->se_cmd.data_length) {
903 		spin_lock_bh(&cmd->istate_lock);
904 		cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
905 		cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
906 		spin_unlock_bh(&cmd->istate_lock);
907 	}
908 
909 	return IMMEDIATE_DATA_NORMAL_OPERATION;
910 }
911 
912 static int
913 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
914 			  bool dump_payload)
915 {
916 	struct iscsi_conn *conn = cmd->conn;
917 	int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
918 	/*
919 	 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
920 	 */
921 	if (dump_payload)
922 		goto after_immediate_data;
923 
924 	immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
925 						 cmd->first_burst_len);
926 after_immediate_data:
927 	if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
928 		/*
929 		 * A PDU/CmdSN carrying Immediate Data passed
930 		 * DataCRC, check against ExpCmdSN/MaxCmdSN if
931 		 * Immediate Bit is not set.
932 		 */
933 		cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
934 						(unsigned char *)hdr,
935 						hdr->cmdsn);
936 		if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
937 			return -1;
938 
939 		if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
940 			target_put_sess_cmd(&cmd->se_cmd);
941 			return 0;
942 		} else if (cmd->unsolicited_data) {
943 			iscsit_set_unsolicited_dataout(cmd);
944 		}
945 
946 	} else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
947 		/*
948 		 * Immediate Data failed DataCRC and ERL>=1,
949 		 * silently drop this PDU and let the initiator
950 		 * plug the CmdSN gap.
951 		 *
952 		 * FIXME: Send Unsolicited NOPIN with reserved
953 		 * TTT here to help the initiator figure out
954 		 * the missing CmdSN, although they should be
955 		 * intelligent enough to determine the missing
956 		 * CmdSN and issue a retry to plug the sequence.
957 		 */
958 		cmd->i_state = ISTATE_REMOVE;
959 		iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
960 	} else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
961 		return -1;
962 
963 	return 0;
964 }
965 
966 static int
967 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
968 {
969 	struct iscsi_conn *conn = csk->conn;
970 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
971 	struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
972 	int rc;
973 	bool dump_payload = false;
974 
975 	rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
976 	if (rc < 0)
977 		return rc;
978 
979 	if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
980 	    (pdu_cb->nr_dfrags == 1))
981 		cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
982 
983 	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
984 	if (rc < 0)
985 		return 0;
986 	else if (rc > 0)
987 		dump_payload = true;
988 
989 	if (!pdu_cb->dlen)
990 		return 0;
991 
992 	return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
993 }
994 
995 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
996 {
997 	struct scatterlist *sg_start;
998 	struct iscsi_conn *conn = csk->conn;
999 	struct iscsi_cmd *cmd = NULL;
1000 	struct cxgbit_cmd *ccmd;
1001 	struct cxgbi_task_tag_info *ttinfo;
1002 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1003 	struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1004 	u32 data_offset = be32_to_cpu(hdr->offset);
1005 	u32 data_len = ntoh24(hdr->dlength);
1006 	int rc, sg_nents, sg_off;
1007 	bool dcrc_err = false;
1008 
1009 	if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1010 		u32 offset = be32_to_cpu(hdr->offset);
1011 		u32 ddp_data_len;
1012 		bool success = false;
1013 
1014 		cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1015 		if (!cmd)
1016 			return 0;
1017 
1018 		ddp_data_len = offset - cmd->write_data_done;
1019 		atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1020 
1021 		cmd->write_data_done = offset;
1022 		cmd->next_burst_len = ddp_data_len;
1023 		cmd->data_sn = be32_to_cpu(hdr->datasn);
1024 
1025 		rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1026 						cmd, data_len, &success);
1027 		if (rc < 0)
1028 			return rc;
1029 		else if (!success)
1030 			return 0;
1031 	} else {
1032 		rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1033 		if (rc < 0)
1034 			return rc;
1035 		else if (!cmd)
1036 			return 0;
1037 	}
1038 
1039 	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1040 		pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1041 		       " DataSN: 0x%08x\n",
1042 		       hdr->itt, hdr->offset, data_len,
1043 		       hdr->datasn);
1044 
1045 		dcrc_err = true;
1046 		goto check_payload;
1047 	}
1048 
1049 	pr_debug("DataOut data_len: %u, "
1050 		"write_data_done: %u, data_length: %u\n",
1051 		  data_len,  cmd->write_data_done,
1052 		  cmd->se_cmd.data_length);
1053 
1054 	if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1055 		u32 skip = data_offset % PAGE_SIZE;
1056 
1057 		sg_off = data_offset / PAGE_SIZE;
1058 		sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1059 		sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1060 
1061 		cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1062 	}
1063 
1064 	ccmd = iscsit_priv_cmd(cmd);
1065 	ttinfo = &ccmd->ttinfo;
1066 
1067 	if (ccmd->release && ttinfo->sgl &&
1068 	    (cmd->se_cmd.data_length ==	(cmd->write_data_done + data_len))) {
1069 		struct cxgbit_device *cdev = csk->com.cdev;
1070 		struct cxgbi_ppm *ppm = cdev2ppm(cdev);
1071 
1072 		dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents,
1073 			     DMA_FROM_DEVICE);
1074 		ttinfo->nents = 0;
1075 		ttinfo->sgl = NULL;
1076 	}
1077 
1078 check_payload:
1079 
1080 	rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1081 	if (rc < 0)
1082 		return rc;
1083 
1084 	return 0;
1085 }
1086 
1087 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1088 {
1089 	struct iscsi_conn *conn = csk->conn;
1090 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1091 	struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1092 	unsigned char *ping_data = NULL;
1093 	u32 payload_length = pdu_cb->dlen;
1094 	int ret;
1095 
1096 	ret = iscsit_setup_nop_out(conn, cmd, hdr);
1097 	if (ret < 0)
1098 		return 0;
1099 
1100 	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1101 		if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1102 			pr_err("Unable to recover from"
1103 			       " NOPOUT Ping DataCRC failure while in"
1104 			       " ERL=0.\n");
1105 			ret = -1;
1106 			goto out;
1107 		} else {
1108 			/*
1109 			 * drop this PDU and let the
1110 			 * initiator plug the CmdSN gap.
1111 			 */
1112 			pr_info("Dropping NOPOUT"
1113 				" Command CmdSN: 0x%08x due to"
1114 				" DataCRC error.\n", hdr->cmdsn);
1115 			ret = 0;
1116 			goto out;
1117 		}
1118 	}
1119 
1120 	/*
1121 	 * Handle NOP-OUT payload for traditional iSCSI sockets
1122 	 */
1123 	if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1124 		ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1125 		if (!ping_data) {
1126 			pr_err("Unable to allocate memory for"
1127 				" NOPOUT ping data.\n");
1128 			ret = -1;
1129 			goto out;
1130 		}
1131 
1132 		skb_copy_bits(csk->skb, pdu_cb->doffset,
1133 			      ping_data, payload_length);
1134 
1135 		ping_data[payload_length] = '\0';
1136 		/*
1137 		 * Attach ping data to struct iscsi_cmd->buf_ptr.
1138 		 */
1139 		cmd->buf_ptr = ping_data;
1140 		cmd->buf_ptr_size = payload_length;
1141 
1142 		pr_debug("Got %u bytes of NOPOUT ping"
1143 			" data.\n", payload_length);
1144 		pr_debug("Ping Data: \"%s\"\n", ping_data);
1145 	}
1146 
1147 	return iscsit_process_nop_out(conn, cmd, hdr);
1148 out:
1149 	if (cmd)
1150 		iscsit_free_cmd(cmd, false);
1151 	return ret;
1152 }
1153 
1154 static int
1155 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1156 {
1157 	struct iscsi_conn *conn = csk->conn;
1158 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1159 	struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1160 	u32 payload_length = pdu_cb->dlen;
1161 	int rc;
1162 	unsigned char *text_in = NULL;
1163 
1164 	rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1165 	if (rc < 0)
1166 		return rc;
1167 
1168 	if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1169 		if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1170 			pr_err("Unable to recover from"
1171 			       " Text Data digest failure while in"
1172 			       " ERL=0.\n");
1173 			goto reject;
1174 		} else {
1175 			/*
1176 			 * drop this PDU and let the
1177 			 * initiator plug the CmdSN gap.
1178 			 */
1179 			pr_info("Dropping Text"
1180 				" Command CmdSN: 0x%08x due to"
1181 				" DataCRC error.\n", hdr->cmdsn);
1182 			return 0;
1183 		}
1184 	}
1185 
1186 	if (payload_length) {
1187 		text_in = kzalloc(payload_length, GFP_KERNEL);
1188 		if (!text_in) {
1189 			pr_err("Unable to allocate text_in of payload_length: %u\n",
1190 			       payload_length);
1191 			return -ENOMEM;
1192 		}
1193 		skb_copy_bits(csk->skb, pdu_cb->doffset,
1194 			      text_in, payload_length);
1195 
1196 		text_in[payload_length - 1] = '\0';
1197 
1198 		cmd->text_in_ptr = text_in;
1199 	}
1200 
1201 	return iscsit_process_text_cmd(conn, cmd, hdr);
1202 
1203 reject:
1204 	return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1205 				 pdu_cb->hdr);
1206 }
1207 
1208 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1209 {
1210 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1211 	struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1212 	struct iscsi_conn *conn = csk->conn;
1213 	struct iscsi_cmd *cmd = NULL;
1214 	u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1215 	int ret = -EINVAL;
1216 
1217 	switch (opcode) {
1218 	case ISCSI_OP_SCSI_CMD:
1219 		cmd = cxgbit_allocate_cmd(csk);
1220 		if (!cmd)
1221 			goto reject;
1222 
1223 		ret = cxgbit_handle_scsi_cmd(csk, cmd);
1224 		break;
1225 	case ISCSI_OP_SCSI_DATA_OUT:
1226 		ret = cxgbit_handle_iscsi_dataout(csk);
1227 		break;
1228 	case ISCSI_OP_NOOP_OUT:
1229 		if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1230 			cmd = cxgbit_allocate_cmd(csk);
1231 			if (!cmd)
1232 				goto reject;
1233 		}
1234 
1235 		ret = cxgbit_handle_nop_out(csk, cmd);
1236 		break;
1237 	case ISCSI_OP_SCSI_TMFUNC:
1238 		cmd = cxgbit_allocate_cmd(csk);
1239 		if (!cmd)
1240 			goto reject;
1241 
1242 		ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1243 						 (unsigned char *)hdr);
1244 		break;
1245 	case ISCSI_OP_TEXT:
1246 		if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1247 			cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1248 			if (!cmd)
1249 				goto reject;
1250 		} else {
1251 			cmd = cxgbit_allocate_cmd(csk);
1252 			if (!cmd)
1253 				goto reject;
1254 		}
1255 
1256 		ret = cxgbit_handle_text_cmd(csk, cmd);
1257 		break;
1258 	case ISCSI_OP_LOGOUT:
1259 		cmd = cxgbit_allocate_cmd(csk);
1260 		if (!cmd)
1261 			goto reject;
1262 
1263 		ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1264 		if (ret > 0)
1265 			wait_for_completion_timeout(&conn->conn_logout_comp,
1266 						    SECONDS_FOR_LOGOUT_COMP
1267 						    * HZ);
1268 		break;
1269 	case ISCSI_OP_SNACK:
1270 		ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1271 		break;
1272 	default:
1273 		pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1274 		dump_stack();
1275 		break;
1276 	}
1277 
1278 	return ret;
1279 
1280 reject:
1281 	return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1282 				 (unsigned char *)hdr);
1283 	return ret;
1284 }
1285 
1286 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1287 {
1288 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1289 	struct iscsi_conn *conn = csk->conn;
1290 	struct iscsi_hdr *hdr = pdu_cb->hdr;
1291 	u8 opcode;
1292 
1293 	if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1294 		atomic_long_inc(&conn->sess->conn_digest_errors);
1295 		goto transport_err;
1296 	}
1297 
1298 	if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1299 		goto transport_err;
1300 
1301 	opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1302 
1303 	if (conn->sess->sess_ops->SessionType &&
1304 	    ((!(opcode & ISCSI_OP_TEXT)) ||
1305 	     (!(opcode & ISCSI_OP_LOGOUT)))) {
1306 		pr_err("Received illegal iSCSI Opcode: 0x%02x"
1307 			" while in Discovery Session, rejecting.\n", opcode);
1308 		iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1309 				  (unsigned char *)hdr);
1310 		goto transport_err;
1311 	}
1312 
1313 	if (cxgbit_target_rx_opcode(csk) < 0)
1314 		goto transport_err;
1315 
1316 	return 0;
1317 
1318 transport_err:
1319 	return -1;
1320 }
1321 
1322 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1323 {
1324 	struct iscsi_conn *conn = csk->conn;
1325 	struct iscsi_login *login = conn->login;
1326 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1327 	struct iscsi_login_req *login_req;
1328 
1329 	login_req = (struct iscsi_login_req *)login->req;
1330 	memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1331 
1332 	pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1333 		" CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1334 		login_req->flags, login_req->itt, login_req->cmdsn,
1335 		login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1336 	/*
1337 	 * Setup the initial iscsi_login values from the leading
1338 	 * login request PDU.
1339 	 */
1340 	if (login->first_request) {
1341 		login_req = (struct iscsi_login_req *)login->req;
1342 		login->leading_connection = (!login_req->tsih) ? 1 : 0;
1343 		login->current_stage	= ISCSI_LOGIN_CURRENT_STAGE(
1344 				login_req->flags);
1345 		login->version_min	= login_req->min_version;
1346 		login->version_max	= login_req->max_version;
1347 		memcpy(login->isid, login_req->isid, 6);
1348 		login->cmd_sn		= be32_to_cpu(login_req->cmdsn);
1349 		login->init_task_tag	= login_req->itt;
1350 		login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1351 		login->cid		= be16_to_cpu(login_req->cid);
1352 		login->tsih		= be16_to_cpu(login_req->tsih);
1353 	}
1354 
1355 	if (iscsi_target_check_login_request(conn, login) < 0)
1356 		return -1;
1357 
1358 	memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1359 	skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1360 
1361 	return 0;
1362 }
1363 
1364 static int
1365 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1366 {
1367 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1368 	int ret;
1369 
1370 	cxgbit_rx_pdu_cb(skb) = pdu_cb;
1371 
1372 	csk->skb = skb;
1373 
1374 	if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1375 		ret = cxgbit_rx_login_pdu(csk);
1376 		set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1377 	} else {
1378 		ret = cxgbit_rx_opcode(csk);
1379 	}
1380 
1381 	return ret;
1382 }
1383 
1384 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1385 {
1386 	struct skb_shared_info *ssi = skb_shinfo(skb);
1387 	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1388 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1389 	u8 i;
1390 
1391 	pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1392 		skb, skb->head, skb->data, skb->len, skb->data_len,
1393 		ssi->nr_frags);
1394 	pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1395 		skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1396 
1397 	for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1398 		pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1399 			"frags %u.\n",
1400 			skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1401 			pdu_cb->ddigest, pdu_cb->frags);
1402 	for (i = 0; i < ssi->nr_frags; i++)
1403 		pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1404 			skb, i, skb_frag_off(&ssi->frags[i]),
1405 			skb_frag_size(&ssi->frags[i]));
1406 }
1407 
1408 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1409 {
1410 	struct sk_buff *skb = csk->lro_hskb;
1411 	struct skb_shared_info *ssi = skb_shinfo(skb);
1412 	u8 i;
1413 
1414 	memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1415 	for (i = 0; i < ssi->nr_frags; i++)
1416 		put_page(skb_frag_page(&ssi->frags[i]));
1417 	ssi->nr_frags = 0;
1418 	skb->data_len = 0;
1419 	skb->truesize -= skb->len;
1420 	skb->len = 0;
1421 }
1422 
1423 static void
1424 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1425 {
1426 	struct sk_buff *hskb = csk->lro_hskb;
1427 	struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1428 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1429 	struct skb_shared_info *hssi = skb_shinfo(hskb);
1430 	struct skb_shared_info *ssi = skb_shinfo(skb);
1431 	unsigned int len = 0;
1432 
1433 	if (pdu_cb->flags & PDUCBF_RX_HDR) {
1434 		u8 hfrag_idx = hssi->nr_frags;
1435 
1436 		hpdu_cb->flags |= pdu_cb->flags;
1437 		hpdu_cb->seq = pdu_cb->seq;
1438 		hpdu_cb->hdr = pdu_cb->hdr;
1439 		hpdu_cb->hlen = pdu_cb->hlen;
1440 
1441 		memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1442 		       sizeof(skb_frag_t));
1443 
1444 		get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1445 		hssi->nr_frags++;
1446 		hpdu_cb->frags++;
1447 		hpdu_cb->hfrag_idx = hfrag_idx;
1448 
1449 		len = skb_frag_size(&hssi->frags[hfrag_idx]);
1450 		hskb->len += len;
1451 		hskb->data_len += len;
1452 		hskb->truesize += len;
1453 	}
1454 
1455 	if (pdu_cb->flags & PDUCBF_RX_DATA) {
1456 		u8 dfrag_idx = hssi->nr_frags, i;
1457 
1458 		hpdu_cb->flags |= pdu_cb->flags;
1459 		hpdu_cb->dfrag_idx = dfrag_idx;
1460 
1461 		len = 0;
1462 		for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1463 			memcpy(&hssi->frags[dfrag_idx],
1464 			       &ssi->frags[pdu_cb->dfrag_idx + i],
1465 			       sizeof(skb_frag_t));
1466 
1467 			get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1468 
1469 			len += skb_frag_size(&hssi->frags[dfrag_idx]);
1470 
1471 			hssi->nr_frags++;
1472 			hpdu_cb->frags++;
1473 		}
1474 
1475 		hpdu_cb->dlen = pdu_cb->dlen;
1476 		hpdu_cb->doffset = hpdu_cb->hlen;
1477 		hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1478 		hskb->len += len;
1479 		hskb->data_len += len;
1480 		hskb->truesize += len;
1481 	}
1482 
1483 	if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1484 		hpdu_cb->flags |= pdu_cb->flags;
1485 
1486 		if (hpdu_cb->flags & PDUCBF_RX_DATA)
1487 			hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1488 
1489 		hpdu_cb->ddigest = pdu_cb->ddigest;
1490 		hpdu_cb->pdulen = pdu_cb->pdulen;
1491 	}
1492 }
1493 
1494 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1495 {
1496 	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1497 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1498 	u8 pdu_idx = 0, last_idx = 0;
1499 	int ret = 0;
1500 
1501 	if (!pdu_cb->complete) {
1502 		cxgbit_lro_skb_merge(csk, skb, 0);
1503 
1504 		if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1505 			struct sk_buff *hskb = csk->lro_hskb;
1506 
1507 			ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1508 
1509 			cxgbit_lro_hskb_reset(csk);
1510 
1511 			if (ret < 0)
1512 				goto out;
1513 		}
1514 
1515 		pdu_idx = 1;
1516 	}
1517 
1518 	if (lro_cb->pdu_idx)
1519 		last_idx = lro_cb->pdu_idx - 1;
1520 
1521 	for (; pdu_idx <= last_idx; pdu_idx++) {
1522 		ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1523 		if (ret < 0)
1524 			goto out;
1525 	}
1526 
1527 	if ((!lro_cb->complete) && lro_cb->pdu_idx)
1528 		cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1529 
1530 out:
1531 	return ret;
1532 }
1533 
1534 static int cxgbit_t5_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1535 {
1536 	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1537 	struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1538 	int ret = -1;
1539 
1540 	if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1541 	    (pdu_cb->seq != csk->rcv_nxt)) {
1542 		pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1543 			csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1544 		cxgbit_lro_skb_dump(skb);
1545 		return ret;
1546 	}
1547 
1548 	csk->rcv_nxt += lro_cb->pdu_totallen;
1549 
1550 	ret = cxgbit_process_lro_skb(csk, skb);
1551 
1552 	csk->rx_credits += lro_cb->pdu_totallen;
1553 
1554 	if (csk->rx_credits >= (csk->rcv_win / 4))
1555 		cxgbit_rx_data_ack(csk);
1556 
1557 	return ret;
1558 }
1559 
1560 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1561 {
1562 	struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1563 	int ret;
1564 
1565 	ret = cxgbit_process_lro_skb(csk, skb);
1566 	if (ret)
1567 		return ret;
1568 
1569 	csk->rx_credits += lro_cb->pdu_totallen;
1570 	if (csk->rx_credits >= csk->rcv_win) {
1571 		csk->rx_credits = 0;
1572 		cxgbit_rx_data_ack(csk);
1573 	}
1574 
1575 	return 0;
1576 }
1577 
1578 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1579 {
1580 	struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1581 	int ret = -1;
1582 
1583 	if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1584 		if (is_t5(lldi->adapter_type))
1585 			ret = cxgbit_t5_rx_lro_skb(csk, skb);
1586 		else
1587 			ret = cxgbit_rx_lro_skb(csk, skb);
1588 	}
1589 
1590 	__kfree_skb(skb);
1591 	return ret;
1592 }
1593 
1594 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1595 {
1596 	spin_lock_bh(&csk->rxq.lock);
1597 	if (skb_queue_len(&csk->rxq)) {
1598 		skb_queue_splice_init(&csk->rxq, rxq);
1599 		spin_unlock_bh(&csk->rxq.lock);
1600 		return true;
1601 	}
1602 	spin_unlock_bh(&csk->rxq.lock);
1603 	return false;
1604 }
1605 
1606 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1607 {
1608 	struct sk_buff *skb;
1609 	struct sk_buff_head rxq;
1610 
1611 	skb_queue_head_init(&rxq);
1612 
1613 	wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1614 
1615 	if (signal_pending(current))
1616 		goto out;
1617 
1618 	while ((skb = __skb_dequeue(&rxq))) {
1619 		if (cxgbit_rx_skb(csk, skb))
1620 			goto out;
1621 	}
1622 
1623 	return 0;
1624 out:
1625 	__skb_queue_purge(&rxq);
1626 	return -1;
1627 }
1628 
1629 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1630 {
1631 	struct cxgbit_sock *csk = conn->context;
1632 	int ret = -1;
1633 
1634 	while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1635 		ret = cxgbit_wait_rxq(csk);
1636 		if (ret) {
1637 			clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1638 			break;
1639 		}
1640 	}
1641 
1642 	return ret;
1643 }
1644 
1645 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1646 {
1647 	struct cxgbit_sock *csk = conn->context;
1648 
1649 	while (!kthread_should_stop()) {
1650 		iscsit_thread_check_cpumask(conn, current, 0);
1651 		if (cxgbit_wait_rxq(csk))
1652 			return;
1653 	}
1654 }
1655