xref: /openbmc/linux/drivers/scsi/libfc/fc_fcp.c (revision e2f1cf25)
1 /*
2  * Copyright(c) 2007 Intel Corporation. All rights reserved.
3  * Copyright(c) 2008 Red Hat, Inc.  All rights reserved.
4  * Copyright(c) 2008 Mike Christie
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms and conditions of the GNU General Public License,
8  * version 2, as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope it will be useful, but WITHOUT
11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13  * more details.
14  *
15  * You should have received a copy of the GNU General Public License along with
16  * this program; if not, write to the Free Software Foundation, Inc.,
17  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18  *
19  * Maintained at www.Open-FCoE.org
20  */
21 
22 #include <linux/module.h>
23 #include <linux/delay.h>
24 #include <linux/kernel.h>
25 #include <linux/types.h>
26 #include <linux/spinlock.h>
27 #include <linux/scatterlist.h>
28 #include <linux/err.h>
29 #include <linux/crc32.h>
30 #include <linux/slab.h>
31 
32 #include <scsi/scsi_tcq.h>
33 #include <scsi/scsi.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_device.h>
36 #include <scsi/scsi_cmnd.h>
37 
38 #include <scsi/fc/fc_fc2.h>
39 
40 #include <scsi/libfc.h>
41 #include <scsi/fc_encode.h>
42 
43 #include "fc_libfc.h"
44 
45 static struct kmem_cache *scsi_pkt_cachep;
46 
47 /* SRB state definitions */
48 #define FC_SRB_FREE		0		/* cmd is free */
49 #define FC_SRB_CMD_SENT		(1 << 0)	/* cmd has been sent */
50 #define FC_SRB_RCV_STATUS	(1 << 1)	/* response has arrived */
51 #define FC_SRB_ABORT_PENDING	(1 << 2)	/* cmd abort sent to device */
52 #define FC_SRB_ABORTED		(1 << 3)	/* abort acknowledged */
53 #define FC_SRB_DISCONTIG	(1 << 4)	/* non-sequential data recvd */
54 #define FC_SRB_COMPL		(1 << 5)	/* fc_io_compl has been run */
55 #define FC_SRB_FCP_PROCESSING_TMO (1 << 6)	/* timer function processing */
56 
57 #define FC_SRB_READ		(1 << 1)
58 #define FC_SRB_WRITE		(1 << 0)
59 
60 /*
61  * The SCp.ptr should be tested and set under the scsi_pkt_queue lock
62  */
63 #define CMD_SP(Cmnd)		    ((struct fc_fcp_pkt *)(Cmnd)->SCp.ptr)
64 #define CMD_ENTRY_STATUS(Cmnd)	    ((Cmnd)->SCp.have_data_in)
65 #define CMD_COMPL_STATUS(Cmnd)	    ((Cmnd)->SCp.this_residual)
66 #define CMD_SCSI_STATUS(Cmnd)	    ((Cmnd)->SCp.Status)
67 #define CMD_RESID_LEN(Cmnd)	    ((Cmnd)->SCp.buffers_residual)
68 
69 /**
70  * struct fc_fcp_internal - FCP layer internal data
71  * @scsi_pkt_pool: Memory pool to draw FCP packets from
72  * @scsi_queue_lock: Protects the scsi_pkt_queue
73  * @scsi_pkt_queue: Current FCP packets
74  * @last_can_queue_ramp_down_time: ramp down time
75  * @last_can_queue_ramp_up_time: ramp up time
76  * @max_can_queue: max can_queue size
77  */
78 struct fc_fcp_internal {
79 	mempool_t		*scsi_pkt_pool;
80 	spinlock_t		scsi_queue_lock;
81 	struct list_head	scsi_pkt_queue;
82 	unsigned long		last_can_queue_ramp_down_time;
83 	unsigned long		last_can_queue_ramp_up_time;
84 	int			max_can_queue;
85 };
86 
87 #define fc_get_scsi_internal(x)	((struct fc_fcp_internal *)(x)->scsi_priv)
88 
89 /*
90  * function prototypes
91  * FC scsi I/O related functions
92  */
93 static void fc_fcp_recv_data(struct fc_fcp_pkt *, struct fc_frame *);
94 static void fc_fcp_recv(struct fc_seq *, struct fc_frame *, void *);
95 static void fc_fcp_resp(struct fc_fcp_pkt *, struct fc_frame *);
96 static void fc_fcp_complete_locked(struct fc_fcp_pkt *);
97 static void fc_tm_done(struct fc_seq *, struct fc_frame *, void *);
98 static void fc_fcp_error(struct fc_fcp_pkt *, struct fc_frame *);
99 static void fc_fcp_recovery(struct fc_fcp_pkt *, u8 code);
100 static void fc_fcp_timeout(unsigned long);
101 static void fc_fcp_rec(struct fc_fcp_pkt *);
102 static void fc_fcp_rec_error(struct fc_fcp_pkt *, struct fc_frame *);
103 static void fc_fcp_rec_resp(struct fc_seq *, struct fc_frame *, void *);
104 static void fc_io_compl(struct fc_fcp_pkt *);
105 
106 static void fc_fcp_srr(struct fc_fcp_pkt *, enum fc_rctl, u32);
107 static void fc_fcp_srr_resp(struct fc_seq *, struct fc_frame *, void *);
108 static void fc_fcp_srr_error(struct fc_fcp_pkt *, struct fc_frame *);
109 
110 /*
111  * command status codes
112  */
113 #define FC_COMPLETE		0
114 #define FC_CMD_ABORTED		1
115 #define FC_CMD_RESET		2
116 #define FC_CMD_PLOGO		3
117 #define FC_SNS_RCV		4
118 #define FC_TRANS_ERR		5
119 #define FC_DATA_OVRRUN		6
120 #define FC_DATA_UNDRUN		7
121 #define FC_ERROR		8
122 #define FC_HRD_ERROR		9
123 #define FC_CRC_ERROR		10
124 #define FC_TIMED_OUT		11
125 
126 /*
127  * Error recovery timeout values.
128  */
129 #define FC_SCSI_TM_TOV		(10 * HZ)
130 #define FC_HOST_RESET_TIMEOUT	(30 * HZ)
131 #define FC_CAN_QUEUE_PERIOD	(60 * HZ)
132 
133 #define FC_MAX_ERROR_CNT	5
134 #define FC_MAX_RECOV_RETRY	3
135 
136 #define FC_FCP_DFLT_QUEUE_DEPTH 32
137 
138 /**
139  * fc_fcp_pkt_alloc() - Allocate a fcp_pkt
140  * @lport: The local port that the FCP packet is for
141  * @gfp:   GFP flags for allocation
142  *
143  * Return value: fcp_pkt structure or null on allocation failure.
144  * Context:	 Can be called from process context, no lock is required.
145  */
146 static struct fc_fcp_pkt *fc_fcp_pkt_alloc(struct fc_lport *lport, gfp_t gfp)
147 {
148 	struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
149 	struct fc_fcp_pkt *fsp;
150 
151 	fsp = mempool_alloc(si->scsi_pkt_pool, gfp);
152 	if (fsp) {
153 		memset(fsp, 0, sizeof(*fsp));
154 		fsp->lp = lport;
155 		fsp->xfer_ddp = FC_XID_UNKNOWN;
156 		atomic_set(&fsp->ref_cnt, 1);
157 		init_timer(&fsp->timer);
158 		fsp->timer.data = (unsigned long)fsp;
159 		INIT_LIST_HEAD(&fsp->list);
160 		spin_lock_init(&fsp->scsi_pkt_lock);
161 	} else {
162 		per_cpu_ptr(lport->stats, get_cpu())->FcpPktAllocFails++;
163 		put_cpu();
164 	}
165 	return fsp;
166 }
167 
168 /**
169  * fc_fcp_pkt_release() - Release hold on a fcp_pkt
170  * @fsp: The FCP packet to be released
171  *
172  * Context: Can be called from process or interrupt context,
173  *	    no lock is required.
174  */
175 static void fc_fcp_pkt_release(struct fc_fcp_pkt *fsp)
176 {
177 	if (atomic_dec_and_test(&fsp->ref_cnt)) {
178 		struct fc_fcp_internal *si = fc_get_scsi_internal(fsp->lp);
179 
180 		mempool_free(fsp, si->scsi_pkt_pool);
181 	}
182 }
183 
184 /**
185  * fc_fcp_pkt_hold() - Hold a fcp_pkt
186  * @fsp: The FCP packet to be held
187  */
188 static void fc_fcp_pkt_hold(struct fc_fcp_pkt *fsp)
189 {
190 	atomic_inc(&fsp->ref_cnt);
191 }
192 
193 /**
194  * fc_fcp_pkt_destory() - Release hold on a fcp_pkt
195  * @seq: The sequence that the FCP packet is on (required by destructor API)
196  * @fsp: The FCP packet to be released
197  *
198  * This routine is called by a destructor callback in the exch_seq_send()
199  * routine of the libfc Transport Template. The 'struct fc_seq' is a required
200  * argument even though it is not used by this routine.
201  *
202  * Context: No locking required.
203  */
204 static void fc_fcp_pkt_destroy(struct fc_seq *seq, void *fsp)
205 {
206 	fc_fcp_pkt_release(fsp);
207 }
208 
209 /**
210  * fc_fcp_lock_pkt() - Lock a fcp_pkt and increase its reference count
211  * @fsp: The FCP packet to be locked and incremented
212  *
213  * We should only return error if we return a command to SCSI-ml before
214  * getting a response. This can happen in cases where we send a abort, but
215  * do not wait for the response and the abort and command can be passing
216  * each other on the wire/network-layer.
217  *
218  * Note: this function locks the packet and gets a reference to allow
219  * callers to call the completion function while the lock is held and
220  * not have to worry about the packets refcount.
221  *
222  * TODO: Maybe we should just have callers grab/release the lock and
223  * have a function that they call to verify the fsp and grab a ref if
224  * needed.
225  */
226 static inline int fc_fcp_lock_pkt(struct fc_fcp_pkt *fsp)
227 {
228 	spin_lock_bh(&fsp->scsi_pkt_lock);
229 	if (fsp->state & FC_SRB_COMPL) {
230 		spin_unlock_bh(&fsp->scsi_pkt_lock);
231 		return -EPERM;
232 	}
233 
234 	fc_fcp_pkt_hold(fsp);
235 	return 0;
236 }
237 
238 /**
239  * fc_fcp_unlock_pkt() - Release a fcp_pkt's lock and decrement its
240  *			 reference count
241  * @fsp: The FCP packet to be unlocked and decremented
242  */
243 static inline void fc_fcp_unlock_pkt(struct fc_fcp_pkt *fsp)
244 {
245 	spin_unlock_bh(&fsp->scsi_pkt_lock);
246 	fc_fcp_pkt_release(fsp);
247 }
248 
249 /**
250  * fc_fcp_timer_set() - Start a timer for a fcp_pkt
251  * @fsp:   The FCP packet to start a timer for
252  * @delay: The timeout period in jiffies
253  */
254 static void fc_fcp_timer_set(struct fc_fcp_pkt *fsp, unsigned long delay)
255 {
256 	if (!(fsp->state & FC_SRB_COMPL))
257 		mod_timer(&fsp->timer, jiffies + delay);
258 }
259 
260 /**
261  * fc_fcp_send_abort() - Send an abort for exchanges associated with a
262  *			 fcp_pkt
263  * @fsp: The FCP packet to abort exchanges on
264  */
265 static int fc_fcp_send_abort(struct fc_fcp_pkt *fsp)
266 {
267 	if (!fsp->seq_ptr)
268 		return -EINVAL;
269 
270 	per_cpu_ptr(fsp->lp->stats, get_cpu())->FcpPktAborts++;
271 	put_cpu();
272 
273 	fsp->state |= FC_SRB_ABORT_PENDING;
274 	return fsp->lp->tt.seq_exch_abort(fsp->seq_ptr, 0);
275 }
276 
277 /**
278  * fc_fcp_retry_cmd() - Retry a fcp_pkt
279  * @fsp: The FCP packet to be retried
280  *
281  * Sets the status code to be FC_ERROR and then calls
282  * fc_fcp_complete_locked() which in turn calls fc_io_compl().
283  * fc_io_compl() will notify the SCSI-ml that the I/O is done.
284  * The SCSI-ml will retry the command.
285  */
286 static void fc_fcp_retry_cmd(struct fc_fcp_pkt *fsp)
287 {
288 	if (fsp->seq_ptr) {
289 		fsp->lp->tt.exch_done(fsp->seq_ptr);
290 		fsp->seq_ptr = NULL;
291 	}
292 
293 	fsp->state &= ~FC_SRB_ABORT_PENDING;
294 	fsp->io_status = 0;
295 	fsp->status_code = FC_ERROR;
296 	fc_fcp_complete_locked(fsp);
297 }
298 
299 /**
300  * fc_fcp_ddp_setup() - Calls a LLD's ddp_setup routine to set up DDP context
301  * @fsp: The FCP packet that will manage the DDP frames
302  * @xid: The XID that will be used for the DDP exchange
303  */
304 void fc_fcp_ddp_setup(struct fc_fcp_pkt *fsp, u16 xid)
305 {
306 	struct fc_lport *lport;
307 
308 	lport = fsp->lp;
309 	if ((fsp->req_flags & FC_SRB_READ) &&
310 	    (lport->lro_enabled) && (lport->tt.ddp_setup)) {
311 		if (lport->tt.ddp_setup(lport, xid, scsi_sglist(fsp->cmd),
312 					scsi_sg_count(fsp->cmd)))
313 			fsp->xfer_ddp = xid;
314 	}
315 }
316 
317 /**
318  * fc_fcp_ddp_done() - Calls a LLD's ddp_done routine to release any
319  *		       DDP related resources for a fcp_pkt
320  * @fsp: The FCP packet that DDP had been used on
321  */
322 void fc_fcp_ddp_done(struct fc_fcp_pkt *fsp)
323 {
324 	struct fc_lport *lport;
325 
326 	if (!fsp)
327 		return;
328 
329 	if (fsp->xfer_ddp == FC_XID_UNKNOWN)
330 		return;
331 
332 	lport = fsp->lp;
333 	if (lport->tt.ddp_done) {
334 		fsp->xfer_len = lport->tt.ddp_done(lport, fsp->xfer_ddp);
335 		fsp->xfer_ddp = FC_XID_UNKNOWN;
336 	}
337 }
338 
339 /**
340  * fc_fcp_can_queue_ramp_up() - increases can_queue
341  * @lport: lport to ramp up can_queue
342  */
343 static void fc_fcp_can_queue_ramp_up(struct fc_lport *lport)
344 {
345 	struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
346 	unsigned long flags;
347 	int can_queue;
348 
349 	spin_lock_irqsave(lport->host->host_lock, flags);
350 
351 	if (si->last_can_queue_ramp_up_time &&
352 	    (time_before(jiffies, si->last_can_queue_ramp_up_time +
353 			 FC_CAN_QUEUE_PERIOD)))
354 		goto unlock;
355 
356 	if (time_before(jiffies, si->last_can_queue_ramp_down_time +
357 			FC_CAN_QUEUE_PERIOD))
358 		goto unlock;
359 
360 	si->last_can_queue_ramp_up_time = jiffies;
361 
362 	can_queue = lport->host->can_queue << 1;
363 	if (can_queue >= si->max_can_queue) {
364 		can_queue = si->max_can_queue;
365 		si->last_can_queue_ramp_down_time = 0;
366 	}
367 	lport->host->can_queue = can_queue;
368 	shost_printk(KERN_ERR, lport->host, "libfc: increased "
369 		     "can_queue to %d.\n", can_queue);
370 
371 unlock:
372 	spin_unlock_irqrestore(lport->host->host_lock, flags);
373 }
374 
375 /**
376  * fc_fcp_can_queue_ramp_down() - reduces can_queue
377  * @lport: lport to reduce can_queue
378  *
379  * If we are getting memory allocation failures, then we may
380  * be trying to execute too many commands. We let the running
381  * commands complete or timeout, then try again with a reduced
382  * can_queue. Eventually we will hit the point where we run
383  * on all reserved structs.
384  */
385 static void fc_fcp_can_queue_ramp_down(struct fc_lport *lport)
386 {
387 	struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
388 	unsigned long flags;
389 	int can_queue;
390 
391 	spin_lock_irqsave(lport->host->host_lock, flags);
392 
393 	if (si->last_can_queue_ramp_down_time &&
394 	    (time_before(jiffies, si->last_can_queue_ramp_down_time +
395 			 FC_CAN_QUEUE_PERIOD)))
396 		goto unlock;
397 
398 	si->last_can_queue_ramp_down_time = jiffies;
399 
400 	can_queue = lport->host->can_queue;
401 	can_queue >>= 1;
402 	if (!can_queue)
403 		can_queue = 1;
404 	lport->host->can_queue = can_queue;
405 	shost_printk(KERN_ERR, lport->host, "libfc: Could not allocate frame.\n"
406 		     "Reducing can_queue to %d.\n", can_queue);
407 
408 unlock:
409 	spin_unlock_irqrestore(lport->host->host_lock, flags);
410 }
411 
412 /*
413  * fc_fcp_frame_alloc() -  Allocates fc_frame structure and buffer.
414  * @lport:	fc lport struct
415  * @len:	payload length
416  *
417  * Allocates fc_frame structure and buffer but if fails to allocate
418  * then reduce can_queue.
419  */
420 static inline struct fc_frame *fc_fcp_frame_alloc(struct fc_lport *lport,
421 						  size_t len)
422 {
423 	struct fc_frame *fp;
424 
425 	fp = fc_frame_alloc(lport, len);
426 	if (likely(fp))
427 		return fp;
428 
429 	per_cpu_ptr(lport->stats, get_cpu())->FcpFrameAllocFails++;
430 	put_cpu();
431 	/* error case */
432 	fc_fcp_can_queue_ramp_down(lport);
433 	return NULL;
434 }
435 
436 /**
437  * fc_fcp_recv_data() - Handler for receiving SCSI-FCP data from a target
438  * @fsp: The FCP packet the data is on
439  * @fp:	 The data frame
440  */
441 static void fc_fcp_recv_data(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
442 {
443 	struct scsi_cmnd *sc = fsp->cmd;
444 	struct fc_lport *lport = fsp->lp;
445 	struct fc_stats *stats;
446 	struct fc_frame_header *fh;
447 	size_t start_offset;
448 	size_t offset;
449 	u32 crc;
450 	u32 copy_len = 0;
451 	size_t len;
452 	void *buf;
453 	struct scatterlist *sg;
454 	u32 nents;
455 	u8 host_bcode = FC_COMPLETE;
456 
457 	fh = fc_frame_header_get(fp);
458 	offset = ntohl(fh->fh_parm_offset);
459 	start_offset = offset;
460 	len = fr_len(fp) - sizeof(*fh);
461 	buf = fc_frame_payload_get(fp, 0);
462 
463 	/*
464 	 * if this I/O is ddped then clear it and initiate recovery since data
465 	 * frames are expected to be placed directly in that case.
466 	 *
467 	 * Indicate error to scsi-ml because something went wrong with the
468 	 * ddp handling to get us here.
469 	 */
470 	if (fsp->xfer_ddp != FC_XID_UNKNOWN) {
471 		fc_fcp_ddp_done(fsp);
472 		FC_FCP_DBG(fsp, "DDP I/O in fc_fcp_recv_data set ERROR\n");
473 		host_bcode = FC_ERROR;
474 		goto err;
475 	}
476 	if (offset + len > fsp->data_len) {
477 		/* this should never happen */
478 		if ((fr_flags(fp) & FCPHF_CRC_UNCHECKED) &&
479 		    fc_frame_crc_check(fp))
480 			goto crc_err;
481 		FC_FCP_DBG(fsp, "data received past end. len %zx offset %zx "
482 			   "data_len %x\n", len, offset, fsp->data_len);
483 
484 		/* Data is corrupted indicate scsi-ml should retry */
485 		host_bcode = FC_DATA_OVRRUN;
486 		goto err;
487 	}
488 	if (offset != fsp->xfer_len)
489 		fsp->state |= FC_SRB_DISCONTIG;
490 
491 	sg = scsi_sglist(sc);
492 	nents = scsi_sg_count(sc);
493 
494 	if (!(fr_flags(fp) & FCPHF_CRC_UNCHECKED)) {
495 		copy_len = fc_copy_buffer_to_sglist(buf, len, sg, &nents,
496 						    &offset, NULL);
497 	} else {
498 		crc = crc32(~0, (u8 *) fh, sizeof(*fh));
499 		copy_len = fc_copy_buffer_to_sglist(buf, len, sg, &nents,
500 						    &offset, &crc);
501 		buf = fc_frame_payload_get(fp, 0);
502 		if (len % 4)
503 			crc = crc32(crc, buf + len, 4 - (len % 4));
504 
505 		if (~crc != le32_to_cpu(fr_crc(fp))) {
506 crc_err:
507 			stats = per_cpu_ptr(lport->stats, get_cpu());
508 			stats->ErrorFrames++;
509 			/* per cpu count, not total count, but OK for limit */
510 			if (stats->InvalidCRCCount++ < FC_MAX_ERROR_CNT)
511 				printk(KERN_WARNING "libfc: CRC error on data "
512 				       "frame for port (%6.6x)\n",
513 				       lport->port_id);
514 			put_cpu();
515 			/*
516 			 * Assume the frame is total garbage.
517 			 * We may have copied it over the good part
518 			 * of the buffer.
519 			 * If so, we need to retry the entire operation.
520 			 * Otherwise, ignore it.
521 			 */
522 			if (fsp->state & FC_SRB_DISCONTIG) {
523 				host_bcode = FC_CRC_ERROR;
524 				goto err;
525 			}
526 			return;
527 		}
528 	}
529 
530 	if (fsp->xfer_contig_end == start_offset)
531 		fsp->xfer_contig_end += copy_len;
532 	fsp->xfer_len += copy_len;
533 
534 	/*
535 	 * In the very rare event that this data arrived after the response
536 	 * and completes the transfer, call the completion handler.
537 	 */
538 	if (unlikely(fsp->state & FC_SRB_RCV_STATUS) &&
539 	    fsp->xfer_len == fsp->data_len - fsp->scsi_resid)
540 		fc_fcp_complete_locked(fsp);
541 	return;
542 err:
543 	fc_fcp_recovery(fsp, host_bcode);
544 }
545 
546 /**
547  * fc_fcp_send_data() - Send SCSI data to a target
548  * @fsp:      The FCP packet the data is on
549  * @sp:	      The sequence the data is to be sent on
550  * @offset:   The starting offset for this data request
551  * @seq_blen: The burst length for this data request
552  *
553  * Called after receiving a Transfer Ready data descriptor.
554  * If the LLD is capable of sequence offload then send down the
555  * seq_blen amount of data in single frame, otherwise send
556  * multiple frames of the maximum frame payload supported by
557  * the target port.
558  */
559 static int fc_fcp_send_data(struct fc_fcp_pkt *fsp, struct fc_seq *seq,
560 			    size_t offset, size_t seq_blen)
561 {
562 	struct fc_exch *ep;
563 	struct scsi_cmnd *sc;
564 	struct scatterlist *sg;
565 	struct fc_frame *fp = NULL;
566 	struct fc_lport *lport = fsp->lp;
567 	struct page *page;
568 	size_t remaining;
569 	size_t t_blen;
570 	size_t tlen;
571 	size_t sg_bytes;
572 	size_t frame_offset, fh_parm_offset;
573 	size_t off;
574 	int error;
575 	void *data = NULL;
576 	void *page_addr;
577 	int using_sg = lport->sg_supp;
578 	u32 f_ctl;
579 
580 	WARN_ON(seq_blen <= 0);
581 	if (unlikely(offset + seq_blen > fsp->data_len)) {
582 		/* this should never happen */
583 		FC_FCP_DBG(fsp, "xfer-ready past end. seq_blen %zx "
584 			   "offset %zx\n", seq_blen, offset);
585 		fc_fcp_send_abort(fsp);
586 		return 0;
587 	} else if (offset != fsp->xfer_len) {
588 		/* Out of Order Data Request - no problem, but unexpected. */
589 		FC_FCP_DBG(fsp, "xfer-ready non-contiguous. "
590 			   "seq_blen %zx offset %zx\n", seq_blen, offset);
591 	}
592 
593 	/*
594 	 * if LLD is capable of seq_offload then set transport
595 	 * burst length (t_blen) to seq_blen, otherwise set t_blen
596 	 * to max FC frame payload previously set in fsp->max_payload.
597 	 */
598 	t_blen = fsp->max_payload;
599 	if (lport->seq_offload) {
600 		t_blen = min(seq_blen, (size_t)lport->lso_max);
601 		FC_FCP_DBG(fsp, "fsp=%p:lso:blen=%zx lso_max=0x%x t_blen=%zx\n",
602 			   fsp, seq_blen, lport->lso_max, t_blen);
603 	}
604 
605 	if (t_blen > 512)
606 		t_blen &= ~(512 - 1);	/* round down to block size */
607 	sc = fsp->cmd;
608 
609 	remaining = seq_blen;
610 	fh_parm_offset = frame_offset = offset;
611 	tlen = 0;
612 	seq = lport->tt.seq_start_next(seq);
613 	f_ctl = FC_FC_REL_OFF;
614 	WARN_ON(!seq);
615 
616 	sg = scsi_sglist(sc);
617 
618 	while (remaining > 0 && sg) {
619 		if (offset >= sg->length) {
620 			offset -= sg->length;
621 			sg = sg_next(sg);
622 			continue;
623 		}
624 		if (!fp) {
625 			tlen = min(t_blen, remaining);
626 
627 			/*
628 			 * TODO.  Temporary workaround.	 fc_seq_send() can't
629 			 * handle odd lengths in non-linear skbs.
630 			 * This will be the final fragment only.
631 			 */
632 			if (tlen % 4)
633 				using_sg = 0;
634 			fp = fc_frame_alloc(lport, using_sg ? 0 : tlen);
635 			if (!fp)
636 				return -ENOMEM;
637 
638 			data = fc_frame_header_get(fp) + 1;
639 			fh_parm_offset = frame_offset;
640 			fr_max_payload(fp) = fsp->max_payload;
641 		}
642 
643 		off = offset + sg->offset;
644 		sg_bytes = min(tlen, sg->length - offset);
645 		sg_bytes = min(sg_bytes,
646 			       (size_t) (PAGE_SIZE - (off & ~PAGE_MASK)));
647 		page = sg_page(sg) + (off >> PAGE_SHIFT);
648 		if (using_sg) {
649 			get_page(page);
650 			skb_fill_page_desc(fp_skb(fp),
651 					   skb_shinfo(fp_skb(fp))->nr_frags,
652 					   page, off & ~PAGE_MASK, sg_bytes);
653 			fp_skb(fp)->data_len += sg_bytes;
654 			fr_len(fp) += sg_bytes;
655 			fp_skb(fp)->truesize += PAGE_SIZE;
656 		} else {
657 			/*
658 			 * The scatterlist item may be bigger than PAGE_SIZE,
659 			 * but we must not cross pages inside the kmap.
660 			 */
661 			page_addr = kmap_atomic(page);
662 			memcpy(data, (char *)page_addr + (off & ~PAGE_MASK),
663 			       sg_bytes);
664 			kunmap_atomic(page_addr);
665 			data += sg_bytes;
666 		}
667 		offset += sg_bytes;
668 		frame_offset += sg_bytes;
669 		tlen -= sg_bytes;
670 		remaining -= sg_bytes;
671 
672 		if ((skb_shinfo(fp_skb(fp))->nr_frags < FC_FRAME_SG_LEN) &&
673 		    (tlen))
674 			continue;
675 
676 		/*
677 		 * Send sequence with transfer sequence initiative in case
678 		 * this is last FCP frame of the sequence.
679 		 */
680 		if (remaining == 0)
681 			f_ctl |= FC_FC_SEQ_INIT | FC_FC_END_SEQ;
682 
683 		ep = fc_seq_exch(seq);
684 		fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid,
685 			       FC_TYPE_FCP, f_ctl, fh_parm_offset);
686 
687 		/*
688 		 * send fragment using for a sequence.
689 		 */
690 		error = lport->tt.seq_send(lport, seq, fp);
691 		if (error) {
692 			WARN_ON(1);		/* send error should be rare */
693 			return error;
694 		}
695 		fp = NULL;
696 	}
697 	fsp->xfer_len += seq_blen;	/* premature count? */
698 	return 0;
699 }
700 
701 /**
702  * fc_fcp_abts_resp() - Receive an ABTS response
703  * @fsp: The FCP packet that is being aborted
704  * @fp:	 The response frame
705  */
706 static void fc_fcp_abts_resp(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
707 {
708 	int ba_done = 1;
709 	struct fc_ba_rjt *brp;
710 	struct fc_frame_header *fh;
711 
712 	fh = fc_frame_header_get(fp);
713 	switch (fh->fh_r_ctl) {
714 	case FC_RCTL_BA_ACC:
715 		break;
716 	case FC_RCTL_BA_RJT:
717 		brp = fc_frame_payload_get(fp, sizeof(*brp));
718 		if (brp && brp->br_reason == FC_BA_RJT_LOG_ERR)
719 			break;
720 		/* fall thru */
721 	default:
722 		/*
723 		 * we will let the command timeout
724 		 * and scsi-ml recover in this case,
725 		 * therefore cleared the ba_done flag.
726 		 */
727 		ba_done = 0;
728 	}
729 
730 	if (ba_done) {
731 		fsp->state |= FC_SRB_ABORTED;
732 		fsp->state &= ~FC_SRB_ABORT_PENDING;
733 
734 		if (fsp->wait_for_comp)
735 			complete(&fsp->tm_done);
736 		else
737 			fc_fcp_complete_locked(fsp);
738 	}
739 }
740 
741 /**
742  * fc_fcp_recv() - Receive an FCP frame
743  * @seq: The sequence the frame is on
744  * @fp:	 The received frame
745  * @arg: The related FCP packet
746  *
747  * Context: Called from Soft IRQ context. Can not be called
748  *	    holding the FCP packet list lock.
749  */
750 static void fc_fcp_recv(struct fc_seq *seq, struct fc_frame *fp, void *arg)
751 {
752 	struct fc_fcp_pkt *fsp = (struct fc_fcp_pkt *)arg;
753 	struct fc_lport *lport = fsp->lp;
754 	struct fc_frame_header *fh;
755 	struct fcp_txrdy *dd;
756 	u8 r_ctl;
757 	int rc = 0;
758 
759 	if (IS_ERR(fp)) {
760 		fc_fcp_error(fsp, fp);
761 		return;
762 	}
763 
764 	fh = fc_frame_header_get(fp);
765 	r_ctl = fh->fh_r_ctl;
766 
767 	if (lport->state != LPORT_ST_READY)
768 		goto out;
769 	if (fc_fcp_lock_pkt(fsp))
770 		goto out;
771 
772 	if (fh->fh_type == FC_TYPE_BLS) {
773 		fc_fcp_abts_resp(fsp, fp);
774 		goto unlock;
775 	}
776 
777 	if (fsp->state & (FC_SRB_ABORTED | FC_SRB_ABORT_PENDING))
778 		goto unlock;
779 
780 	if (r_ctl == FC_RCTL_DD_DATA_DESC) {
781 		/*
782 		 * received XFER RDY from the target
783 		 * need to send data to the target
784 		 */
785 		WARN_ON(fr_flags(fp) & FCPHF_CRC_UNCHECKED);
786 		dd = fc_frame_payload_get(fp, sizeof(*dd));
787 		WARN_ON(!dd);
788 
789 		rc = fc_fcp_send_data(fsp, seq,
790 				      (size_t) ntohl(dd->ft_data_ro),
791 				      (size_t) ntohl(dd->ft_burst_len));
792 		if (!rc)
793 			seq->rec_data = fsp->xfer_len;
794 	} else if (r_ctl == FC_RCTL_DD_SOL_DATA) {
795 		/*
796 		 * received a DATA frame
797 		 * next we will copy the data to the system buffer
798 		 */
799 		WARN_ON(fr_len(fp) < sizeof(*fh));	/* len may be 0 */
800 		fc_fcp_recv_data(fsp, fp);
801 		seq->rec_data = fsp->xfer_contig_end;
802 	} else if (r_ctl == FC_RCTL_DD_CMD_STATUS) {
803 		WARN_ON(fr_flags(fp) & FCPHF_CRC_UNCHECKED);
804 
805 		fc_fcp_resp(fsp, fp);
806 	} else {
807 		FC_FCP_DBG(fsp, "unexpected frame.  r_ctl %x\n", r_ctl);
808 	}
809 unlock:
810 	fc_fcp_unlock_pkt(fsp);
811 out:
812 	fc_frame_free(fp);
813 }
814 
815 /**
816  * fc_fcp_resp() - Handler for FCP responses
817  * @fsp: The FCP packet the response is for
818  * @fp:	 The response frame
819  */
820 static void fc_fcp_resp(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
821 {
822 	struct fc_frame_header *fh;
823 	struct fcp_resp *fc_rp;
824 	struct fcp_resp_ext *rp_ex;
825 	struct fcp_resp_rsp_info *fc_rp_info;
826 	u32 plen;
827 	u32 expected_len;
828 	u32 respl = 0;
829 	u32 snsl = 0;
830 	u8 flags = 0;
831 
832 	plen = fr_len(fp);
833 	fh = (struct fc_frame_header *)fr_hdr(fp);
834 	if (unlikely(plen < sizeof(*fh) + sizeof(*fc_rp)))
835 		goto len_err;
836 	plen -= sizeof(*fh);
837 	fc_rp = (struct fcp_resp *)(fh + 1);
838 	fsp->cdb_status = fc_rp->fr_status;
839 	flags = fc_rp->fr_flags;
840 	fsp->scsi_comp_flags = flags;
841 	expected_len = fsp->data_len;
842 
843 	/* if ddp, update xfer len */
844 	fc_fcp_ddp_done(fsp);
845 
846 	if (unlikely((flags & ~FCP_CONF_REQ) || fc_rp->fr_status)) {
847 		rp_ex = (void *)(fc_rp + 1);
848 		if (flags & (FCP_RSP_LEN_VAL | FCP_SNS_LEN_VAL)) {
849 			if (plen < sizeof(*fc_rp) + sizeof(*rp_ex))
850 				goto len_err;
851 			fc_rp_info = (struct fcp_resp_rsp_info *)(rp_ex + 1);
852 			if (flags & FCP_RSP_LEN_VAL) {
853 				respl = ntohl(rp_ex->fr_rsp_len);
854 				if ((respl != FCP_RESP_RSP_INFO_LEN4) &&
855 				    (respl != FCP_RESP_RSP_INFO_LEN8))
856 					goto len_err;
857 				if (fsp->wait_for_comp) {
858 					/* Abuse cdb_status for rsp code */
859 					fsp->cdb_status = fc_rp_info->rsp_code;
860 					complete(&fsp->tm_done);
861 					/*
862 					 * tmfs will not have any scsi cmd so
863 					 * exit here
864 					 */
865 					return;
866 				}
867 			}
868 			if (flags & FCP_SNS_LEN_VAL) {
869 				snsl = ntohl(rp_ex->fr_sns_len);
870 				if (snsl > SCSI_SENSE_BUFFERSIZE)
871 					snsl = SCSI_SENSE_BUFFERSIZE;
872 				memcpy(fsp->cmd->sense_buffer,
873 				       (char *)fc_rp_info + respl, snsl);
874 			}
875 		}
876 		if (flags & (FCP_RESID_UNDER | FCP_RESID_OVER)) {
877 			if (plen < sizeof(*fc_rp) + sizeof(rp_ex->fr_resid))
878 				goto len_err;
879 			if (flags & FCP_RESID_UNDER) {
880 				fsp->scsi_resid = ntohl(rp_ex->fr_resid);
881 				/*
882 				 * The cmnd->underflow is the minimum number of
883 				 * bytes that must be transferred for this
884 				 * command.  Provided a sense condition is not
885 				 * present, make sure the actual amount
886 				 * transferred is at least the underflow value
887 				 * or fail.
888 				 */
889 				if (!(flags & FCP_SNS_LEN_VAL) &&
890 				    (fc_rp->fr_status == 0) &&
891 				    (scsi_bufflen(fsp->cmd) -
892 				     fsp->scsi_resid) < fsp->cmd->underflow)
893 					goto err;
894 				expected_len -= fsp->scsi_resid;
895 			} else {
896 				fsp->status_code = FC_ERROR;
897 			}
898 		}
899 	}
900 	fsp->state |= FC_SRB_RCV_STATUS;
901 
902 	/*
903 	 * Check for missing or extra data frames.
904 	 */
905 	if (unlikely(fsp->cdb_status == SAM_STAT_GOOD &&
906 		     fsp->xfer_len != expected_len)) {
907 		if (fsp->xfer_len < expected_len) {
908 			/*
909 			 * Some data may be queued locally,
910 			 * Wait a at least one jiffy to see if it is delivered.
911 			 * If this expires without data, we may do SRR.
912 			 */
913 			fc_fcp_timer_set(fsp, 2);
914 			return;
915 		}
916 		fsp->status_code = FC_DATA_OVRRUN;
917 		FC_FCP_DBG(fsp, "tgt %6.6x xfer len %zx greater than expected, "
918 			   "len %x, data len %x\n",
919 			   fsp->rport->port_id,
920 			   fsp->xfer_len, expected_len, fsp->data_len);
921 	}
922 	fc_fcp_complete_locked(fsp);
923 	return;
924 
925 len_err:
926 	FC_FCP_DBG(fsp, "short FCP response. flags 0x%x len %u respl %u "
927 		   "snsl %u\n", flags, fr_len(fp), respl, snsl);
928 err:
929 	fsp->status_code = FC_ERROR;
930 	fc_fcp_complete_locked(fsp);
931 }
932 
933 /**
934  * fc_fcp_complete_locked() - Complete processing of a fcp_pkt with the
935  *			      fcp_pkt lock held
936  * @fsp: The FCP packet to be completed
937  *
938  * This function may sleep if a timer is pending. The packet lock must be
939  * held, and the host lock must not be held.
940  */
941 static void fc_fcp_complete_locked(struct fc_fcp_pkt *fsp)
942 {
943 	struct fc_lport *lport = fsp->lp;
944 	struct fc_seq *seq;
945 	struct fc_exch *ep;
946 	u32 f_ctl;
947 
948 	if (fsp->state & FC_SRB_ABORT_PENDING)
949 		return;
950 
951 	if (fsp->state & FC_SRB_ABORTED) {
952 		if (!fsp->status_code)
953 			fsp->status_code = FC_CMD_ABORTED;
954 	} else {
955 		/*
956 		 * Test for transport underrun, independent of response
957 		 * underrun status.
958 		 */
959 		if (fsp->cdb_status == SAM_STAT_GOOD &&
960 		    fsp->xfer_len < fsp->data_len && !fsp->io_status &&
961 		    (!(fsp->scsi_comp_flags & FCP_RESID_UNDER) ||
962 		     fsp->xfer_len < fsp->data_len - fsp->scsi_resid))
963 			fsp->status_code = FC_DATA_UNDRUN;
964 	}
965 
966 	seq = fsp->seq_ptr;
967 	if (seq) {
968 		fsp->seq_ptr = NULL;
969 		if (unlikely(fsp->scsi_comp_flags & FCP_CONF_REQ)) {
970 			struct fc_frame *conf_frame;
971 			struct fc_seq *csp;
972 
973 			csp = lport->tt.seq_start_next(seq);
974 			conf_frame = fc_fcp_frame_alloc(fsp->lp, 0);
975 			if (conf_frame) {
976 				f_ctl = FC_FC_SEQ_INIT;
977 				f_ctl |= FC_FC_LAST_SEQ | FC_FC_END_SEQ;
978 				ep = fc_seq_exch(seq);
979 				fc_fill_fc_hdr(conf_frame, FC_RCTL_DD_SOL_CTL,
980 					       ep->did, ep->sid,
981 					       FC_TYPE_FCP, f_ctl, 0);
982 				lport->tt.seq_send(lport, csp, conf_frame);
983 			}
984 		}
985 		lport->tt.exch_done(seq);
986 	}
987 	/*
988 	 * Some resets driven by SCSI are not I/Os and do not have
989 	 * SCSI commands associated with the requests. We should not
990 	 * call I/O completion if we do not have a SCSI command.
991 	 */
992 	if (fsp->cmd)
993 		fc_io_compl(fsp);
994 }
995 
996 /**
997  * fc_fcp_cleanup_cmd() - Cancel the active exchange on a fcp_pkt
998  * @fsp:   The FCP packet whose exchanges should be canceled
999  * @error: The reason for the cancellation
1000  */
1001 static void fc_fcp_cleanup_cmd(struct fc_fcp_pkt *fsp, int error)
1002 {
1003 	struct fc_lport *lport = fsp->lp;
1004 
1005 	if (fsp->seq_ptr) {
1006 		lport->tt.exch_done(fsp->seq_ptr);
1007 		fsp->seq_ptr = NULL;
1008 	}
1009 	fsp->status_code = error;
1010 }
1011 
1012 /**
1013  * fc_fcp_cleanup_each_cmd() - Cancel all exchanges on a local port
1014  * @lport: The local port whose exchanges should be canceled
1015  * @id:	   The target's ID
1016  * @lun:   The LUN
1017  * @error: The reason for cancellation
1018  *
1019  * If lun or id is -1, they are ignored.
1020  */
1021 static void fc_fcp_cleanup_each_cmd(struct fc_lport *lport, unsigned int id,
1022 				    unsigned int lun, int error)
1023 {
1024 	struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
1025 	struct fc_fcp_pkt *fsp;
1026 	struct scsi_cmnd *sc_cmd;
1027 	unsigned long flags;
1028 
1029 	spin_lock_irqsave(&si->scsi_queue_lock, flags);
1030 restart:
1031 	list_for_each_entry(fsp, &si->scsi_pkt_queue, list) {
1032 		sc_cmd = fsp->cmd;
1033 		if (id != -1 && scmd_id(sc_cmd) != id)
1034 			continue;
1035 
1036 		if (lun != -1 && sc_cmd->device->lun != lun)
1037 			continue;
1038 
1039 		fc_fcp_pkt_hold(fsp);
1040 		spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
1041 
1042 		spin_lock_bh(&fsp->scsi_pkt_lock);
1043 		if (!(fsp->state & FC_SRB_COMPL)) {
1044 			fsp->state |= FC_SRB_COMPL;
1045 			/*
1046 			 * TODO: dropping scsi_pkt_lock and then reacquiring
1047 			 * again around fc_fcp_cleanup_cmd() is required,
1048 			 * since fc_fcp_cleanup_cmd() calls into
1049 			 * fc_seq_set_resp() and that func preempts cpu using
1050 			 * schedule. May be schedule and related code should be
1051 			 * removed instead of unlocking here to avoid scheduling
1052 			 * while atomic bug.
1053 			 */
1054 			spin_unlock_bh(&fsp->scsi_pkt_lock);
1055 
1056 			fc_fcp_cleanup_cmd(fsp, error);
1057 
1058 			spin_lock_bh(&fsp->scsi_pkt_lock);
1059 			fc_io_compl(fsp);
1060 		}
1061 		spin_unlock_bh(&fsp->scsi_pkt_lock);
1062 
1063 		fc_fcp_pkt_release(fsp);
1064 		spin_lock_irqsave(&si->scsi_queue_lock, flags);
1065 		/*
1066 		 * while we dropped the lock multiple pkts could
1067 		 * have been released, so we have to start over.
1068 		 */
1069 		goto restart;
1070 	}
1071 	spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
1072 }
1073 
1074 /**
1075  * fc_fcp_abort_io() - Abort all FCP-SCSI exchanges on a local port
1076  * @lport: The local port whose exchanges are to be aborted
1077  */
1078 static void fc_fcp_abort_io(struct fc_lport *lport)
1079 {
1080 	fc_fcp_cleanup_each_cmd(lport, -1, -1, FC_HRD_ERROR);
1081 }
1082 
1083 /**
1084  * fc_fcp_pkt_send() - Send a fcp_pkt
1085  * @lport: The local port to send the FCP packet on
1086  * @fsp:   The FCP packet to send
1087  *
1088  * Return:  Zero for success and -1 for failure
1089  * Locks:   Called without locks held
1090  */
1091 static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp)
1092 {
1093 	struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
1094 	unsigned long flags;
1095 	int rc;
1096 
1097 	fsp->cmd->SCp.ptr = (char *)fsp;
1098 	fsp->cdb_cmd.fc_dl = htonl(fsp->data_len);
1099 	fsp->cdb_cmd.fc_flags = fsp->req_flags & ~FCP_CFL_LEN_MASK;
1100 
1101 	int_to_scsilun(fsp->cmd->device->lun, &fsp->cdb_cmd.fc_lun);
1102 	memcpy(fsp->cdb_cmd.fc_cdb, fsp->cmd->cmnd, fsp->cmd->cmd_len);
1103 
1104 	spin_lock_irqsave(&si->scsi_queue_lock, flags);
1105 	list_add_tail(&fsp->list, &si->scsi_pkt_queue);
1106 	spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
1107 	rc = lport->tt.fcp_cmd_send(lport, fsp, fc_fcp_recv);
1108 	if (unlikely(rc)) {
1109 		spin_lock_irqsave(&si->scsi_queue_lock, flags);
1110 		fsp->cmd->SCp.ptr = NULL;
1111 		list_del(&fsp->list);
1112 		spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
1113 	}
1114 
1115 	return rc;
1116 }
1117 
1118 /**
1119  * get_fsp_rec_tov() - Helper function to get REC_TOV
1120  * @fsp: the FCP packet
1121  *
1122  * Returns rec tov in jiffies as rpriv->e_d_tov + 1 second
1123  */
1124 static inline unsigned int get_fsp_rec_tov(struct fc_fcp_pkt *fsp)
1125 {
1126 	struct fc_rport_libfc_priv *rpriv = fsp->rport->dd_data;
1127 
1128 	return msecs_to_jiffies(rpriv->e_d_tov) + HZ;
1129 }
1130 
1131 /**
1132  * fc_fcp_cmd_send() - Send a FCP command
1133  * @lport: The local port to send the command on
1134  * @fsp:   The FCP packet the command is on
1135  * @resp:  The handler for the response
1136  */
1137 static int fc_fcp_cmd_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp,
1138 			   void (*resp)(struct fc_seq *,
1139 					struct fc_frame *fp,
1140 					void *arg))
1141 {
1142 	struct fc_frame *fp;
1143 	struct fc_seq *seq;
1144 	struct fc_rport *rport;
1145 	struct fc_rport_libfc_priv *rpriv;
1146 	const size_t len = sizeof(fsp->cdb_cmd);
1147 	int rc = 0;
1148 
1149 	if (fc_fcp_lock_pkt(fsp))
1150 		return 0;
1151 
1152 	fp = fc_fcp_frame_alloc(lport, sizeof(fsp->cdb_cmd));
1153 	if (!fp) {
1154 		rc = -1;
1155 		goto unlock;
1156 	}
1157 
1158 	memcpy(fc_frame_payload_get(fp, len), &fsp->cdb_cmd, len);
1159 	fr_fsp(fp) = fsp;
1160 	rport = fsp->rport;
1161 	fsp->max_payload = rport->maxframe_size;
1162 	rpriv = rport->dd_data;
1163 
1164 	fc_fill_fc_hdr(fp, FC_RCTL_DD_UNSOL_CMD, rport->port_id,
1165 		       rpriv->local_port->port_id, FC_TYPE_FCP,
1166 		       FC_FCTL_REQ, 0);
1167 
1168 	seq = lport->tt.exch_seq_send(lport, fp, resp, fc_fcp_pkt_destroy,
1169 				      fsp, 0);
1170 	if (!seq) {
1171 		rc = -1;
1172 		goto unlock;
1173 	}
1174 	fsp->seq_ptr = seq;
1175 	fc_fcp_pkt_hold(fsp);	/* hold for fc_fcp_pkt_destroy */
1176 
1177 	setup_timer(&fsp->timer, fc_fcp_timeout, (unsigned long)fsp);
1178 	if (rpriv->flags & FC_RP_FLAGS_REC_SUPPORTED)
1179 		fc_fcp_timer_set(fsp, get_fsp_rec_tov(fsp));
1180 
1181 unlock:
1182 	fc_fcp_unlock_pkt(fsp);
1183 	return rc;
1184 }
1185 
1186 /**
1187  * fc_fcp_error() - Handler for FCP layer errors
1188  * @fsp: The FCP packet the error is on
1189  * @fp:	 The frame that has errored
1190  */
1191 static void fc_fcp_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
1192 {
1193 	int error = PTR_ERR(fp);
1194 
1195 	if (fc_fcp_lock_pkt(fsp))
1196 		return;
1197 
1198 	if (error == -FC_EX_CLOSED) {
1199 		fc_fcp_retry_cmd(fsp);
1200 		goto unlock;
1201 	}
1202 
1203 	/*
1204 	 * clear abort pending, because the lower layer
1205 	 * decided to force completion.
1206 	 */
1207 	fsp->state &= ~FC_SRB_ABORT_PENDING;
1208 	fsp->status_code = FC_CMD_PLOGO;
1209 	fc_fcp_complete_locked(fsp);
1210 unlock:
1211 	fc_fcp_unlock_pkt(fsp);
1212 }
1213 
1214 /**
1215  * fc_fcp_pkt_abort() - Abort a fcp_pkt
1216  * @fsp:   The FCP packet to abort on
1217  *
1218  * Called to send an abort and then wait for abort completion
1219  */
1220 static int fc_fcp_pkt_abort(struct fc_fcp_pkt *fsp)
1221 {
1222 	int rc = FAILED;
1223 	unsigned long ticks_left;
1224 
1225 	if (fc_fcp_send_abort(fsp))
1226 		return FAILED;
1227 
1228 	init_completion(&fsp->tm_done);
1229 	fsp->wait_for_comp = 1;
1230 
1231 	spin_unlock_bh(&fsp->scsi_pkt_lock);
1232 	ticks_left = wait_for_completion_timeout(&fsp->tm_done,
1233 							FC_SCSI_TM_TOV);
1234 	spin_lock_bh(&fsp->scsi_pkt_lock);
1235 	fsp->wait_for_comp = 0;
1236 
1237 	if (!ticks_left) {
1238 		FC_FCP_DBG(fsp, "target abort cmd  failed\n");
1239 	} else if (fsp->state & FC_SRB_ABORTED) {
1240 		FC_FCP_DBG(fsp, "target abort cmd  passed\n");
1241 		rc = SUCCESS;
1242 		fc_fcp_complete_locked(fsp);
1243 	}
1244 
1245 	return rc;
1246 }
1247 
1248 /**
1249  * fc_lun_reset_send() - Send LUN reset command
1250  * @data: The FCP packet that identifies the LUN to be reset
1251  */
1252 static void fc_lun_reset_send(unsigned long data)
1253 {
1254 	struct fc_fcp_pkt *fsp = (struct fc_fcp_pkt *)data;
1255 	struct fc_lport *lport = fsp->lp;
1256 
1257 	if (lport->tt.fcp_cmd_send(lport, fsp, fc_tm_done)) {
1258 		if (fsp->recov_retry++ >= FC_MAX_RECOV_RETRY)
1259 			return;
1260 		if (fc_fcp_lock_pkt(fsp))
1261 			return;
1262 		setup_timer(&fsp->timer, fc_lun_reset_send, (unsigned long)fsp);
1263 		fc_fcp_timer_set(fsp, get_fsp_rec_tov(fsp));
1264 		fc_fcp_unlock_pkt(fsp);
1265 	}
1266 }
1267 
1268 /**
1269  * fc_lun_reset() - Send a LUN RESET command to a device
1270  *		    and wait for the reply
1271  * @lport: The local port to sent the command on
1272  * @fsp:   The FCP packet that identifies the LUN to be reset
1273  * @id:	   The SCSI command ID
1274  * @lun:   The LUN ID to be reset
1275  */
1276 static int fc_lun_reset(struct fc_lport *lport, struct fc_fcp_pkt *fsp,
1277 			unsigned int id, unsigned int lun)
1278 {
1279 	int rc;
1280 
1281 	fsp->cdb_cmd.fc_dl = htonl(fsp->data_len);
1282 	fsp->cdb_cmd.fc_tm_flags = FCP_TMF_LUN_RESET;
1283 	int_to_scsilun(lun, &fsp->cdb_cmd.fc_lun);
1284 
1285 	fsp->wait_for_comp = 1;
1286 	init_completion(&fsp->tm_done);
1287 
1288 	fc_lun_reset_send((unsigned long)fsp);
1289 
1290 	/*
1291 	 * wait for completion of reset
1292 	 * after that make sure all commands are terminated
1293 	 */
1294 	rc = wait_for_completion_timeout(&fsp->tm_done, FC_SCSI_TM_TOV);
1295 
1296 	spin_lock_bh(&fsp->scsi_pkt_lock);
1297 	fsp->state |= FC_SRB_COMPL;
1298 	spin_unlock_bh(&fsp->scsi_pkt_lock);
1299 
1300 	del_timer_sync(&fsp->timer);
1301 
1302 	spin_lock_bh(&fsp->scsi_pkt_lock);
1303 	if (fsp->seq_ptr) {
1304 		lport->tt.exch_done(fsp->seq_ptr);
1305 		fsp->seq_ptr = NULL;
1306 	}
1307 	fsp->wait_for_comp = 0;
1308 	spin_unlock_bh(&fsp->scsi_pkt_lock);
1309 
1310 	if (!rc) {
1311 		FC_SCSI_DBG(lport, "lun reset failed\n");
1312 		return FAILED;
1313 	}
1314 
1315 	/* cdb_status holds the tmf's rsp code */
1316 	if (fsp->cdb_status != FCP_TMF_CMPL)
1317 		return FAILED;
1318 
1319 	FC_SCSI_DBG(lport, "lun reset to lun %u completed\n", lun);
1320 	fc_fcp_cleanup_each_cmd(lport, id, lun, FC_CMD_ABORTED);
1321 	return SUCCESS;
1322 }
1323 
1324 /**
1325  * fc_tm_done() - Task Management response handler
1326  * @seq: The sequence that the response is on
1327  * @fp:	 The response frame
1328  * @arg: The FCP packet the response is for
1329  */
1330 static void fc_tm_done(struct fc_seq *seq, struct fc_frame *fp, void *arg)
1331 {
1332 	struct fc_fcp_pkt *fsp = arg;
1333 	struct fc_frame_header *fh;
1334 
1335 	if (IS_ERR(fp)) {
1336 		/*
1337 		 * If there is an error just let it timeout or wait
1338 		 * for TMF to be aborted if it timedout.
1339 		 *
1340 		 * scsi-eh will escalate for when either happens.
1341 		 */
1342 		return;
1343 	}
1344 
1345 	if (fc_fcp_lock_pkt(fsp))
1346 		goto out;
1347 
1348 	/*
1349 	 * raced with eh timeout handler.
1350 	 */
1351 	if (!fsp->seq_ptr || !fsp->wait_for_comp)
1352 		goto out_unlock;
1353 
1354 	fh = fc_frame_header_get(fp);
1355 	if (fh->fh_type != FC_TYPE_BLS)
1356 		fc_fcp_resp(fsp, fp);
1357 	fsp->seq_ptr = NULL;
1358 	fsp->lp->tt.exch_done(seq);
1359 out_unlock:
1360 	fc_fcp_unlock_pkt(fsp);
1361 out:
1362 	fc_frame_free(fp);
1363 }
1364 
1365 /**
1366  * fc_fcp_cleanup() - Cleanup all FCP exchanges on a local port
1367  * @lport: The local port to be cleaned up
1368  */
1369 static void fc_fcp_cleanup(struct fc_lport *lport)
1370 {
1371 	fc_fcp_cleanup_each_cmd(lport, -1, -1, FC_ERROR);
1372 }
1373 
1374 /**
1375  * fc_fcp_timeout() - Handler for fcp_pkt timeouts
1376  * @data: The FCP packet that has timed out
1377  *
1378  * If REC is supported then just issue it and return. The REC exchange will
1379  * complete or time out and recovery can continue at that point. Otherwise,
1380  * if the response has been received without all the data it has been
1381  * ER_TIMEOUT since the response was received. If the response has not been
1382  * received we see if data was received recently. If it has been then we
1383  * continue waiting, otherwise, we abort the command.
1384  */
1385 static void fc_fcp_timeout(unsigned long data)
1386 {
1387 	struct fc_fcp_pkt *fsp = (struct fc_fcp_pkt *)data;
1388 	struct fc_rport *rport = fsp->rport;
1389 	struct fc_rport_libfc_priv *rpriv = rport->dd_data;
1390 
1391 	if (fc_fcp_lock_pkt(fsp))
1392 		return;
1393 
1394 	if (fsp->cdb_cmd.fc_tm_flags)
1395 		goto unlock;
1396 
1397 	fsp->state |= FC_SRB_FCP_PROCESSING_TMO;
1398 
1399 	if (rpriv->flags & FC_RP_FLAGS_REC_SUPPORTED)
1400 		fc_fcp_rec(fsp);
1401 	else if (fsp->state & FC_SRB_RCV_STATUS)
1402 		fc_fcp_complete_locked(fsp);
1403 	else
1404 		fc_fcp_recovery(fsp, FC_TIMED_OUT);
1405 	fsp->state &= ~FC_SRB_FCP_PROCESSING_TMO;
1406 unlock:
1407 	fc_fcp_unlock_pkt(fsp);
1408 }
1409 
1410 /**
1411  * fc_fcp_rec() - Send a REC ELS request
1412  * @fsp: The FCP packet to send the REC request on
1413  */
1414 static void fc_fcp_rec(struct fc_fcp_pkt *fsp)
1415 {
1416 	struct fc_lport *lport;
1417 	struct fc_frame *fp;
1418 	struct fc_rport *rport;
1419 	struct fc_rport_libfc_priv *rpriv;
1420 
1421 	lport = fsp->lp;
1422 	rport = fsp->rport;
1423 	rpriv = rport->dd_data;
1424 	if (!fsp->seq_ptr || rpriv->rp_state != RPORT_ST_READY) {
1425 		fsp->status_code = FC_HRD_ERROR;
1426 		fsp->io_status = 0;
1427 		fc_fcp_complete_locked(fsp);
1428 		return;
1429 	}
1430 
1431 	fp = fc_fcp_frame_alloc(lport, sizeof(struct fc_els_rec));
1432 	if (!fp)
1433 		goto retry;
1434 
1435 	fr_seq(fp) = fsp->seq_ptr;
1436 	fc_fill_fc_hdr(fp, FC_RCTL_ELS_REQ, rport->port_id,
1437 		       rpriv->local_port->port_id, FC_TYPE_ELS,
1438 		       FC_FCTL_REQ, 0);
1439 	if (lport->tt.elsct_send(lport, rport->port_id, fp, ELS_REC,
1440 				 fc_fcp_rec_resp, fsp,
1441 				 2 * lport->r_a_tov)) {
1442 		fc_fcp_pkt_hold(fsp);		/* hold while REC outstanding */
1443 		return;
1444 	}
1445 retry:
1446 	if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY)
1447 		fc_fcp_timer_set(fsp, get_fsp_rec_tov(fsp));
1448 	else
1449 		fc_fcp_recovery(fsp, FC_TIMED_OUT);
1450 }
1451 
1452 /**
1453  * fc_fcp_rec_resp() - Handler for REC ELS responses
1454  * @seq: The sequence the response is on
1455  * @fp:	 The response frame
1456  * @arg: The FCP packet the response is on
1457  *
1458  * If the response is a reject then the scsi layer will handle
1459  * the timeout. If the response is a LS_ACC then if the I/O was not completed
1460  * set the timeout and return. If the I/O was completed then complete the
1461  * exchange and tell the SCSI layer.
1462  */
1463 static void fc_fcp_rec_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg)
1464 {
1465 	struct fc_fcp_pkt *fsp = (struct fc_fcp_pkt *)arg;
1466 	struct fc_els_rec_acc *recp;
1467 	struct fc_els_ls_rjt *rjt;
1468 	u32 e_stat;
1469 	u8 opcode;
1470 	u32 offset;
1471 	enum dma_data_direction data_dir;
1472 	enum fc_rctl r_ctl;
1473 	struct fc_rport_libfc_priv *rpriv;
1474 
1475 	if (IS_ERR(fp)) {
1476 		fc_fcp_rec_error(fsp, fp);
1477 		return;
1478 	}
1479 
1480 	if (fc_fcp_lock_pkt(fsp))
1481 		goto out;
1482 
1483 	fsp->recov_retry = 0;
1484 	opcode = fc_frame_payload_op(fp);
1485 	if (opcode == ELS_LS_RJT) {
1486 		rjt = fc_frame_payload_get(fp, sizeof(*rjt));
1487 		switch (rjt->er_reason) {
1488 		default:
1489 			FC_FCP_DBG(fsp, "device %x unexpected REC reject "
1490 				   "reason %d expl %d\n",
1491 				   fsp->rport->port_id, rjt->er_reason,
1492 				   rjt->er_explan);
1493 			/* fall through */
1494 		case ELS_RJT_UNSUP:
1495 			FC_FCP_DBG(fsp, "device does not support REC\n");
1496 			rpriv = fsp->rport->dd_data;
1497 			/*
1498 			 * if we do not spport RECs or got some bogus
1499 			 * reason then resetup timer so we check for
1500 			 * making progress.
1501 			 */
1502 			rpriv->flags &= ~FC_RP_FLAGS_REC_SUPPORTED;
1503 			break;
1504 		case ELS_RJT_LOGIC:
1505 		case ELS_RJT_UNAB:
1506 			/*
1507 			 * If no data transfer, the command frame got dropped
1508 			 * so we just retry.  If data was transferred, we
1509 			 * lost the response but the target has no record,
1510 			 * so we abort and retry.
1511 			 */
1512 			if (rjt->er_explan == ELS_EXPL_OXID_RXID &&
1513 			    fsp->xfer_len == 0) {
1514 				fc_fcp_retry_cmd(fsp);
1515 				break;
1516 			}
1517 			fc_fcp_recovery(fsp, FC_ERROR);
1518 			break;
1519 		}
1520 	} else if (opcode == ELS_LS_ACC) {
1521 		if (fsp->state & FC_SRB_ABORTED)
1522 			goto unlock_out;
1523 
1524 		data_dir = fsp->cmd->sc_data_direction;
1525 		recp = fc_frame_payload_get(fp, sizeof(*recp));
1526 		offset = ntohl(recp->reca_fc4value);
1527 		e_stat = ntohl(recp->reca_e_stat);
1528 
1529 		if (e_stat & ESB_ST_COMPLETE) {
1530 
1531 			/*
1532 			 * The exchange is complete.
1533 			 *
1534 			 * For output, we must've lost the response.
1535 			 * For input, all data must've been sent.
1536 			 * We lost may have lost the response
1537 			 * (and a confirmation was requested) and maybe
1538 			 * some data.
1539 			 *
1540 			 * If all data received, send SRR
1541 			 * asking for response.	 If partial data received,
1542 			 * or gaps, SRR requests data at start of gap.
1543 			 * Recovery via SRR relies on in-order-delivery.
1544 			 */
1545 			if (data_dir == DMA_TO_DEVICE) {
1546 				r_ctl = FC_RCTL_DD_CMD_STATUS;
1547 			} else if (fsp->xfer_contig_end == offset) {
1548 				r_ctl = FC_RCTL_DD_CMD_STATUS;
1549 			} else {
1550 				offset = fsp->xfer_contig_end;
1551 				r_ctl = FC_RCTL_DD_SOL_DATA;
1552 			}
1553 			fc_fcp_srr(fsp, r_ctl, offset);
1554 		} else if (e_stat & ESB_ST_SEQ_INIT) {
1555 			/*
1556 			 * The remote port has the initiative, so just
1557 			 * keep waiting for it to complete.
1558 			 */
1559 			fc_fcp_timer_set(fsp,  get_fsp_rec_tov(fsp));
1560 		} else {
1561 
1562 			/*
1563 			 * The exchange is incomplete, we have seq. initiative.
1564 			 * Lost response with requested confirmation,
1565 			 * lost confirmation, lost transfer ready or
1566 			 * lost write data.
1567 			 *
1568 			 * For output, if not all data was received, ask
1569 			 * for transfer ready to be repeated.
1570 			 *
1571 			 * If we received or sent all the data, send SRR to
1572 			 * request response.
1573 			 *
1574 			 * If we lost a response, we may have lost some read
1575 			 * data as well.
1576 			 */
1577 			r_ctl = FC_RCTL_DD_SOL_DATA;
1578 			if (data_dir == DMA_TO_DEVICE) {
1579 				r_ctl = FC_RCTL_DD_CMD_STATUS;
1580 				if (offset < fsp->data_len)
1581 					r_ctl = FC_RCTL_DD_DATA_DESC;
1582 			} else if (offset == fsp->xfer_contig_end) {
1583 				r_ctl = FC_RCTL_DD_CMD_STATUS;
1584 			} else if (fsp->xfer_contig_end < offset) {
1585 				offset = fsp->xfer_contig_end;
1586 			}
1587 			fc_fcp_srr(fsp, r_ctl, offset);
1588 		}
1589 	}
1590 unlock_out:
1591 	fc_fcp_unlock_pkt(fsp);
1592 out:
1593 	fc_fcp_pkt_release(fsp);	/* drop hold for outstanding REC */
1594 	fc_frame_free(fp);
1595 }
1596 
1597 /**
1598  * fc_fcp_rec_error() - Handler for REC errors
1599  * @fsp: The FCP packet the error is on
1600  * @fp:	 The REC frame
1601  */
1602 static void fc_fcp_rec_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
1603 {
1604 	int error = PTR_ERR(fp);
1605 
1606 	if (fc_fcp_lock_pkt(fsp))
1607 		goto out;
1608 
1609 	switch (error) {
1610 	case -FC_EX_CLOSED:
1611 		fc_fcp_retry_cmd(fsp);
1612 		break;
1613 
1614 	default:
1615 		FC_FCP_DBG(fsp, "REC %p fid %6.6x error unexpected error %d\n",
1616 			   fsp, fsp->rport->port_id, error);
1617 		fsp->status_code = FC_CMD_PLOGO;
1618 		/* fall through */
1619 
1620 	case -FC_EX_TIMEOUT:
1621 		/*
1622 		 * Assume REC or LS_ACC was lost.
1623 		 * The exchange manager will have aborted REC, so retry.
1624 		 */
1625 		FC_FCP_DBG(fsp, "REC fid %6.6x error error %d retry %d/%d\n",
1626 			   fsp->rport->port_id, error, fsp->recov_retry,
1627 			   FC_MAX_RECOV_RETRY);
1628 		if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY)
1629 			fc_fcp_rec(fsp);
1630 		else
1631 			fc_fcp_recovery(fsp, FC_ERROR);
1632 		break;
1633 	}
1634 	fc_fcp_unlock_pkt(fsp);
1635 out:
1636 	fc_fcp_pkt_release(fsp);	/* drop hold for outstanding REC */
1637 }
1638 
1639 /**
1640  * fc_fcp_recovery() - Handler for fcp_pkt recovery
1641  * @fsp: The FCP pkt that needs to be aborted
1642  */
1643 static void fc_fcp_recovery(struct fc_fcp_pkt *fsp, u8 code)
1644 {
1645 	fsp->status_code = code;
1646 	fsp->cdb_status = 0;
1647 	fsp->io_status = 0;
1648 	/*
1649 	 * if this fails then we let the scsi command timer fire and
1650 	 * scsi-ml escalate.
1651 	 */
1652 	fc_fcp_send_abort(fsp);
1653 }
1654 
1655 /**
1656  * fc_fcp_srr() - Send a SRR request (Sequence Retransmission Request)
1657  * @fsp:   The FCP packet the SRR is to be sent on
1658  * @r_ctl: The R_CTL field for the SRR request
1659  * This is called after receiving status but insufficient data, or
1660  * when expecting status but the request has timed out.
1661  */
1662 static void fc_fcp_srr(struct fc_fcp_pkt *fsp, enum fc_rctl r_ctl, u32 offset)
1663 {
1664 	struct fc_lport *lport = fsp->lp;
1665 	struct fc_rport *rport;
1666 	struct fc_rport_libfc_priv *rpriv;
1667 	struct fc_exch *ep = fc_seq_exch(fsp->seq_ptr);
1668 	struct fc_seq *seq;
1669 	struct fcp_srr *srr;
1670 	struct fc_frame *fp;
1671 	unsigned int rec_tov;
1672 
1673 	rport = fsp->rport;
1674 	rpriv = rport->dd_data;
1675 
1676 	if (!(rpriv->flags & FC_RP_FLAGS_RETRY) ||
1677 	    rpriv->rp_state != RPORT_ST_READY)
1678 		goto retry;			/* shouldn't happen */
1679 	fp = fc_fcp_frame_alloc(lport, sizeof(*srr));
1680 	if (!fp)
1681 		goto retry;
1682 
1683 	srr = fc_frame_payload_get(fp, sizeof(*srr));
1684 	memset(srr, 0, sizeof(*srr));
1685 	srr->srr_op = ELS_SRR;
1686 	srr->srr_ox_id = htons(ep->oxid);
1687 	srr->srr_rx_id = htons(ep->rxid);
1688 	srr->srr_r_ctl = r_ctl;
1689 	srr->srr_rel_off = htonl(offset);
1690 
1691 	fc_fill_fc_hdr(fp, FC_RCTL_ELS4_REQ, rport->port_id,
1692 		       rpriv->local_port->port_id, FC_TYPE_FCP,
1693 		       FC_FCTL_REQ, 0);
1694 
1695 	rec_tov = get_fsp_rec_tov(fsp);
1696 	seq = lport->tt.exch_seq_send(lport, fp, fc_fcp_srr_resp,
1697 				      fc_fcp_pkt_destroy,
1698 				      fsp, jiffies_to_msecs(rec_tov));
1699 	if (!seq)
1700 		goto retry;
1701 
1702 	fsp->recov_seq = seq;
1703 	fsp->xfer_len = offset;
1704 	fsp->xfer_contig_end = offset;
1705 	fsp->state &= ~FC_SRB_RCV_STATUS;
1706 	fc_fcp_pkt_hold(fsp);		/* hold for outstanding SRR */
1707 	return;
1708 retry:
1709 	fc_fcp_retry_cmd(fsp);
1710 }
1711 
1712 /**
1713  * fc_fcp_srr_resp() - Handler for SRR response
1714  * @seq: The sequence the SRR is on
1715  * @fp:	 The SRR frame
1716  * @arg: The FCP packet the SRR is on
1717  */
1718 static void fc_fcp_srr_resp(struct fc_seq *seq, struct fc_frame *fp, void *arg)
1719 {
1720 	struct fc_fcp_pkt *fsp = arg;
1721 	struct fc_frame_header *fh;
1722 
1723 	if (IS_ERR(fp)) {
1724 		fc_fcp_srr_error(fsp, fp);
1725 		return;
1726 	}
1727 
1728 	if (fc_fcp_lock_pkt(fsp))
1729 		goto out;
1730 
1731 	fh = fc_frame_header_get(fp);
1732 	/*
1733 	 * BUG? fc_fcp_srr_error calls exch_done which would release
1734 	 * the ep. But if fc_fcp_srr_error had got -FC_EX_TIMEOUT,
1735 	 * then fc_exch_timeout would be sending an abort. The exch_done
1736 	 * call by fc_fcp_srr_error would prevent fc_exch.c from seeing
1737 	 * an abort response though.
1738 	 */
1739 	if (fh->fh_type == FC_TYPE_BLS) {
1740 		fc_fcp_unlock_pkt(fsp);
1741 		return;
1742 	}
1743 
1744 	switch (fc_frame_payload_op(fp)) {
1745 	case ELS_LS_ACC:
1746 		fsp->recov_retry = 0;
1747 		fc_fcp_timer_set(fsp, get_fsp_rec_tov(fsp));
1748 		break;
1749 	case ELS_LS_RJT:
1750 	default:
1751 		fc_fcp_recovery(fsp, FC_ERROR);
1752 		break;
1753 	}
1754 	fc_fcp_unlock_pkt(fsp);
1755 out:
1756 	fsp->lp->tt.exch_done(seq);
1757 	fc_frame_free(fp);
1758 }
1759 
1760 /**
1761  * fc_fcp_srr_error() - Handler for SRR errors
1762  * @fsp: The FCP packet that the SRR error is on
1763  * @fp:	 The SRR frame
1764  */
1765 static void fc_fcp_srr_error(struct fc_fcp_pkt *fsp, struct fc_frame *fp)
1766 {
1767 	if (fc_fcp_lock_pkt(fsp))
1768 		goto out;
1769 	switch (PTR_ERR(fp)) {
1770 	case -FC_EX_TIMEOUT:
1771 		if (fsp->recov_retry++ < FC_MAX_RECOV_RETRY)
1772 			fc_fcp_rec(fsp);
1773 		else
1774 			fc_fcp_recovery(fsp, FC_TIMED_OUT);
1775 		break;
1776 	case -FC_EX_CLOSED:			/* e.g., link failure */
1777 		/* fall through */
1778 	default:
1779 		fc_fcp_retry_cmd(fsp);
1780 		break;
1781 	}
1782 	fc_fcp_unlock_pkt(fsp);
1783 out:
1784 	fsp->lp->tt.exch_done(fsp->recov_seq);
1785 }
1786 
1787 /**
1788  * fc_fcp_lport_queue_ready() - Determine if the lport and it's queue is ready
1789  * @lport: The local port to be checked
1790  */
1791 static inline int fc_fcp_lport_queue_ready(struct fc_lport *lport)
1792 {
1793 	/* lock ? */
1794 	return (lport->state == LPORT_ST_READY) &&
1795 		lport->link_up && !lport->qfull;
1796 }
1797 
1798 /**
1799  * fc_queuecommand() - The queuecommand function of the SCSI template
1800  * @shost: The Scsi_Host that the command was issued to
1801  * @cmd:   The scsi_cmnd to be executed
1802  *
1803  * This is the i/o strategy routine, called by the SCSI layer.
1804  */
1805 int fc_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *sc_cmd)
1806 {
1807 	struct fc_lport *lport = shost_priv(shost);
1808 	struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
1809 	struct fc_fcp_pkt *fsp;
1810 	struct fc_rport_libfc_priv *rpriv;
1811 	int rval;
1812 	int rc = 0;
1813 	struct fc_stats *stats;
1814 
1815 	rval = fc_remote_port_chkready(rport);
1816 	if (rval) {
1817 		sc_cmd->result = rval;
1818 		sc_cmd->scsi_done(sc_cmd);
1819 		return 0;
1820 	}
1821 
1822 	if (!*(struct fc_remote_port **)rport->dd_data) {
1823 		/*
1824 		 * rport is transitioning from blocked/deleted to
1825 		 * online
1826 		 */
1827 		sc_cmd->result = DID_IMM_RETRY << 16;
1828 		sc_cmd->scsi_done(sc_cmd);
1829 		goto out;
1830 	}
1831 
1832 	rpriv = rport->dd_data;
1833 
1834 	if (!fc_fcp_lport_queue_ready(lport)) {
1835 		if (lport->qfull)
1836 			fc_fcp_can_queue_ramp_down(lport);
1837 		rc = SCSI_MLQUEUE_HOST_BUSY;
1838 		goto out;
1839 	}
1840 
1841 	fsp = fc_fcp_pkt_alloc(lport, GFP_ATOMIC);
1842 	if (fsp == NULL) {
1843 		rc = SCSI_MLQUEUE_HOST_BUSY;
1844 		goto out;
1845 	}
1846 
1847 	/*
1848 	 * build the libfc request pkt
1849 	 */
1850 	fsp->cmd = sc_cmd;	/* save the cmd */
1851 	fsp->rport = rport;	/* set the remote port ptr */
1852 
1853 	/*
1854 	 * set up the transfer length
1855 	 */
1856 	fsp->data_len = scsi_bufflen(sc_cmd);
1857 	fsp->xfer_len = 0;
1858 
1859 	/*
1860 	 * setup the data direction
1861 	 */
1862 	stats = per_cpu_ptr(lport->stats, get_cpu());
1863 	if (sc_cmd->sc_data_direction == DMA_FROM_DEVICE) {
1864 		fsp->req_flags = FC_SRB_READ;
1865 		stats->InputRequests++;
1866 		stats->InputBytes += fsp->data_len;
1867 	} else if (sc_cmd->sc_data_direction == DMA_TO_DEVICE) {
1868 		fsp->req_flags = FC_SRB_WRITE;
1869 		stats->OutputRequests++;
1870 		stats->OutputBytes += fsp->data_len;
1871 	} else {
1872 		fsp->req_flags = 0;
1873 		stats->ControlRequests++;
1874 	}
1875 	put_cpu();
1876 
1877 	/*
1878 	 * send it to the lower layer
1879 	 * if we get -1 return then put the request in the pending
1880 	 * queue.
1881 	 */
1882 	rval = fc_fcp_pkt_send(lport, fsp);
1883 	if (rval != 0) {
1884 		fsp->state = FC_SRB_FREE;
1885 		fc_fcp_pkt_release(fsp);
1886 		rc = SCSI_MLQUEUE_HOST_BUSY;
1887 	}
1888 out:
1889 	return rc;
1890 }
1891 EXPORT_SYMBOL(fc_queuecommand);
1892 
1893 /**
1894  * fc_io_compl() - Handle responses for completed commands
1895  * @fsp: The FCP packet that is complete
1896  *
1897  * Translates fcp_pkt errors to a Linux SCSI errors.
1898  * The fcp packet lock must be held when calling.
1899  */
1900 static void fc_io_compl(struct fc_fcp_pkt *fsp)
1901 {
1902 	struct fc_fcp_internal *si;
1903 	struct scsi_cmnd *sc_cmd;
1904 	struct fc_lport *lport;
1905 	unsigned long flags;
1906 
1907 	/* release outstanding ddp context */
1908 	fc_fcp_ddp_done(fsp);
1909 
1910 	fsp->state |= FC_SRB_COMPL;
1911 	if (!(fsp->state & FC_SRB_FCP_PROCESSING_TMO)) {
1912 		spin_unlock_bh(&fsp->scsi_pkt_lock);
1913 		del_timer_sync(&fsp->timer);
1914 		spin_lock_bh(&fsp->scsi_pkt_lock);
1915 	}
1916 
1917 	lport = fsp->lp;
1918 	si = fc_get_scsi_internal(lport);
1919 
1920 	/*
1921 	 * if can_queue ramp down is done then try can_queue ramp up
1922 	 * since commands are completing now.
1923 	 */
1924 	if (si->last_can_queue_ramp_down_time)
1925 		fc_fcp_can_queue_ramp_up(lport);
1926 
1927 	sc_cmd = fsp->cmd;
1928 	CMD_SCSI_STATUS(sc_cmd) = fsp->cdb_status;
1929 	switch (fsp->status_code) {
1930 	case FC_COMPLETE:
1931 		if (fsp->cdb_status == 0) {
1932 			/*
1933 			 * good I/O status
1934 			 */
1935 			sc_cmd->result = DID_OK << 16;
1936 			if (fsp->scsi_resid)
1937 				CMD_RESID_LEN(sc_cmd) = fsp->scsi_resid;
1938 		} else {
1939 			/*
1940 			 * transport level I/O was ok but scsi
1941 			 * has non zero status
1942 			 */
1943 			sc_cmd->result = (DID_OK << 16) | fsp->cdb_status;
1944 		}
1945 		break;
1946 	case FC_ERROR:
1947 		FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
1948 			   "due to FC_ERROR\n");
1949 		sc_cmd->result = DID_ERROR << 16;
1950 		break;
1951 	case FC_DATA_UNDRUN:
1952 		if ((fsp->cdb_status == 0) && !(fsp->req_flags & FC_SRB_READ)) {
1953 			/*
1954 			 * scsi status is good but transport level
1955 			 * underrun.
1956 			 */
1957 			if (fsp->state & FC_SRB_RCV_STATUS) {
1958 				sc_cmd->result = DID_OK << 16;
1959 			} else {
1960 				FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml"
1961 					   " due to FC_DATA_UNDRUN (trans)\n");
1962 				sc_cmd->result = DID_ERROR << 16;
1963 			}
1964 		} else {
1965 			/*
1966 			 * scsi got underrun, this is an error
1967 			 */
1968 			FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
1969 				   "due to FC_DATA_UNDRUN (scsi)\n");
1970 			CMD_RESID_LEN(sc_cmd) = fsp->scsi_resid;
1971 			sc_cmd->result = (DID_ERROR << 16) | fsp->cdb_status;
1972 		}
1973 		break;
1974 	case FC_DATA_OVRRUN:
1975 		/*
1976 		 * overrun is an error
1977 		 */
1978 		FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
1979 			   "due to FC_DATA_OVRRUN\n");
1980 		sc_cmd->result = (DID_ERROR << 16) | fsp->cdb_status;
1981 		break;
1982 	case FC_CMD_ABORTED:
1983 		FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
1984 			  "due to FC_CMD_ABORTED\n");
1985 		sc_cmd->result = (DID_ERROR << 16) | fsp->io_status;
1986 		break;
1987 	case FC_CMD_RESET:
1988 		FC_FCP_DBG(fsp, "Returning DID_RESET to scsi-ml "
1989 			   "due to FC_CMD_RESET\n");
1990 		sc_cmd->result = (DID_RESET << 16);
1991 		break;
1992 	case FC_HRD_ERROR:
1993 		FC_FCP_DBG(fsp, "Returning DID_NO_CONNECT to scsi-ml "
1994 			   "due to FC_HRD_ERROR\n");
1995 		sc_cmd->result = (DID_NO_CONNECT << 16);
1996 		break;
1997 	case FC_CRC_ERROR:
1998 		FC_FCP_DBG(fsp, "Returning DID_PARITY to scsi-ml "
1999 			   "due to FC_CRC_ERROR\n");
2000 		sc_cmd->result = (DID_PARITY << 16);
2001 		break;
2002 	case FC_TIMED_OUT:
2003 		FC_FCP_DBG(fsp, "Returning DID_BUS_BUSY to scsi-ml "
2004 			   "due to FC_TIMED_OUT\n");
2005 		sc_cmd->result = (DID_BUS_BUSY << 16) | fsp->io_status;
2006 		break;
2007 	default:
2008 		FC_FCP_DBG(fsp, "Returning DID_ERROR to scsi-ml "
2009 			   "due to unknown error\n");
2010 		sc_cmd->result = (DID_ERROR << 16);
2011 		break;
2012 	}
2013 
2014 	if (lport->state != LPORT_ST_READY && fsp->status_code != FC_COMPLETE)
2015 		sc_cmd->result = (DID_TRANSPORT_DISRUPTED << 16);
2016 
2017 	spin_lock_irqsave(&si->scsi_queue_lock, flags);
2018 	list_del(&fsp->list);
2019 	sc_cmd->SCp.ptr = NULL;
2020 	spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
2021 	sc_cmd->scsi_done(sc_cmd);
2022 
2023 	/* release ref from initial allocation in queue command */
2024 	fc_fcp_pkt_release(fsp);
2025 }
2026 
2027 /**
2028  * fc_eh_abort() - Abort a command
2029  * @sc_cmd: The SCSI command to abort
2030  *
2031  * From SCSI host template.
2032  * Send an ABTS to the target device and wait for the response.
2033  */
2034 int fc_eh_abort(struct scsi_cmnd *sc_cmd)
2035 {
2036 	struct fc_fcp_pkt *fsp;
2037 	struct fc_lport *lport;
2038 	struct fc_fcp_internal *si;
2039 	int rc = FAILED;
2040 	unsigned long flags;
2041 	int rval;
2042 
2043 	rval = fc_block_scsi_eh(sc_cmd);
2044 	if (rval)
2045 		return rval;
2046 
2047 	lport = shost_priv(sc_cmd->device->host);
2048 	if (lport->state != LPORT_ST_READY)
2049 		return rc;
2050 	else if (!lport->link_up)
2051 		return rc;
2052 
2053 	si = fc_get_scsi_internal(lport);
2054 	spin_lock_irqsave(&si->scsi_queue_lock, flags);
2055 	fsp = CMD_SP(sc_cmd);
2056 	if (!fsp) {
2057 		/* command completed while scsi eh was setting up */
2058 		spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
2059 		return SUCCESS;
2060 	}
2061 	/* grab a ref so the fsp and sc_cmd cannot be released from under us */
2062 	fc_fcp_pkt_hold(fsp);
2063 	spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
2064 
2065 	if (fc_fcp_lock_pkt(fsp)) {
2066 		/* completed while we were waiting for timer to be deleted */
2067 		rc = SUCCESS;
2068 		goto release_pkt;
2069 	}
2070 
2071 	rc = fc_fcp_pkt_abort(fsp);
2072 	fc_fcp_unlock_pkt(fsp);
2073 
2074 release_pkt:
2075 	fc_fcp_pkt_release(fsp);
2076 	return rc;
2077 }
2078 EXPORT_SYMBOL(fc_eh_abort);
2079 
2080 /**
2081  * fc_eh_device_reset() - Reset a single LUN
2082  * @sc_cmd: The SCSI command which identifies the device whose
2083  *	    LUN is to be reset
2084  *
2085  * Set from SCSI host template.
2086  */
2087 int fc_eh_device_reset(struct scsi_cmnd *sc_cmd)
2088 {
2089 	struct fc_lport *lport;
2090 	struct fc_fcp_pkt *fsp;
2091 	struct fc_rport *rport = starget_to_rport(scsi_target(sc_cmd->device));
2092 	int rc = FAILED;
2093 	int rval;
2094 
2095 	rval = fc_block_scsi_eh(sc_cmd);
2096 	if (rval)
2097 		return rval;
2098 
2099 	lport = shost_priv(sc_cmd->device->host);
2100 
2101 	if (lport->state != LPORT_ST_READY)
2102 		return rc;
2103 
2104 	FC_SCSI_DBG(lport, "Resetting rport (%6.6x)\n", rport->port_id);
2105 
2106 	fsp = fc_fcp_pkt_alloc(lport, GFP_NOIO);
2107 	if (fsp == NULL) {
2108 		printk(KERN_WARNING "libfc: could not allocate scsi_pkt\n");
2109 		goto out;
2110 	}
2111 
2112 	/*
2113 	 * Build the libfc request pkt. Do not set the scsi cmnd, because
2114 	 * the sc passed in is not setup for execution like when sent
2115 	 * through the queuecommand callout.
2116 	 */
2117 	fsp->rport = rport;	/* set the remote port ptr */
2118 
2119 	/*
2120 	 * flush outstanding commands
2121 	 */
2122 	rc = fc_lun_reset(lport, fsp, scmd_id(sc_cmd), sc_cmd->device->lun);
2123 	fsp->state = FC_SRB_FREE;
2124 	fc_fcp_pkt_release(fsp);
2125 
2126 out:
2127 	return rc;
2128 }
2129 EXPORT_SYMBOL(fc_eh_device_reset);
2130 
2131 /**
2132  * fc_eh_host_reset() - Reset a Scsi_Host.
2133  * @sc_cmd: The SCSI command that identifies the SCSI host to be reset
2134  */
2135 int fc_eh_host_reset(struct scsi_cmnd *sc_cmd)
2136 {
2137 	struct Scsi_Host *shost = sc_cmd->device->host;
2138 	struct fc_lport *lport = shost_priv(shost);
2139 	unsigned long wait_tmo;
2140 
2141 	FC_SCSI_DBG(lport, "Resetting host\n");
2142 
2143 	fc_block_scsi_eh(sc_cmd);
2144 
2145 	lport->tt.lport_reset(lport);
2146 	wait_tmo = jiffies + FC_HOST_RESET_TIMEOUT;
2147 	while (!fc_fcp_lport_queue_ready(lport) && time_before(jiffies,
2148 							       wait_tmo))
2149 		msleep(1000);
2150 
2151 	if (fc_fcp_lport_queue_ready(lport)) {
2152 		shost_printk(KERN_INFO, shost, "libfc: Host reset succeeded "
2153 			     "on port (%6.6x)\n", lport->port_id);
2154 		return SUCCESS;
2155 	} else {
2156 		shost_printk(KERN_INFO, shost, "libfc: Host reset failed, "
2157 			     "port (%6.6x) is not ready.\n",
2158 			     lport->port_id);
2159 		return FAILED;
2160 	}
2161 }
2162 EXPORT_SYMBOL(fc_eh_host_reset);
2163 
2164 /**
2165  * fc_slave_alloc() - Configure the queue depth of a Scsi_Host
2166  * @sdev: The SCSI device that identifies the SCSI host
2167  *
2168  * Configures queue depth based on host's cmd_per_len. If not set
2169  * then we use the libfc default.
2170  */
2171 int fc_slave_alloc(struct scsi_device *sdev)
2172 {
2173 	struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
2174 
2175 	if (!rport || fc_remote_port_chkready(rport))
2176 		return -ENXIO;
2177 
2178 	scsi_change_queue_depth(sdev, FC_FCP_DFLT_QUEUE_DEPTH);
2179 	return 0;
2180 }
2181 EXPORT_SYMBOL(fc_slave_alloc);
2182 
2183 /**
2184  * fc_fcp_destory() - Tear down the FCP layer for a given local port
2185  * @lport: The local port that no longer needs the FCP layer
2186  */
2187 void fc_fcp_destroy(struct fc_lport *lport)
2188 {
2189 	struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
2190 
2191 	if (!list_empty(&si->scsi_pkt_queue))
2192 		printk(KERN_ERR "libfc: Leaked SCSI packets when destroying "
2193 		       "port (%6.6x)\n", lport->port_id);
2194 
2195 	mempool_destroy(si->scsi_pkt_pool);
2196 	kfree(si);
2197 	lport->scsi_priv = NULL;
2198 }
2199 EXPORT_SYMBOL(fc_fcp_destroy);
2200 
2201 int fc_setup_fcp(void)
2202 {
2203 	int rc = 0;
2204 
2205 	scsi_pkt_cachep = kmem_cache_create("libfc_fcp_pkt",
2206 					    sizeof(struct fc_fcp_pkt),
2207 					    0, SLAB_HWCACHE_ALIGN, NULL);
2208 	if (!scsi_pkt_cachep) {
2209 		printk(KERN_ERR "libfc: Unable to allocate SRB cache, "
2210 		       "module load failed!");
2211 		rc = -ENOMEM;
2212 	}
2213 
2214 	return rc;
2215 }
2216 
2217 void fc_destroy_fcp(void)
2218 {
2219 	if (scsi_pkt_cachep)
2220 		kmem_cache_destroy(scsi_pkt_cachep);
2221 }
2222 
2223 /**
2224  * fc_fcp_init() - Initialize the FCP layer for a local port
2225  * @lport: The local port to initialize the exchange layer for
2226  */
2227 int fc_fcp_init(struct fc_lport *lport)
2228 {
2229 	int rc;
2230 	struct fc_fcp_internal *si;
2231 
2232 	if (!lport->tt.fcp_cmd_send)
2233 		lport->tt.fcp_cmd_send = fc_fcp_cmd_send;
2234 
2235 	if (!lport->tt.fcp_cleanup)
2236 		lport->tt.fcp_cleanup = fc_fcp_cleanup;
2237 
2238 	if (!lport->tt.fcp_abort_io)
2239 		lport->tt.fcp_abort_io = fc_fcp_abort_io;
2240 
2241 	si = kzalloc(sizeof(struct fc_fcp_internal), GFP_KERNEL);
2242 	if (!si)
2243 		return -ENOMEM;
2244 	lport->scsi_priv = si;
2245 	si->max_can_queue = lport->host->can_queue;
2246 	INIT_LIST_HEAD(&si->scsi_pkt_queue);
2247 	spin_lock_init(&si->scsi_queue_lock);
2248 
2249 	si->scsi_pkt_pool = mempool_create_slab_pool(2, scsi_pkt_cachep);
2250 	if (!si->scsi_pkt_pool) {
2251 		rc = -ENOMEM;
2252 		goto free_internal;
2253 	}
2254 	return 0;
2255 
2256 free_internal:
2257 	kfree(si);
2258 	return rc;
2259 }
2260 EXPORT_SYMBOL(fc_fcp_init);
2261