1 /*
2  * This file is part of the Chelsio FCoE driver for Linux.
3  *
4  * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #include <linux/device.h>
36 #include <linux/delay.h>
37 #include <linux/ctype.h>
38 #include <linux/kernel.h>
39 #include <linux/slab.h>
40 #include <linux/string.h>
41 #include <linux/compiler.h>
42 #include <linux/export.h>
43 #include <linux/module.h>
44 #include <asm/unaligned.h>
45 #include <asm/page.h>
46 #include <scsi/scsi.h>
47 #include <scsi/scsi_device.h>
48 #include <scsi/scsi_transport_fc.h>
49 
50 #include "csio_hw.h"
51 #include "csio_lnode.h"
52 #include "csio_rnode.h"
53 #include "csio_scsi.h"
54 #include "csio_init.h"
55 
56 int csio_scsi_eqsize = 65536;
57 int csio_scsi_iqlen = 128;
58 int csio_scsi_ioreqs = 2048;
59 uint32_t csio_max_scan_tmo;
60 uint32_t csio_delta_scan_tmo = 5;
61 int csio_lun_qdepth = 32;
62 
63 static int csio_ddp_descs = 128;
64 
65 static int csio_do_abrt_cls(struct csio_hw *,
66 				      struct csio_ioreq *, bool);
67 
68 static void csio_scsis_uninit(struct csio_ioreq *, enum csio_scsi_ev);
69 static void csio_scsis_io_active(struct csio_ioreq *, enum csio_scsi_ev);
70 static void csio_scsis_tm_active(struct csio_ioreq *, enum csio_scsi_ev);
71 static void csio_scsis_aborting(struct csio_ioreq *, enum csio_scsi_ev);
72 static void csio_scsis_closing(struct csio_ioreq *, enum csio_scsi_ev);
73 static void csio_scsis_shost_cmpl_await(struct csio_ioreq *, enum csio_scsi_ev);
74 
75 /*
76  * csio_scsi_match_io - Match an ioreq with the given SCSI level data.
77  * @ioreq: The I/O request
78  * @sld: Level information
79  *
80  * Should be called with lock held.
81  *
82  */
83 static bool
84 csio_scsi_match_io(struct csio_ioreq *ioreq, struct csio_scsi_level_data *sld)
85 {
86 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(ioreq);
87 
88 	switch (sld->level) {
89 	case CSIO_LEV_LUN:
90 		if (scmnd == NULL)
91 			return false;
92 
93 		return ((ioreq->lnode == sld->lnode) &&
94 			(ioreq->rnode == sld->rnode) &&
95 			((uint64_t)scmnd->device->lun == sld->oslun));
96 
97 	case CSIO_LEV_RNODE:
98 		return ((ioreq->lnode == sld->lnode) &&
99 				(ioreq->rnode == sld->rnode));
100 	case CSIO_LEV_LNODE:
101 		return (ioreq->lnode == sld->lnode);
102 	case CSIO_LEV_ALL:
103 		return true;
104 	default:
105 		return false;
106 	}
107 }
108 
109 /*
110  * csio_scsi_gather_active_ios - Gather active I/Os based on level
111  * @scm: SCSI module
112  * @sld: Level information
113  * @dest: The queue where these I/Os have to be gathered.
114  *
115  * Should be called with lock held.
116  */
117 static void
118 csio_scsi_gather_active_ios(struct csio_scsim *scm,
119 			    struct csio_scsi_level_data *sld,
120 			    struct list_head *dest)
121 {
122 	struct list_head *tmp, *next;
123 
124 	if (list_empty(&scm->active_q))
125 		return;
126 
127 	/* Just splice the entire active_q into dest */
128 	if (sld->level == CSIO_LEV_ALL) {
129 		list_splice_tail_init(&scm->active_q, dest);
130 		return;
131 	}
132 
133 	list_for_each_safe(tmp, next, &scm->active_q) {
134 		if (csio_scsi_match_io((struct csio_ioreq *)tmp, sld)) {
135 			list_del_init(tmp);
136 			list_add_tail(tmp, dest);
137 		}
138 	}
139 }
140 
141 static inline bool
142 csio_scsi_itnexus_loss_error(uint16_t error)
143 {
144 	switch (error) {
145 	case FW_ERR_LINK_DOWN:
146 	case FW_RDEV_NOT_READY:
147 	case FW_ERR_RDEV_LOST:
148 	case FW_ERR_RDEV_LOGO:
149 	case FW_ERR_RDEV_IMPL_LOGO:
150 		return true;
151 	}
152 	return false;
153 }
154 
155 /*
156  * csio_scsi_fcp_cmnd - Frame the SCSI FCP command paylod.
157  * @req: IO req structure.
158  * @addr: DMA location to place the payload.
159  *
160  * This routine is shared between FCP_WRITE, FCP_READ and FCP_CMD requests.
161  */
162 static inline void
163 csio_scsi_fcp_cmnd(struct csio_ioreq *req, void *addr)
164 {
165 	struct fcp_cmnd *fcp_cmnd = (struct fcp_cmnd *)addr;
166 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
167 
168 	/* Check for Task Management */
169 	if (likely(csio_priv(scmnd)->fc_tm_flags == 0)) {
170 		int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
171 		fcp_cmnd->fc_tm_flags = 0;
172 		fcp_cmnd->fc_cmdref = 0;
173 
174 		memcpy(fcp_cmnd->fc_cdb, scmnd->cmnd, 16);
175 		fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
176 		fcp_cmnd->fc_dl = cpu_to_be32(scsi_bufflen(scmnd));
177 
178 		if (req->nsge)
179 			if (req->datadir == DMA_TO_DEVICE)
180 				fcp_cmnd->fc_flags = FCP_CFL_WRDATA;
181 			else
182 				fcp_cmnd->fc_flags = FCP_CFL_RDDATA;
183 		else
184 			fcp_cmnd->fc_flags = 0;
185 	} else {
186 		memset(fcp_cmnd, 0, sizeof(*fcp_cmnd));
187 		int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
188 		fcp_cmnd->fc_tm_flags = csio_priv(scmnd)->fc_tm_flags;
189 	}
190 }
191 
192 /*
193  * csio_scsi_init_cmd_wr - Initialize the SCSI CMD WR.
194  * @req: IO req structure.
195  * @addr: DMA location to place the payload.
196  * @size: Size of WR (including FW WR + immed data + rsp SG entry
197  *
198  * Wrapper for populating fw_scsi_cmd_wr.
199  */
200 static inline void
201 csio_scsi_init_cmd_wr(struct csio_ioreq *req, void *addr, uint32_t size)
202 {
203 	struct csio_hw *hw = req->lnode->hwp;
204 	struct csio_rnode *rn = req->rnode;
205 	struct fw_scsi_cmd_wr *wr = (struct fw_scsi_cmd_wr *)addr;
206 	struct csio_dma_buf *dma_buf;
207 	uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
208 
209 	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_CMD_WR) |
210 					  FW_SCSI_CMD_WR_IMMDLEN(imm));
211 	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
212 					    FW_WR_LEN16_V(
213 						DIV_ROUND_UP(size, 16)));
214 
215 	wr->cookie = (uintptr_t) req;
216 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
217 	wr->tmo_val = (uint8_t) req->tmo;
218 	wr->r3 = 0;
219 	memset(&wr->r5, 0, 8);
220 
221 	/* Get RSP DMA buffer */
222 	dma_buf = &req->dma_buf;
223 
224 	/* Prepare RSP SGL */
225 	wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
226 	wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
227 
228 	wr->r6 = 0;
229 
230 	wr->u.fcoe.ctl_pri = 0;
231 	wr->u.fcoe.cp_en_class = 0;
232 	wr->u.fcoe.r4_lo[0] = 0;
233 	wr->u.fcoe.r4_lo[1] = 0;
234 
235 	/* Frame a FCP command */
236 	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)addr +
237 				    sizeof(struct fw_scsi_cmd_wr)));
238 }
239 
240 #define CSIO_SCSI_CMD_WR_SZ(_imm)					\
241 	(sizeof(struct fw_scsi_cmd_wr) +		/* WR size */	\
242 	 ALIGN((_imm), 16))				/* Immed data */
243 
244 #define CSIO_SCSI_CMD_WR_SZ_16(_imm)					\
245 			(ALIGN(CSIO_SCSI_CMD_WR_SZ((_imm)), 16))
246 
247 /*
248  * csio_scsi_cmd - Create a SCSI CMD WR.
249  * @req: IO req structure.
250  *
251  * Gets a WR slot in the ingress queue and initializes it with SCSI CMD WR.
252  *
253  */
254 static inline void
255 csio_scsi_cmd(struct csio_ioreq *req)
256 {
257 	struct csio_wr_pair wrp;
258 	struct csio_hw *hw = req->lnode->hwp;
259 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
260 	uint32_t size = CSIO_SCSI_CMD_WR_SZ_16(scsim->proto_cmd_len);
261 
262 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
263 	if (unlikely(req->drv_status != 0))
264 		return;
265 
266 	if (wrp.size1 >= size) {
267 		/* Initialize WR in one shot */
268 		csio_scsi_init_cmd_wr(req, wrp.addr1, size);
269 	} else {
270 		uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
271 
272 		/*
273 		 * Make a temporary copy of the WR and write back
274 		 * the copy into the WR pair.
275 		 */
276 		csio_scsi_init_cmd_wr(req, (void *)tmpwr, size);
277 		memcpy(wrp.addr1, tmpwr, wrp.size1);
278 		memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
279 	}
280 }
281 
282 /*
283  * csio_scsi_init_ulptx_dsgl - Fill in a ULP_TX_SC_DSGL
284  * @hw: HW module
285  * @req: IO request
286  * @sgl: ULP TX SGL pointer.
287  *
288  */
289 static inline void
290 csio_scsi_init_ultptx_dsgl(struct csio_hw *hw, struct csio_ioreq *req,
291 			   struct ulptx_sgl *sgl)
292 {
293 	struct ulptx_sge_pair *sge_pair = NULL;
294 	struct scatterlist *sgel;
295 	uint32_t i = 0;
296 	uint32_t xfer_len;
297 	struct list_head *tmp;
298 	struct csio_dma_buf *dma_buf;
299 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
300 
301 	sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | ULPTX_MORE_F |
302 				     ULPTX_NSGE_V(req->nsge));
303 	/* Now add the data SGLs */
304 	if (likely(!req->dcopy)) {
305 		scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
306 			if (i == 0) {
307 				sgl->addr0 = cpu_to_be64(sg_dma_address(sgel));
308 				sgl->len0 = cpu_to_be32(sg_dma_len(sgel));
309 				sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
310 				continue;
311 			}
312 			if ((i - 1) & 0x1) {
313 				sge_pair->addr[1] = cpu_to_be64(
314 							sg_dma_address(sgel));
315 				sge_pair->len[1] = cpu_to_be32(
316 							sg_dma_len(sgel));
317 				sge_pair++;
318 			} else {
319 				sge_pair->addr[0] = cpu_to_be64(
320 							sg_dma_address(sgel));
321 				sge_pair->len[0] = cpu_to_be32(
322 							sg_dma_len(sgel));
323 			}
324 		}
325 	} else {
326 		/* Program sg elements with driver's DDP buffer */
327 		xfer_len = scsi_bufflen(scmnd);
328 		list_for_each(tmp, &req->gen_list) {
329 			dma_buf = (struct csio_dma_buf *)tmp;
330 			if (i == 0) {
331 				sgl->addr0 = cpu_to_be64(dma_buf->paddr);
332 				sgl->len0 = cpu_to_be32(
333 						min(xfer_len, dma_buf->len));
334 				sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
335 			} else if ((i - 1) & 0x1) {
336 				sge_pair->addr[1] = cpu_to_be64(dma_buf->paddr);
337 				sge_pair->len[1] = cpu_to_be32(
338 						min(xfer_len, dma_buf->len));
339 				sge_pair++;
340 			} else {
341 				sge_pair->addr[0] = cpu_to_be64(dma_buf->paddr);
342 				sge_pair->len[0] = cpu_to_be32(
343 						min(xfer_len, dma_buf->len));
344 			}
345 			xfer_len -= min(xfer_len, dma_buf->len);
346 			i++;
347 		}
348 	}
349 }
350 
351 /*
352  * csio_scsi_init_read_wr - Initialize the READ SCSI WR.
353  * @req: IO req structure.
354  * @wrp: DMA location to place the payload.
355  * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
356  *
357  * Wrapper for populating fw_scsi_read_wr.
358  */
359 static inline void
360 csio_scsi_init_read_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
361 {
362 	struct csio_hw *hw = req->lnode->hwp;
363 	struct csio_rnode *rn = req->rnode;
364 	struct fw_scsi_read_wr *wr = (struct fw_scsi_read_wr *)wrp;
365 	struct ulptx_sgl *sgl;
366 	struct csio_dma_buf *dma_buf;
367 	uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
368 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
369 
370 	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_READ_WR) |
371 				     FW_SCSI_READ_WR_IMMDLEN(imm));
372 	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
373 				       FW_WR_LEN16_V(DIV_ROUND_UP(size, 16)));
374 	wr->cookie = (uintptr_t)req;
375 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
376 	wr->tmo_val = (uint8_t)(req->tmo);
377 	wr->use_xfer_cnt = 1;
378 	wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
379 	wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
380 	/* Get RSP DMA buffer */
381 	dma_buf = &req->dma_buf;
382 
383 	/* Prepare RSP SGL */
384 	wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
385 	wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
386 
387 	wr->r4 = 0;
388 
389 	wr->u.fcoe.ctl_pri = 0;
390 	wr->u.fcoe.cp_en_class = 0;
391 	wr->u.fcoe.r3_lo[0] = 0;
392 	wr->u.fcoe.r3_lo[1] = 0;
393 	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
394 					sizeof(struct fw_scsi_read_wr)));
395 
396 	/* Move WR pointer past command and immediate data */
397 	sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
398 			      sizeof(struct fw_scsi_read_wr) + ALIGN(imm, 16));
399 
400 	/* Fill in the DSGL */
401 	csio_scsi_init_ultptx_dsgl(hw, req, sgl);
402 }
403 
404 /*
405  * csio_scsi_init_write_wr - Initialize the WRITE SCSI WR.
406  * @req: IO req structure.
407  * @wrp: DMA location to place the payload.
408  * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
409  *
410  * Wrapper for populating fw_scsi_write_wr.
411  */
412 static inline void
413 csio_scsi_init_write_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
414 {
415 	struct csio_hw *hw = req->lnode->hwp;
416 	struct csio_rnode *rn = req->rnode;
417 	struct fw_scsi_write_wr *wr = (struct fw_scsi_write_wr *)wrp;
418 	struct ulptx_sgl *sgl;
419 	struct csio_dma_buf *dma_buf;
420 	uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
421 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
422 
423 	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_WRITE_WR) |
424 				     FW_SCSI_WRITE_WR_IMMDLEN(imm));
425 	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
426 				       FW_WR_LEN16_V(DIV_ROUND_UP(size, 16)));
427 	wr->cookie = (uintptr_t)req;
428 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
429 	wr->tmo_val = (uint8_t)(req->tmo);
430 	wr->use_xfer_cnt = 1;
431 	wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
432 	wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
433 	/* Get RSP DMA buffer */
434 	dma_buf = &req->dma_buf;
435 
436 	/* Prepare RSP SGL */
437 	wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
438 	wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
439 
440 	wr->r4 = 0;
441 
442 	wr->u.fcoe.ctl_pri = 0;
443 	wr->u.fcoe.cp_en_class = 0;
444 	wr->u.fcoe.r3_lo[0] = 0;
445 	wr->u.fcoe.r3_lo[1] = 0;
446 	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
447 					sizeof(struct fw_scsi_write_wr)));
448 
449 	/* Move WR pointer past command and immediate data */
450 	sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
451 			      sizeof(struct fw_scsi_write_wr) + ALIGN(imm, 16));
452 
453 	/* Fill in the DSGL */
454 	csio_scsi_init_ultptx_dsgl(hw, req, sgl);
455 }
456 
457 /* Calculate WR size needed for fw_scsi_read_wr/fw_scsi_write_wr */
458 #define CSIO_SCSI_DATA_WRSZ(req, oper, sz, imm)				       \
459 do {									       \
460 	(sz) = sizeof(struct fw_scsi_##oper##_wr) +	/* WR size */          \
461 	       ALIGN((imm), 16) +			/* Immed data */       \
462 	       sizeof(struct ulptx_sgl);		/* ulptx_sgl */	       \
463 									       \
464 	if (unlikely((req)->nsge > 1))				               \
465 		(sz) += (sizeof(struct ulptx_sge_pair) *		       \
466 				(ALIGN(((req)->nsge - 1), 2) / 2));            \
467 							/* Data SGE */	       \
468 } while (0)
469 
470 /*
471  * csio_scsi_read - Create a SCSI READ WR.
472  * @req: IO req structure.
473  *
474  * Gets a WR slot in the ingress queue and initializes it with
475  * SCSI READ WR.
476  *
477  */
478 static inline void
479 csio_scsi_read(struct csio_ioreq *req)
480 {
481 	struct csio_wr_pair wrp;
482 	uint32_t size;
483 	struct csio_hw *hw = req->lnode->hwp;
484 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
485 
486 	CSIO_SCSI_DATA_WRSZ(req, read, size, scsim->proto_cmd_len);
487 	size = ALIGN(size, 16);
488 
489 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
490 	if (likely(req->drv_status == 0)) {
491 		if (likely(wrp.size1 >= size)) {
492 			/* Initialize WR in one shot */
493 			csio_scsi_init_read_wr(req, wrp.addr1, size);
494 		} else {
495 			uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
496 			/*
497 			 * Make a temporary copy of the WR and write back
498 			 * the copy into the WR pair.
499 			 */
500 			csio_scsi_init_read_wr(req, (void *)tmpwr, size);
501 			memcpy(wrp.addr1, tmpwr, wrp.size1);
502 			memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
503 		}
504 	}
505 }
506 
507 /*
508  * csio_scsi_write - Create a SCSI WRITE WR.
509  * @req: IO req structure.
510  *
511  * Gets a WR slot in the ingress queue and initializes it with
512  * SCSI WRITE WR.
513  *
514  */
515 static inline void
516 csio_scsi_write(struct csio_ioreq *req)
517 {
518 	struct csio_wr_pair wrp;
519 	uint32_t size;
520 	struct csio_hw *hw = req->lnode->hwp;
521 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
522 
523 	CSIO_SCSI_DATA_WRSZ(req, write, size, scsim->proto_cmd_len);
524 	size = ALIGN(size, 16);
525 
526 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
527 	if (likely(req->drv_status == 0)) {
528 		if (likely(wrp.size1 >= size)) {
529 			/* Initialize WR in one shot */
530 			csio_scsi_init_write_wr(req, wrp.addr1, size);
531 		} else {
532 			uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
533 			/*
534 			 * Make a temporary copy of the WR and write back
535 			 * the copy into the WR pair.
536 			 */
537 			csio_scsi_init_write_wr(req, (void *)tmpwr, size);
538 			memcpy(wrp.addr1, tmpwr, wrp.size1);
539 			memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
540 		}
541 	}
542 }
543 
544 /*
545  * csio_setup_ddp - Setup DDP buffers for Read request.
546  * @req: IO req structure.
547  *
548  * Checks SGLs/Data buffers are virtually contiguous required for DDP.
549  * If contiguous,driver posts SGLs in the WR otherwise post internal
550  * buffers for such request for DDP.
551  */
552 static inline void
553 csio_setup_ddp(struct csio_scsim *scsim, struct csio_ioreq *req)
554 {
555 #ifdef __CSIO_DEBUG__
556 	struct csio_hw *hw = req->lnode->hwp;
557 #endif
558 	struct scatterlist *sgel = NULL;
559 	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
560 	uint64_t sg_addr = 0;
561 	uint32_t ddp_pagesz = 4096;
562 	uint32_t buf_off;
563 	struct csio_dma_buf *dma_buf = NULL;
564 	uint32_t alloc_len = 0;
565 	uint32_t xfer_len = 0;
566 	uint32_t sg_len = 0;
567 	uint32_t i;
568 
569 	scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
570 		sg_addr = sg_dma_address(sgel);
571 		sg_len	= sg_dma_len(sgel);
572 
573 		buf_off = sg_addr & (ddp_pagesz - 1);
574 
575 		/* Except 1st buffer,all buffer addr have to be Page aligned */
576 		if (i != 0 && buf_off) {
577 			csio_dbg(hw, "SGL addr not DDP aligned (%llx:%d)\n",
578 				 sg_addr, sg_len);
579 			goto unaligned;
580 		}
581 
582 		/* Except last buffer,all buffer must end on page boundary */
583 		if ((i != (req->nsge - 1)) &&
584 			((buf_off + sg_len) & (ddp_pagesz - 1))) {
585 			csio_dbg(hw,
586 				 "SGL addr not ending on page boundary"
587 				 "(%llx:%d)\n", sg_addr, sg_len);
588 			goto unaligned;
589 		}
590 	}
591 
592 	/* SGL's are virtually contiguous. HW will DDP to SGLs */
593 	req->dcopy = 0;
594 	csio_scsi_read(req);
595 
596 	return;
597 
598 unaligned:
599 	CSIO_INC_STATS(scsim, n_unaligned);
600 	/*
601 	 * For unaligned SGLs, driver will allocate internal DDP buffer.
602 	 * Once command is completed data from DDP buffer copied to SGLs
603 	 */
604 	req->dcopy = 1;
605 
606 	/* Use gen_list to store the DDP buffers */
607 	INIT_LIST_HEAD(&req->gen_list);
608 	xfer_len = scsi_bufflen(scmnd);
609 
610 	i = 0;
611 	/* Allocate ddp buffers for this request */
612 	while (alloc_len < xfer_len) {
613 		dma_buf = csio_get_scsi_ddp(scsim);
614 		if (dma_buf == NULL || i > scsim->max_sge) {
615 			req->drv_status = -EBUSY;
616 			break;
617 		}
618 		alloc_len += dma_buf->len;
619 		/* Added to IO req */
620 		list_add_tail(&dma_buf->list, &req->gen_list);
621 		i++;
622 	}
623 
624 	if (!req->drv_status) {
625 		/* set number of ddp bufs used */
626 		req->nsge = i;
627 		csio_scsi_read(req);
628 		return;
629 	}
630 
631 	 /* release dma descs */
632 	if (i > 0)
633 		csio_put_scsi_ddp_list(scsim, &req->gen_list, i);
634 }
635 
636 /*
637  * csio_scsi_init_abrt_cls_wr - Initialize an ABORT/CLOSE WR.
638  * @req: IO req structure.
639  * @addr: DMA location to place the payload.
640  * @size: Size of WR
641  * @abort: abort OR close
642  *
643  * Wrapper for populating fw_scsi_cmd_wr.
644  */
645 static inline void
646 csio_scsi_init_abrt_cls_wr(struct csio_ioreq *req, void *addr, uint32_t size,
647 			   bool abort)
648 {
649 	struct csio_hw *hw = req->lnode->hwp;
650 	struct csio_rnode *rn = req->rnode;
651 	struct fw_scsi_abrt_cls_wr *wr = (struct fw_scsi_abrt_cls_wr *)addr;
652 
653 	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_ABRT_CLS_WR));
654 	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
655 					    FW_WR_LEN16_V(
656 						DIV_ROUND_UP(size, 16)));
657 
658 	wr->cookie = (uintptr_t) req;
659 	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
660 	wr->tmo_val = (uint8_t) req->tmo;
661 	/* 0 for CHK_ALL_IO tells FW to look up t_cookie */
662 	wr->sub_opcode_to_chk_all_io =
663 				(FW_SCSI_ABRT_CLS_WR_SUB_OPCODE(abort) |
664 				 FW_SCSI_ABRT_CLS_WR_CHK_ALL_IO(0));
665 	wr->r3[0] = 0;
666 	wr->r3[1] = 0;
667 	wr->r3[2] = 0;
668 	wr->r3[3] = 0;
669 	/* Since we re-use the same ioreq for abort as well */
670 	wr->t_cookie = (uintptr_t) req;
671 }
672 
673 static inline void
674 csio_scsi_abrt_cls(struct csio_ioreq *req, bool abort)
675 {
676 	struct csio_wr_pair wrp;
677 	struct csio_hw *hw = req->lnode->hwp;
678 	uint32_t size = ALIGN(sizeof(struct fw_scsi_abrt_cls_wr), 16);
679 
680 	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
681 	if (req->drv_status != 0)
682 		return;
683 
684 	if (wrp.size1 >= size) {
685 		/* Initialize WR in one shot */
686 		csio_scsi_init_abrt_cls_wr(req, wrp.addr1, size, abort);
687 	} else {
688 		uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
689 		/*
690 		 * Make a temporary copy of the WR and write back
691 		 * the copy into the WR pair.
692 		 */
693 		csio_scsi_init_abrt_cls_wr(req, (void *)tmpwr, size, abort);
694 		memcpy(wrp.addr1, tmpwr, wrp.size1);
695 		memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
696 	}
697 }
698 
699 /*****************************************************************************/
700 /* START: SCSI SM                                                            */
701 /*****************************************************************************/
702 static void
703 csio_scsis_uninit(struct csio_ioreq *req, enum csio_scsi_ev evt)
704 {
705 	struct csio_hw *hw = req->lnode->hwp;
706 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
707 
708 	switch (evt) {
709 	case CSIO_SCSIE_START_IO:
710 
711 		if (req->nsge) {
712 			if (req->datadir == DMA_TO_DEVICE) {
713 				req->dcopy = 0;
714 				csio_scsi_write(req);
715 			} else
716 				csio_setup_ddp(scsim, req);
717 		} else {
718 			csio_scsi_cmd(req);
719 		}
720 
721 		if (likely(req->drv_status == 0)) {
722 			/* change state and enqueue on active_q */
723 			csio_set_state(&req->sm, csio_scsis_io_active);
724 			list_add_tail(&req->sm.sm_list, &scsim->active_q);
725 			csio_wr_issue(hw, req->eq_idx, false);
726 			CSIO_INC_STATS(scsim, n_active);
727 
728 			return;
729 		}
730 		break;
731 
732 	case CSIO_SCSIE_START_TM:
733 		csio_scsi_cmd(req);
734 		if (req->drv_status == 0) {
735 			/*
736 			 * NOTE: We collect the affected I/Os prior to issuing
737 			 * LUN reset, and not after it. This is to prevent
738 			 * aborting I/Os that get issued after the LUN reset,
739 			 * but prior to LUN reset completion (in the event that
740 			 * the host stack has not blocked I/Os to a LUN that is
741 			 * being reset.
742 			 */
743 			csio_set_state(&req->sm, csio_scsis_tm_active);
744 			list_add_tail(&req->sm.sm_list, &scsim->active_q);
745 			csio_wr_issue(hw, req->eq_idx, false);
746 			CSIO_INC_STATS(scsim, n_tm_active);
747 		}
748 		return;
749 
750 	case CSIO_SCSIE_ABORT:
751 	case CSIO_SCSIE_CLOSE:
752 		/*
753 		 * NOTE:
754 		 * We could get here due to  :
755 		 * - a window in the cleanup path of the SCSI module
756 		 *   (csio_scsi_abort_io()). Please see NOTE in this function.
757 		 * - a window in the time we tried to issue an abort/close
758 		 *   of a request to FW, and the FW completed the request
759 		 *   itself.
760 		 *   Print a message for now, and return INVAL either way.
761 		 */
762 		req->drv_status = -EINVAL;
763 		csio_warn(hw, "Trying to abort/close completed IO:%p!\n", req);
764 		break;
765 
766 	default:
767 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
768 		CSIO_DB_ASSERT(0);
769 	}
770 }
771 
772 static void
773 csio_scsis_io_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
774 {
775 	struct csio_hw *hw = req->lnode->hwp;
776 	struct csio_scsim *scm = csio_hw_to_scsim(hw);
777 	struct csio_rnode *rn;
778 
779 	switch (evt) {
780 	case CSIO_SCSIE_COMPLETED:
781 		CSIO_DEC_STATS(scm, n_active);
782 		list_del_init(&req->sm.sm_list);
783 		csio_set_state(&req->sm, csio_scsis_uninit);
784 		/*
785 		 * In MSIX mode, with multiple queues, the SCSI compeltions
786 		 * could reach us sooner than the FW events sent to indicate
787 		 * I-T nexus loss (link down, remote device logo etc). We
788 		 * dont want to be returning such I/Os to the upper layer
789 		 * immediately, since we wouldnt have reported the I-T nexus
790 		 * loss itself. This forces us to serialize such completions
791 		 * with the reporting of the I-T nexus loss. Therefore, we
792 		 * internally queue up such up such completions in the rnode.
793 		 * The reporting of I-T nexus loss to the upper layer is then
794 		 * followed by the returning of I/Os in this internal queue.
795 		 * Having another state alongwith another queue helps us take
796 		 * actions for events such as ABORT received while we are
797 		 * in this rnode queue.
798 		 */
799 		if (unlikely(req->wr_status != FW_SUCCESS)) {
800 			rn = req->rnode;
801 			/*
802 			 * FW says remote device is lost, but rnode
803 			 * doesnt reflect it.
804 			 */
805 			if (csio_scsi_itnexus_loss_error(req->wr_status) &&
806 						csio_is_rnode_ready(rn)) {
807 				csio_set_state(&req->sm,
808 						csio_scsis_shost_cmpl_await);
809 				list_add_tail(&req->sm.sm_list,
810 					      &rn->host_cmpl_q);
811 			}
812 		}
813 
814 		break;
815 
816 	case CSIO_SCSIE_ABORT:
817 		csio_scsi_abrt_cls(req, SCSI_ABORT);
818 		if (req->drv_status == 0) {
819 			csio_wr_issue(hw, req->eq_idx, false);
820 			csio_set_state(&req->sm, csio_scsis_aborting);
821 		}
822 		break;
823 
824 	case CSIO_SCSIE_CLOSE:
825 		csio_scsi_abrt_cls(req, SCSI_CLOSE);
826 		if (req->drv_status == 0) {
827 			csio_wr_issue(hw, req->eq_idx, false);
828 			csio_set_state(&req->sm, csio_scsis_closing);
829 		}
830 		break;
831 
832 	case CSIO_SCSIE_DRVCLEANUP:
833 		req->wr_status = FW_HOSTERROR;
834 		CSIO_DEC_STATS(scm, n_active);
835 		csio_set_state(&req->sm, csio_scsis_uninit);
836 		break;
837 
838 	default:
839 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
840 		CSIO_DB_ASSERT(0);
841 	}
842 }
843 
844 static void
845 csio_scsis_tm_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
846 {
847 	struct csio_hw *hw = req->lnode->hwp;
848 	struct csio_scsim *scm = csio_hw_to_scsim(hw);
849 
850 	switch (evt) {
851 	case CSIO_SCSIE_COMPLETED:
852 		CSIO_DEC_STATS(scm, n_tm_active);
853 		list_del_init(&req->sm.sm_list);
854 		csio_set_state(&req->sm, csio_scsis_uninit);
855 
856 		break;
857 
858 	case CSIO_SCSIE_ABORT:
859 		csio_scsi_abrt_cls(req, SCSI_ABORT);
860 		if (req->drv_status == 0) {
861 			csio_wr_issue(hw, req->eq_idx, false);
862 			csio_set_state(&req->sm, csio_scsis_aborting);
863 		}
864 		break;
865 
866 
867 	case CSIO_SCSIE_CLOSE:
868 		csio_scsi_abrt_cls(req, SCSI_CLOSE);
869 		if (req->drv_status == 0) {
870 			csio_wr_issue(hw, req->eq_idx, false);
871 			csio_set_state(&req->sm, csio_scsis_closing);
872 		}
873 		break;
874 
875 	case CSIO_SCSIE_DRVCLEANUP:
876 		req->wr_status = FW_HOSTERROR;
877 		CSIO_DEC_STATS(scm, n_tm_active);
878 		csio_set_state(&req->sm, csio_scsis_uninit);
879 		break;
880 
881 	default:
882 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
883 		CSIO_DB_ASSERT(0);
884 	}
885 }
886 
887 static void
888 csio_scsis_aborting(struct csio_ioreq *req, enum csio_scsi_ev evt)
889 {
890 	struct csio_hw *hw = req->lnode->hwp;
891 	struct csio_scsim *scm = csio_hw_to_scsim(hw);
892 
893 	switch (evt) {
894 	case CSIO_SCSIE_COMPLETED:
895 		csio_dbg(hw,
896 			 "ioreq %p recvd cmpltd (wr_status:%d) "
897 			 "in aborting st\n", req, req->wr_status);
898 		/*
899 		 * Use -ECANCELED to explicitly tell the ABORTED event that
900 		 * the original I/O was returned to driver by FW.
901 		 * We dont really care if the I/O was returned with success by
902 		 * FW (because the ABORT and completion of the I/O crossed each
903 		 * other), or any other return value. Once we are in aborting
904 		 * state, the success or failure of the I/O is unimportant to
905 		 * us.
906 		 */
907 		req->drv_status = -ECANCELED;
908 		break;
909 
910 	case CSIO_SCSIE_ABORT:
911 		CSIO_INC_STATS(scm, n_abrt_dups);
912 		break;
913 
914 	case CSIO_SCSIE_ABORTED:
915 
916 		csio_dbg(hw, "abort of %p return status:0x%x drv_status:%x\n",
917 			 req, req->wr_status, req->drv_status);
918 		/*
919 		 * Check if original I/O WR completed before the Abort
920 		 * completion.
921 		 */
922 		if (req->drv_status != -ECANCELED) {
923 			csio_warn(hw,
924 				  "Abort completed before original I/O,"
925 				   " req:%p\n", req);
926 			CSIO_DB_ASSERT(0);
927 		}
928 
929 		/*
930 		 * There are the following possible scenarios:
931 		 * 1. The abort completed successfully, FW returned FW_SUCCESS.
932 		 * 2. The completion of an I/O and the receipt of
933 		 *    abort for that I/O by the FW crossed each other.
934 		 *    The FW returned FW_EINVAL. The original I/O would have
935 		 *    returned with FW_SUCCESS or any other SCSI error.
936 		 * 3. The FW couldn't sent the abort out on the wire, as there
937 		 *    was an I-T nexus loss (link down, remote device logged
938 		 *    out etc). FW sent back an appropriate IT nexus loss status
939 		 *    for the abort.
940 		 * 4. FW sent an abort, but abort timed out (remote device
941 		 *    didnt respond). FW replied back with
942 		 *    FW_SCSI_ABORT_TIMEDOUT.
943 		 * 5. FW couldn't genuinely abort the request for some reason,
944 		 *    and sent us an error.
945 		 *
946 		 * The first 3 scenarios are treated as  succesful abort
947 		 * operations by the host, while the last 2 are failed attempts
948 		 * to abort. Manipulate the return value of the request
949 		 * appropriately, so that host can convey these results
950 		 * back to the upper layer.
951 		 */
952 		if ((req->wr_status == FW_SUCCESS) ||
953 		    (req->wr_status == FW_EINVAL) ||
954 		    csio_scsi_itnexus_loss_error(req->wr_status))
955 			req->wr_status = FW_SCSI_ABORT_REQUESTED;
956 
957 		CSIO_DEC_STATS(scm, n_active);
958 		list_del_init(&req->sm.sm_list);
959 		csio_set_state(&req->sm, csio_scsis_uninit);
960 		break;
961 
962 	case CSIO_SCSIE_DRVCLEANUP:
963 		req->wr_status = FW_HOSTERROR;
964 		CSIO_DEC_STATS(scm, n_active);
965 		csio_set_state(&req->sm, csio_scsis_uninit);
966 		break;
967 
968 	case CSIO_SCSIE_CLOSE:
969 		/*
970 		 * We can receive this event from the module
971 		 * cleanup paths, if the FW forgot to reply to the ABORT WR
972 		 * and left this ioreq in this state. For now, just ignore
973 		 * the event. The CLOSE event is sent to this state, as
974 		 * the LINK may have already gone down.
975 		 */
976 		break;
977 
978 	default:
979 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
980 		CSIO_DB_ASSERT(0);
981 	}
982 }
983 
984 static void
985 csio_scsis_closing(struct csio_ioreq *req, enum csio_scsi_ev evt)
986 {
987 	struct csio_hw *hw = req->lnode->hwp;
988 	struct csio_scsim *scm = csio_hw_to_scsim(hw);
989 
990 	switch (evt) {
991 	case CSIO_SCSIE_COMPLETED:
992 		csio_dbg(hw,
993 			 "ioreq %p recvd cmpltd (wr_status:%d) "
994 			 "in closing st\n", req, req->wr_status);
995 		/*
996 		 * Use -ECANCELED to explicitly tell the CLOSED event that
997 		 * the original I/O was returned to driver by FW.
998 		 * We dont really care if the I/O was returned with success by
999 		 * FW (because the CLOSE and completion of the I/O crossed each
1000 		 * other), or any other return value. Once we are in aborting
1001 		 * state, the success or failure of the I/O is unimportant to
1002 		 * us.
1003 		 */
1004 		req->drv_status = -ECANCELED;
1005 		break;
1006 
1007 	case CSIO_SCSIE_CLOSED:
1008 		/*
1009 		 * Check if original I/O WR completed before the Close
1010 		 * completion.
1011 		 */
1012 		if (req->drv_status != -ECANCELED) {
1013 			csio_fatal(hw,
1014 				   "Close completed before original I/O,"
1015 				   " req:%p\n", req);
1016 			CSIO_DB_ASSERT(0);
1017 		}
1018 
1019 		/*
1020 		 * Either close succeeded, or we issued close to FW at the
1021 		 * same time FW compelted it to us. Either way, the I/O
1022 		 * is closed.
1023 		 */
1024 		CSIO_DB_ASSERT((req->wr_status == FW_SUCCESS) ||
1025 					(req->wr_status == FW_EINVAL));
1026 		req->wr_status = FW_SCSI_CLOSE_REQUESTED;
1027 
1028 		CSIO_DEC_STATS(scm, n_active);
1029 		list_del_init(&req->sm.sm_list);
1030 		csio_set_state(&req->sm, csio_scsis_uninit);
1031 		break;
1032 
1033 	case CSIO_SCSIE_CLOSE:
1034 		break;
1035 
1036 	case CSIO_SCSIE_DRVCLEANUP:
1037 		req->wr_status = FW_HOSTERROR;
1038 		CSIO_DEC_STATS(scm, n_active);
1039 		csio_set_state(&req->sm, csio_scsis_uninit);
1040 		break;
1041 
1042 	default:
1043 		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
1044 		CSIO_DB_ASSERT(0);
1045 	}
1046 }
1047 
1048 static void
1049 csio_scsis_shost_cmpl_await(struct csio_ioreq *req, enum csio_scsi_ev evt)
1050 {
1051 	switch (evt) {
1052 	case CSIO_SCSIE_ABORT:
1053 	case CSIO_SCSIE_CLOSE:
1054 		/*
1055 		 * Just succeed the abort request, and hope that
1056 		 * the remote device unregister path will cleanup
1057 		 * this I/O to the upper layer within a sane
1058 		 * amount of time.
1059 		 */
1060 		/*
1061 		 * A close can come in during a LINK DOWN. The FW would have
1062 		 * returned us the I/O back, but not the remote device lost
1063 		 * FW event. In this interval, if the I/O times out at the upper
1064 		 * layer, a close can come in. Take the same action as abort:
1065 		 * return success, and hope that the remote device unregister
1066 		 * path will cleanup this I/O. If the FW still doesnt send
1067 		 * the msg, the close times out, and the upper layer resorts
1068 		 * to the next level of error recovery.
1069 		 */
1070 		req->drv_status = 0;
1071 		break;
1072 	case CSIO_SCSIE_DRVCLEANUP:
1073 		csio_set_state(&req->sm, csio_scsis_uninit);
1074 		break;
1075 	default:
1076 		csio_dbg(req->lnode->hwp, "Unhandled event:%d sent to req:%p\n",
1077 			 evt, req);
1078 		CSIO_DB_ASSERT(0);
1079 	}
1080 }
1081 
1082 /*
1083  * csio_scsi_cmpl_handler - WR completion handler for SCSI.
1084  * @hw: HW module.
1085  * @wr: The completed WR from the ingress queue.
1086  * @len: Length of the WR.
1087  * @flb: Freelist buffer array.
1088  * @priv: Private object
1089  * @scsiwr: Pointer to SCSI WR.
1090  *
1091  * This is the WR completion handler called per completion from the
1092  * ISR. It is called with lock held. It walks past the RSS and CPL message
1093  * header where the actual WR is present.
1094  * It then gets the status, WR handle (ioreq pointer) and the len of
1095  * the WR, based on WR opcode. Only on a non-good status is the entire
1096  * WR copied into the WR cache (ioreq->fw_wr).
1097  * The ioreq corresponding to the WR is returned to the caller.
1098  * NOTE: The SCSI queue doesnt allocate a freelist today, hence
1099  * no freelist buffer is expected.
1100  */
1101 struct csio_ioreq *
1102 csio_scsi_cmpl_handler(struct csio_hw *hw, void *wr, uint32_t len,
1103 		     struct csio_fl_dma_buf *flb, void *priv, uint8_t **scsiwr)
1104 {
1105 	struct csio_ioreq *ioreq = NULL;
1106 	struct cpl_fw6_msg *cpl;
1107 	uint8_t *tempwr;
1108 	uint8_t	status;
1109 	struct csio_scsim *scm = csio_hw_to_scsim(hw);
1110 
1111 	/* skip RSS header */
1112 	cpl = (struct cpl_fw6_msg *)((uintptr_t)wr + sizeof(__be64));
1113 
1114 	if (unlikely(cpl->opcode != CPL_FW6_MSG)) {
1115 		csio_warn(hw, "Error: Invalid CPL msg %x recvd on SCSI q\n",
1116 			  cpl->opcode);
1117 		CSIO_INC_STATS(scm, n_inval_cplop);
1118 		return NULL;
1119 	}
1120 
1121 	tempwr = (uint8_t *)(cpl->data);
1122 	status = csio_wr_status(tempwr);
1123 	*scsiwr = tempwr;
1124 
1125 	if (likely((*tempwr == FW_SCSI_READ_WR) ||
1126 			(*tempwr == FW_SCSI_WRITE_WR) ||
1127 			(*tempwr == FW_SCSI_CMD_WR))) {
1128 		ioreq = (struct csio_ioreq *)((uintptr_t)
1129 				 (((struct fw_scsi_read_wr *)tempwr)->cookie));
1130 		CSIO_DB_ASSERT(virt_addr_valid(ioreq));
1131 
1132 		ioreq->wr_status = status;
1133 
1134 		return ioreq;
1135 	}
1136 
1137 	if (*tempwr == FW_SCSI_ABRT_CLS_WR) {
1138 		ioreq = (struct csio_ioreq *)((uintptr_t)
1139 			 (((struct fw_scsi_abrt_cls_wr *)tempwr)->cookie));
1140 		CSIO_DB_ASSERT(virt_addr_valid(ioreq));
1141 
1142 		ioreq->wr_status = status;
1143 		return ioreq;
1144 	}
1145 
1146 	csio_warn(hw, "WR with invalid opcode in SCSI IQ: %x\n", *tempwr);
1147 	CSIO_INC_STATS(scm, n_inval_scsiop);
1148 	return NULL;
1149 }
1150 
1151 /*
1152  * csio_scsi_cleanup_io_q - Cleanup the given queue.
1153  * @scm: SCSI module.
1154  * @q: Queue to be cleaned up.
1155  *
1156  * Called with lock held. Has to exit with lock held.
1157  */
1158 void
1159 csio_scsi_cleanup_io_q(struct csio_scsim *scm, struct list_head *q)
1160 {
1161 	struct csio_hw *hw = scm->hw;
1162 	struct csio_ioreq *ioreq;
1163 	struct list_head *tmp, *next;
1164 	struct scsi_cmnd *scmnd;
1165 
1166 	/* Call back the completion routines of the active_q */
1167 	list_for_each_safe(tmp, next, q) {
1168 		ioreq = (struct csio_ioreq *)tmp;
1169 		csio_scsi_drvcleanup(ioreq);
1170 		list_del_init(&ioreq->sm.sm_list);
1171 		scmnd = csio_scsi_cmnd(ioreq);
1172 		spin_unlock_irq(&hw->lock);
1173 
1174 		/*
1175 		 * Upper layers may have cleared this command, hence this
1176 		 * check to avoid accessing stale references.
1177 		 */
1178 		if (scmnd != NULL)
1179 			ioreq->io_cbfn(hw, ioreq);
1180 
1181 		spin_lock_irq(&scm->freelist_lock);
1182 		csio_put_scsi_ioreq(scm, ioreq);
1183 		spin_unlock_irq(&scm->freelist_lock);
1184 
1185 		spin_lock_irq(&hw->lock);
1186 	}
1187 }
1188 
1189 #define CSIO_SCSI_ABORT_Q_POLL_MS		2000
1190 
1191 static void
1192 csio_abrt_cls(struct csio_ioreq *ioreq, struct scsi_cmnd *scmnd)
1193 {
1194 	struct csio_lnode *ln = ioreq->lnode;
1195 	struct csio_hw *hw = ln->hwp;
1196 	int ready = 0;
1197 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1198 	int rv;
1199 
1200 	if (csio_scsi_cmnd(ioreq) != scmnd) {
1201 		CSIO_INC_STATS(scsim, n_abrt_race_comp);
1202 		return;
1203 	}
1204 
1205 	ready = csio_is_lnode_ready(ln);
1206 
1207 	rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
1208 	if (rv != 0) {
1209 		if (ready)
1210 			CSIO_INC_STATS(scsim, n_abrt_busy_error);
1211 		else
1212 			CSIO_INC_STATS(scsim, n_cls_busy_error);
1213 	}
1214 }
1215 
1216 /*
1217  * csio_scsi_abort_io_q - Abort all I/Os on given queue
1218  * @scm: SCSI module.
1219  * @q: Queue to abort.
1220  * @tmo: Timeout in ms
1221  *
1222  * Attempt to abort all I/Os on given queue, and wait for a max
1223  * of tmo milliseconds for them to complete. Returns success
1224  * if all I/Os are aborted. Else returns -ETIMEDOUT.
1225  * Should be entered with lock held. Exits with lock held.
1226  * NOTE:
1227  * Lock has to be held across the loop that aborts I/Os, since dropping the lock
1228  * in between can cause the list to be corrupted. As a result, the caller
1229  * of this function has to ensure that the number of I/os to be aborted
1230  * is finite enough to not cause lock-held-for-too-long issues.
1231  */
1232 static int
1233 csio_scsi_abort_io_q(struct csio_scsim *scm, struct list_head *q, uint32_t tmo)
1234 {
1235 	struct csio_hw *hw = scm->hw;
1236 	struct list_head *tmp, *next;
1237 	int count = DIV_ROUND_UP(tmo, CSIO_SCSI_ABORT_Q_POLL_MS);
1238 	struct scsi_cmnd *scmnd;
1239 
1240 	if (list_empty(q))
1241 		return 0;
1242 
1243 	csio_dbg(hw, "Aborting SCSI I/Os\n");
1244 
1245 	/* Now abort/close I/Os in the queue passed */
1246 	list_for_each_safe(tmp, next, q) {
1247 		scmnd = csio_scsi_cmnd((struct csio_ioreq *)tmp);
1248 		csio_abrt_cls((struct csio_ioreq *)tmp, scmnd);
1249 	}
1250 
1251 	/* Wait till all active I/Os are completed/aborted/closed */
1252 	while (!list_empty(q) && count--) {
1253 		spin_unlock_irq(&hw->lock);
1254 		msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1255 		spin_lock_irq(&hw->lock);
1256 	}
1257 
1258 	/* all aborts completed */
1259 	if (list_empty(q))
1260 		return 0;
1261 
1262 	return -ETIMEDOUT;
1263 }
1264 
1265 /*
1266  * csio_scsim_cleanup_io - Cleanup all I/Os in SCSI module.
1267  * @scm: SCSI module.
1268  * @abort: abort required.
1269  * Called with lock held, should exit with lock held.
1270  * Can sleep when waiting for I/Os to complete.
1271  */
1272 int
1273 csio_scsim_cleanup_io(struct csio_scsim *scm, bool abort)
1274 {
1275 	struct csio_hw *hw = scm->hw;
1276 	int rv = 0;
1277 	int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
1278 
1279 	/* No I/Os pending */
1280 	if (list_empty(&scm->active_q))
1281 		return 0;
1282 
1283 	/* Wait until all active I/Os are completed */
1284 	while (!list_empty(&scm->active_q) && count--) {
1285 		spin_unlock_irq(&hw->lock);
1286 		msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1287 		spin_lock_irq(&hw->lock);
1288 	}
1289 
1290 	/* all I/Os completed */
1291 	if (list_empty(&scm->active_q))
1292 		return 0;
1293 
1294 	/* Else abort */
1295 	if (abort) {
1296 		rv = csio_scsi_abort_io_q(scm, &scm->active_q, 30000);
1297 		if (rv == 0)
1298 			return rv;
1299 		csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
1300 	}
1301 
1302 	csio_scsi_cleanup_io_q(scm, &scm->active_q);
1303 
1304 	CSIO_DB_ASSERT(list_empty(&scm->active_q));
1305 
1306 	return rv;
1307 }
1308 
1309 /*
1310  * csio_scsim_cleanup_io_lnode - Cleanup all I/Os of given lnode.
1311  * @scm: SCSI module.
1312  * @lnode: lnode
1313  *
1314  * Called with lock held, should exit with lock held.
1315  * Can sleep (with dropped lock) when waiting for I/Os to complete.
1316  */
1317 int
1318 csio_scsim_cleanup_io_lnode(struct csio_scsim *scm, struct csio_lnode *ln)
1319 {
1320 	struct csio_hw *hw = scm->hw;
1321 	struct csio_scsi_level_data sld;
1322 	int rv;
1323 	int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
1324 
1325 	csio_dbg(hw, "Gathering all SCSI I/Os on lnode %p\n", ln);
1326 
1327 	sld.level = CSIO_LEV_LNODE;
1328 	sld.lnode = ln;
1329 	INIT_LIST_HEAD(&ln->cmpl_q);
1330 	csio_scsi_gather_active_ios(scm, &sld, &ln->cmpl_q);
1331 
1332 	/* No I/Os pending on this lnode  */
1333 	if (list_empty(&ln->cmpl_q))
1334 		return 0;
1335 
1336 	/* Wait until all active I/Os on this lnode are completed */
1337 	while (!list_empty(&ln->cmpl_q) && count--) {
1338 		spin_unlock_irq(&hw->lock);
1339 		msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1340 		spin_lock_irq(&hw->lock);
1341 	}
1342 
1343 	/* all I/Os completed */
1344 	if (list_empty(&ln->cmpl_q))
1345 		return 0;
1346 
1347 	csio_dbg(hw, "Some I/Os pending on ln:%p, aborting them..\n", ln);
1348 
1349 	/* I/Os are pending, abort them */
1350 	rv = csio_scsi_abort_io_q(scm, &ln->cmpl_q, 30000);
1351 	if (rv != 0) {
1352 		csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
1353 		csio_scsi_cleanup_io_q(scm, &ln->cmpl_q);
1354 	}
1355 
1356 	CSIO_DB_ASSERT(list_empty(&ln->cmpl_q));
1357 
1358 	return rv;
1359 }
1360 
1361 static ssize_t
1362 csio_show_hw_state(struct device *dev,
1363 		   struct device_attribute *attr, char *buf)
1364 {
1365 	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1366 	struct csio_hw *hw = csio_lnode_to_hw(ln);
1367 
1368 	if (csio_is_hw_ready(hw))
1369 		return sysfs_emit(buf, "ready\n");
1370 
1371 	return sysfs_emit(buf, "not ready\n");
1372 }
1373 
1374 /* Device reset */
1375 static ssize_t
1376 csio_device_reset(struct device *dev,
1377 		   struct device_attribute *attr, const char *buf, size_t count)
1378 {
1379 	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1380 	struct csio_hw *hw = csio_lnode_to_hw(ln);
1381 
1382 	if (*buf != '1')
1383 		return -EINVAL;
1384 
1385 	/* Delete NPIV lnodes */
1386 	csio_lnodes_exit(hw, 1);
1387 
1388 	/* Block upper IOs */
1389 	csio_lnodes_block_request(hw);
1390 
1391 	spin_lock_irq(&hw->lock);
1392 	csio_hw_reset(hw);
1393 	spin_unlock_irq(&hw->lock);
1394 
1395 	/* Unblock upper IOs */
1396 	csio_lnodes_unblock_request(hw);
1397 	return count;
1398 }
1399 
1400 /* disable port */
1401 static ssize_t
1402 csio_disable_port(struct device *dev,
1403 		   struct device_attribute *attr, const char *buf, size_t count)
1404 {
1405 	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1406 	struct csio_hw *hw = csio_lnode_to_hw(ln);
1407 	bool disable;
1408 
1409 	if (*buf == '1' || *buf == '0')
1410 		disable = (*buf == '1') ? true : false;
1411 	else
1412 		return -EINVAL;
1413 
1414 	/* Block upper IOs */
1415 	csio_lnodes_block_by_port(hw, ln->portid);
1416 
1417 	spin_lock_irq(&hw->lock);
1418 	csio_disable_lnodes(hw, ln->portid, disable);
1419 	spin_unlock_irq(&hw->lock);
1420 
1421 	/* Unblock upper IOs */
1422 	csio_lnodes_unblock_by_port(hw, ln->portid);
1423 	return count;
1424 }
1425 
1426 /* Show debug level */
1427 static ssize_t
1428 csio_show_dbg_level(struct device *dev,
1429 		   struct device_attribute *attr, char *buf)
1430 {
1431 	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1432 
1433 	return sysfs_emit(buf, "%x\n", ln->params.log_level);
1434 }
1435 
1436 /* Store debug level */
1437 static ssize_t
1438 csio_store_dbg_level(struct device *dev,
1439 		   struct device_attribute *attr, const char *buf, size_t count)
1440 {
1441 	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1442 	struct csio_hw *hw = csio_lnode_to_hw(ln);
1443 	uint32_t dbg_level = 0;
1444 
1445 	if (!isdigit(buf[0]))
1446 		return -EINVAL;
1447 
1448 	if (sscanf(buf, "%i", &dbg_level))
1449 		return -EINVAL;
1450 
1451 	ln->params.log_level = dbg_level;
1452 	hw->params.log_level = dbg_level;
1453 
1454 	return 0;
1455 }
1456 
1457 static DEVICE_ATTR(hw_state, S_IRUGO, csio_show_hw_state, NULL);
1458 static DEVICE_ATTR(device_reset, S_IWUSR, NULL, csio_device_reset);
1459 static DEVICE_ATTR(disable_port, S_IWUSR, NULL, csio_disable_port);
1460 static DEVICE_ATTR(dbg_level, S_IRUGO | S_IWUSR, csio_show_dbg_level,
1461 		  csio_store_dbg_level);
1462 
1463 static struct attribute *csio_fcoe_lport_attrs[] = {
1464 	&dev_attr_hw_state.attr,
1465 	&dev_attr_device_reset.attr,
1466 	&dev_attr_disable_port.attr,
1467 	&dev_attr_dbg_level.attr,
1468 	NULL,
1469 };
1470 
1471 ATTRIBUTE_GROUPS(csio_fcoe_lport);
1472 
1473 static ssize_t
1474 csio_show_num_reg_rnodes(struct device *dev,
1475 		     struct device_attribute *attr, char *buf)
1476 {
1477 	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1478 
1479 	return sysfs_emit(buf, "%d\n", ln->num_reg_rnodes);
1480 }
1481 
1482 static DEVICE_ATTR(num_reg_rnodes, S_IRUGO, csio_show_num_reg_rnodes, NULL);
1483 
1484 static struct attribute *csio_fcoe_vport_attrs[] = {
1485 	&dev_attr_num_reg_rnodes.attr,
1486 	&dev_attr_dbg_level.attr,
1487 	NULL,
1488 };
1489 
1490 ATTRIBUTE_GROUPS(csio_fcoe_vport);
1491 
1492 static inline uint32_t
1493 csio_scsi_copy_to_sgl(struct csio_hw *hw, struct csio_ioreq *req)
1494 {
1495 	struct scsi_cmnd *scmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1496 	struct scatterlist *sg;
1497 	uint32_t bytes_left;
1498 	uint32_t bytes_copy;
1499 	uint32_t buf_off = 0;
1500 	uint32_t start_off = 0;
1501 	uint32_t sg_off = 0;
1502 	void *sg_addr;
1503 	void *buf_addr;
1504 	struct csio_dma_buf *dma_buf;
1505 
1506 	bytes_left = scsi_bufflen(scmnd);
1507 	sg = scsi_sglist(scmnd);
1508 	dma_buf = (struct csio_dma_buf *)csio_list_next(&req->gen_list);
1509 
1510 	/* Copy data from driver buffer to SGs of SCSI CMD */
1511 	while (bytes_left > 0 && sg && dma_buf) {
1512 		if (buf_off >= dma_buf->len) {
1513 			buf_off = 0;
1514 			dma_buf = (struct csio_dma_buf *)
1515 					csio_list_next(dma_buf);
1516 			continue;
1517 		}
1518 
1519 		if (start_off >= sg->length) {
1520 			start_off -= sg->length;
1521 			sg = sg_next(sg);
1522 			continue;
1523 		}
1524 
1525 		buf_addr = dma_buf->vaddr + buf_off;
1526 		sg_off = sg->offset + start_off;
1527 		bytes_copy = min((dma_buf->len - buf_off),
1528 				sg->length - start_off);
1529 		bytes_copy = min((uint32_t)(PAGE_SIZE - (sg_off & ~PAGE_MASK)),
1530 				 bytes_copy);
1531 
1532 		sg_addr = kmap_atomic(sg_page(sg) + (sg_off >> PAGE_SHIFT));
1533 		if (!sg_addr) {
1534 			csio_err(hw, "failed to kmap sg:%p of ioreq:%p\n",
1535 				sg, req);
1536 			break;
1537 		}
1538 
1539 		csio_dbg(hw, "copy_to_sgl:sg_addr %p sg_off %d buf %p len %d\n",
1540 				sg_addr, sg_off, buf_addr, bytes_copy);
1541 		memcpy(sg_addr + (sg_off & ~PAGE_MASK), buf_addr, bytes_copy);
1542 		kunmap_atomic(sg_addr);
1543 
1544 		start_off +=  bytes_copy;
1545 		buf_off += bytes_copy;
1546 		bytes_left -= bytes_copy;
1547 	}
1548 
1549 	if (bytes_left > 0)
1550 		return DID_ERROR;
1551 	else
1552 		return DID_OK;
1553 }
1554 
1555 /*
1556  * csio_scsi_err_handler - SCSI error handler.
1557  * @hw: HW module.
1558  * @req: IO request.
1559  *
1560  */
1561 static inline void
1562 csio_scsi_err_handler(struct csio_hw *hw, struct csio_ioreq *req)
1563 {
1564 	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1565 	struct csio_scsim *scm = csio_hw_to_scsim(hw);
1566 	struct fcp_resp_with_ext *fcp_resp;
1567 	struct fcp_resp_rsp_info *rsp_info;
1568 	struct csio_dma_buf *dma_buf;
1569 	uint8_t flags, scsi_status = 0;
1570 	uint32_t host_status = DID_OK;
1571 	uint32_t rsp_len = 0, sns_len = 0;
1572 	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1573 
1574 
1575 	switch (req->wr_status) {
1576 	case FW_HOSTERROR:
1577 		if (unlikely(!csio_is_hw_ready(hw)))
1578 			return;
1579 
1580 		host_status = DID_ERROR;
1581 		CSIO_INC_STATS(scm, n_hosterror);
1582 
1583 		break;
1584 	case FW_SCSI_RSP_ERR:
1585 		dma_buf = &req->dma_buf;
1586 		fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
1587 		rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
1588 		flags = fcp_resp->resp.fr_flags;
1589 		scsi_status = fcp_resp->resp.fr_status;
1590 
1591 		if (flags & FCP_RSP_LEN_VAL) {
1592 			rsp_len = be32_to_cpu(fcp_resp->ext.fr_rsp_len);
1593 			if ((rsp_len != 0 && rsp_len != 4 && rsp_len != 8) ||
1594 				(rsp_info->rsp_code != FCP_TMF_CMPL)) {
1595 				host_status = DID_ERROR;
1596 				goto out;
1597 			}
1598 		}
1599 
1600 		if ((flags & FCP_SNS_LEN_VAL) && fcp_resp->ext.fr_sns_len) {
1601 			sns_len = be32_to_cpu(fcp_resp->ext.fr_sns_len);
1602 			if (sns_len > SCSI_SENSE_BUFFERSIZE)
1603 				sns_len = SCSI_SENSE_BUFFERSIZE;
1604 
1605 			memcpy(cmnd->sense_buffer,
1606 			       &rsp_info->_fr_resvd[0] + rsp_len, sns_len);
1607 			CSIO_INC_STATS(scm, n_autosense);
1608 		}
1609 
1610 		scsi_set_resid(cmnd, 0);
1611 
1612 		/* Under run */
1613 		if (flags & FCP_RESID_UNDER) {
1614 			scsi_set_resid(cmnd,
1615 				       be32_to_cpu(fcp_resp->ext.fr_resid));
1616 
1617 			if (!(flags & FCP_SNS_LEN_VAL) &&
1618 			    (scsi_status == SAM_STAT_GOOD) &&
1619 			    ((scsi_bufflen(cmnd) - scsi_get_resid(cmnd))
1620 							< cmnd->underflow))
1621 				host_status = DID_ERROR;
1622 		} else if (flags & FCP_RESID_OVER)
1623 			host_status = DID_ERROR;
1624 
1625 		CSIO_INC_STATS(scm, n_rsperror);
1626 		break;
1627 
1628 	case FW_SCSI_OVER_FLOW_ERR:
1629 		csio_warn(hw,
1630 			  "Over-flow error,cmnd:0x%x expected len:0x%x"
1631 			  " resid:0x%x\n", cmnd->cmnd[0],
1632 			  scsi_bufflen(cmnd), scsi_get_resid(cmnd));
1633 		host_status = DID_ERROR;
1634 		CSIO_INC_STATS(scm, n_ovflerror);
1635 		break;
1636 
1637 	case FW_SCSI_UNDER_FLOW_ERR:
1638 		csio_warn(hw,
1639 			  "Under-flow error,cmnd:0x%x expected"
1640 			  " len:0x%x resid:0x%x lun:0x%llx ssn:0x%x\n",
1641 			  cmnd->cmnd[0], scsi_bufflen(cmnd),
1642 			  scsi_get_resid(cmnd), cmnd->device->lun,
1643 			  rn->flowid);
1644 		host_status = DID_ERROR;
1645 		CSIO_INC_STATS(scm, n_unflerror);
1646 		break;
1647 
1648 	case FW_SCSI_ABORT_REQUESTED:
1649 	case FW_SCSI_ABORTED:
1650 	case FW_SCSI_CLOSE_REQUESTED:
1651 		csio_dbg(hw, "Req %p cmd:%p op:%x %s\n", req, cmnd,
1652 			     cmnd->cmnd[0],
1653 			    (req->wr_status == FW_SCSI_CLOSE_REQUESTED) ?
1654 			    "closed" : "aborted");
1655 		/*
1656 		 * csio_eh_abort_handler checks this value to
1657 		 * succeed or fail the abort request.
1658 		 */
1659 		host_status = DID_REQUEUE;
1660 		if (req->wr_status == FW_SCSI_CLOSE_REQUESTED)
1661 			CSIO_INC_STATS(scm, n_closed);
1662 		else
1663 			CSIO_INC_STATS(scm, n_aborted);
1664 		break;
1665 
1666 	case FW_SCSI_ABORT_TIMEDOUT:
1667 		/* FW timed out the abort itself */
1668 		csio_dbg(hw, "FW timed out abort req:%p cmnd:%p status:%x\n",
1669 			 req, cmnd, req->wr_status);
1670 		host_status = DID_ERROR;
1671 		CSIO_INC_STATS(scm, n_abrt_timedout);
1672 		break;
1673 
1674 	case FW_RDEV_NOT_READY:
1675 		/*
1676 		 * In firmware, a RDEV can get into this state
1677 		 * temporarily, before moving into dissapeared/lost
1678 		 * state. So, the driver should complete the request equivalent
1679 		 * to device-disappeared!
1680 		 */
1681 		CSIO_INC_STATS(scm, n_rdev_nr_error);
1682 		host_status = DID_ERROR;
1683 		break;
1684 
1685 	case FW_ERR_RDEV_LOST:
1686 		CSIO_INC_STATS(scm, n_rdev_lost_error);
1687 		host_status = DID_ERROR;
1688 		break;
1689 
1690 	case FW_ERR_RDEV_LOGO:
1691 		CSIO_INC_STATS(scm, n_rdev_logo_error);
1692 		host_status = DID_ERROR;
1693 		break;
1694 
1695 	case FW_ERR_RDEV_IMPL_LOGO:
1696 		host_status = DID_ERROR;
1697 		break;
1698 
1699 	case FW_ERR_LINK_DOWN:
1700 		CSIO_INC_STATS(scm, n_link_down_error);
1701 		host_status = DID_ERROR;
1702 		break;
1703 
1704 	case FW_FCOE_NO_XCHG:
1705 		CSIO_INC_STATS(scm, n_no_xchg_error);
1706 		host_status = DID_ERROR;
1707 		break;
1708 
1709 	default:
1710 		csio_err(hw, "Unknown SCSI FW WR status:%d req:%p cmnd:%p\n",
1711 			    req->wr_status, req, cmnd);
1712 		CSIO_DB_ASSERT(0);
1713 
1714 		CSIO_INC_STATS(scm, n_unknown_error);
1715 		host_status = DID_ERROR;
1716 		break;
1717 	}
1718 
1719 out:
1720 	if (req->nsge > 0) {
1721 		scsi_dma_unmap(cmnd);
1722 		if (req->dcopy && (host_status == DID_OK))
1723 			host_status = csio_scsi_copy_to_sgl(hw, req);
1724 	}
1725 
1726 	cmnd->result = (((host_status) << 16) | scsi_status);
1727 	scsi_done(cmnd);
1728 
1729 	/* Wake up waiting threads */
1730 	csio_scsi_cmnd(req) = NULL;
1731 	complete(&req->cmplobj);
1732 }
1733 
1734 /*
1735  * csio_scsi_cbfn - SCSI callback function.
1736  * @hw: HW module.
1737  * @req: IO request.
1738  *
1739  */
1740 static void
1741 csio_scsi_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
1742 {
1743 	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1744 	uint8_t scsi_status = SAM_STAT_GOOD;
1745 	uint32_t host_status = DID_OK;
1746 
1747 	if (likely(req->wr_status == FW_SUCCESS)) {
1748 		if (req->nsge > 0) {
1749 			scsi_dma_unmap(cmnd);
1750 			if (req->dcopy)
1751 				host_status = csio_scsi_copy_to_sgl(hw, req);
1752 		}
1753 
1754 		cmnd->result = (((host_status) << 16) | scsi_status);
1755 		scsi_done(cmnd);
1756 		csio_scsi_cmnd(req) = NULL;
1757 		CSIO_INC_STATS(csio_hw_to_scsim(hw), n_tot_success);
1758 	} else {
1759 		/* Error handling */
1760 		csio_scsi_err_handler(hw, req);
1761 	}
1762 }
1763 
1764 /**
1765  * csio_queuecommand - Entry point to kickstart an I/O request.
1766  * @host:	The scsi_host pointer.
1767  * @cmnd:	The I/O request from ML.
1768  *
1769  * This routine does the following:
1770  *	- Checks for HW and Rnode module readiness.
1771  *	- Gets a free ioreq structure (which is already initialized
1772  *	  to uninit during its allocation).
1773  *	- Maps SG elements.
1774  *	- Initializes ioreq members.
1775  *	- Kicks off the SCSI state machine for this IO.
1776  *	- Returns busy status on error.
1777  */
1778 static int
1779 csio_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmnd)
1780 {
1781 	struct csio_lnode *ln = shost_priv(host);
1782 	struct csio_hw *hw = csio_lnode_to_hw(ln);
1783 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1784 	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1785 	struct csio_ioreq *ioreq = NULL;
1786 	unsigned long flags;
1787 	int nsge = 0;
1788 	int rv = SCSI_MLQUEUE_HOST_BUSY, nr;
1789 	int retval;
1790 	struct csio_scsi_qset *sqset;
1791 	struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
1792 
1793 	sqset = &hw->sqset[ln->portid][blk_mq_rq_cpu(scsi_cmd_to_rq(cmnd))];
1794 
1795 	nr = fc_remote_port_chkready(rport);
1796 	if (nr) {
1797 		cmnd->result = nr;
1798 		CSIO_INC_STATS(scsim, n_rn_nr_error);
1799 		goto err_done;
1800 	}
1801 
1802 	if (unlikely(!csio_is_hw_ready(hw))) {
1803 		cmnd->result = (DID_REQUEUE << 16);
1804 		CSIO_INC_STATS(scsim, n_hw_nr_error);
1805 		goto err_done;
1806 	}
1807 
1808 	/* Get req->nsge, if there are SG elements to be mapped  */
1809 	nsge = scsi_dma_map(cmnd);
1810 	if (unlikely(nsge < 0)) {
1811 		CSIO_INC_STATS(scsim, n_dmamap_error);
1812 		goto err;
1813 	}
1814 
1815 	/* Do we support so many mappings? */
1816 	if (unlikely(nsge > scsim->max_sge)) {
1817 		csio_warn(hw,
1818 			  "More SGEs than can be supported."
1819 			  " SGEs: %d, Max SGEs: %d\n", nsge, scsim->max_sge);
1820 		CSIO_INC_STATS(scsim, n_unsupp_sge_error);
1821 		goto err_dma_unmap;
1822 	}
1823 
1824 	/* Get a free ioreq structure - SM is already set to uninit */
1825 	ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
1826 	if (!ioreq) {
1827 		csio_err(hw, "Out of I/O request elements. Active #:%d\n",
1828 			 scsim->stats.n_active);
1829 		CSIO_INC_STATS(scsim, n_no_req_error);
1830 		goto err_dma_unmap;
1831 	}
1832 
1833 	ioreq->nsge		= nsge;
1834 	ioreq->lnode		= ln;
1835 	ioreq->rnode		= rn;
1836 	ioreq->iq_idx		= sqset->iq_idx;
1837 	ioreq->eq_idx		= sqset->eq_idx;
1838 	ioreq->wr_status	= 0;
1839 	ioreq->drv_status	= 0;
1840 	csio_scsi_cmnd(ioreq)	= (void *)cmnd;
1841 	ioreq->tmo		= 0;
1842 	ioreq->datadir		= cmnd->sc_data_direction;
1843 
1844 	if (cmnd->sc_data_direction == DMA_TO_DEVICE) {
1845 		CSIO_INC_STATS(ln, n_output_requests);
1846 		ln->stats.n_output_bytes += scsi_bufflen(cmnd);
1847 	} else if (cmnd->sc_data_direction == DMA_FROM_DEVICE) {
1848 		CSIO_INC_STATS(ln, n_input_requests);
1849 		ln->stats.n_input_bytes += scsi_bufflen(cmnd);
1850 	} else
1851 		CSIO_INC_STATS(ln, n_control_requests);
1852 
1853 	/* Set cbfn */
1854 	ioreq->io_cbfn = csio_scsi_cbfn;
1855 
1856 	/* Needed during abort */
1857 	cmnd->host_scribble = (unsigned char *)ioreq;
1858 	csio_priv(cmnd)->fc_tm_flags = 0;
1859 
1860 	/* Kick off SCSI IO SM on the ioreq */
1861 	spin_lock_irqsave(&hw->lock, flags);
1862 	retval = csio_scsi_start_io(ioreq);
1863 	spin_unlock_irqrestore(&hw->lock, flags);
1864 
1865 	if (retval != 0) {
1866 		csio_err(hw, "ioreq: %p couldn't be started, status:%d\n",
1867 			 ioreq, retval);
1868 		CSIO_INC_STATS(scsim, n_busy_error);
1869 		goto err_put_req;
1870 	}
1871 
1872 	return 0;
1873 
1874 err_put_req:
1875 	csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
1876 err_dma_unmap:
1877 	if (nsge > 0)
1878 		scsi_dma_unmap(cmnd);
1879 err:
1880 	return rv;
1881 
1882 err_done:
1883 	scsi_done(cmnd);
1884 	return 0;
1885 }
1886 
1887 static int
1888 csio_do_abrt_cls(struct csio_hw *hw, struct csio_ioreq *ioreq, bool abort)
1889 {
1890 	int rv;
1891 	int cpu = smp_processor_id();
1892 	struct csio_lnode *ln = ioreq->lnode;
1893 	struct csio_scsi_qset *sqset = &hw->sqset[ln->portid][cpu];
1894 
1895 	ioreq->tmo = CSIO_SCSI_ABRT_TMO_MS;
1896 	/*
1897 	 * Use current processor queue for posting the abort/close, but retain
1898 	 * the ingress queue ID of the original I/O being aborted/closed - we
1899 	 * need the abort/close completion to be received on the same queue
1900 	 * as the original I/O.
1901 	 */
1902 	ioreq->eq_idx = sqset->eq_idx;
1903 
1904 	if (abort == SCSI_ABORT)
1905 		rv = csio_scsi_abort(ioreq);
1906 	else
1907 		rv = csio_scsi_close(ioreq);
1908 
1909 	return rv;
1910 }
1911 
1912 static int
1913 csio_eh_abort_handler(struct scsi_cmnd *cmnd)
1914 {
1915 	struct csio_ioreq *ioreq;
1916 	struct csio_lnode *ln = shost_priv(cmnd->device->host);
1917 	struct csio_hw *hw = csio_lnode_to_hw(ln);
1918 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1919 	int ready = 0, ret;
1920 	unsigned long tmo = 0;
1921 	int rv;
1922 	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1923 
1924 	ret = fc_block_scsi_eh(cmnd);
1925 	if (ret)
1926 		return ret;
1927 
1928 	ioreq = (struct csio_ioreq *)cmnd->host_scribble;
1929 	if (!ioreq)
1930 		return SUCCESS;
1931 
1932 	if (!rn)
1933 		return FAILED;
1934 
1935 	csio_dbg(hw,
1936 		 "Request to abort ioreq:%p cmd:%p cdb:%08llx"
1937 		 " ssni:0x%x lun:%llu iq:0x%x\n",
1938 		ioreq, cmnd, *((uint64_t *)cmnd->cmnd), rn->flowid,
1939 		cmnd->device->lun, csio_q_physiqid(hw, ioreq->iq_idx));
1940 
1941 	if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) != cmnd) {
1942 		CSIO_INC_STATS(scsim, n_abrt_race_comp);
1943 		return SUCCESS;
1944 	}
1945 
1946 	ready = csio_is_lnode_ready(ln);
1947 	tmo = CSIO_SCSI_ABRT_TMO_MS;
1948 
1949 	reinit_completion(&ioreq->cmplobj);
1950 	spin_lock_irq(&hw->lock);
1951 	rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
1952 	spin_unlock_irq(&hw->lock);
1953 
1954 	if (rv != 0) {
1955 		if (rv == -EINVAL) {
1956 			/* Return success, if abort/close request issued on
1957 			 * already completed IO
1958 			 */
1959 			return SUCCESS;
1960 		}
1961 		if (ready)
1962 			CSIO_INC_STATS(scsim, n_abrt_busy_error);
1963 		else
1964 			CSIO_INC_STATS(scsim, n_cls_busy_error);
1965 
1966 		goto inval_scmnd;
1967 	}
1968 
1969 	wait_for_completion_timeout(&ioreq->cmplobj, msecs_to_jiffies(tmo));
1970 
1971 	/* FW didnt respond to abort within our timeout */
1972 	if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
1973 
1974 		csio_err(hw, "Abort timed out -- req: %p\n", ioreq);
1975 		CSIO_INC_STATS(scsim, n_abrt_timedout);
1976 
1977 inval_scmnd:
1978 		if (ioreq->nsge > 0)
1979 			scsi_dma_unmap(cmnd);
1980 
1981 		spin_lock_irq(&hw->lock);
1982 		csio_scsi_cmnd(ioreq) = NULL;
1983 		spin_unlock_irq(&hw->lock);
1984 
1985 		cmnd->result = (DID_ERROR << 16);
1986 		scsi_done(cmnd);
1987 
1988 		return FAILED;
1989 	}
1990 
1991 	/* FW successfully aborted the request */
1992 	if (host_byte(cmnd->result) == DID_REQUEUE) {
1993 		csio_info(hw,
1994 			"Aborted SCSI command to (%d:%llu) tag %u\n",
1995 			cmnd->device->id, cmnd->device->lun,
1996 			scsi_cmd_to_rq(cmnd)->tag);
1997 		return SUCCESS;
1998 	} else {
1999 		csio_info(hw,
2000 			"Failed to abort SCSI command, (%d:%llu) tag %u\n",
2001 			cmnd->device->id, cmnd->device->lun,
2002 			scsi_cmd_to_rq(cmnd)->tag);
2003 		return FAILED;
2004 	}
2005 }
2006 
2007 /*
2008  * csio_tm_cbfn - TM callback function.
2009  * @hw: HW module.
2010  * @req: IO request.
2011  *
2012  * Cache the result in 'cmnd', since ioreq will be freed soon
2013  * after we return from here, and the waiting thread shouldnt trust
2014  * the ioreq contents.
2015  */
2016 static void
2017 csio_tm_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
2018 {
2019 	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
2020 	struct csio_dma_buf *dma_buf;
2021 	uint8_t flags = 0;
2022 	struct fcp_resp_with_ext *fcp_resp;
2023 	struct fcp_resp_rsp_info *rsp_info;
2024 
2025 	csio_dbg(hw, "req: %p in csio_tm_cbfn status: %d\n",
2026 		      req, req->wr_status);
2027 
2028 	/* Cache FW return status */
2029 	csio_priv(cmnd)->wr_status = req->wr_status;
2030 
2031 	/* Special handling based on FCP response */
2032 
2033 	/*
2034 	 * FW returns us this error, if flags were set. FCP4 says
2035 	 * FCP_RSP_LEN_VAL in flags shall be set for TM completions.
2036 	 * So if a target were to set this bit, we expect that the
2037 	 * rsp_code is set to FCP_TMF_CMPL for a successful TM
2038 	 * completion. Any other rsp_code means TM operation failed.
2039 	 * If a target were to just ignore setting flags, we treat
2040 	 * the TM operation as success, and FW returns FW_SUCCESS.
2041 	 */
2042 	if (req->wr_status == FW_SCSI_RSP_ERR) {
2043 		dma_buf = &req->dma_buf;
2044 		fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
2045 		rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
2046 
2047 		flags = fcp_resp->resp.fr_flags;
2048 
2049 		/* Modify return status if flags indicate success */
2050 		if (flags & FCP_RSP_LEN_VAL)
2051 			if (rsp_info->rsp_code == FCP_TMF_CMPL)
2052 				csio_priv(cmnd)->wr_status = FW_SUCCESS;
2053 
2054 		csio_dbg(hw, "TM FCP rsp code: %d\n", rsp_info->rsp_code);
2055 	}
2056 
2057 	/* Wake up the TM handler thread */
2058 	csio_scsi_cmnd(req) = NULL;
2059 }
2060 
2061 static int
2062 csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd)
2063 {
2064 	struct csio_lnode *ln = shost_priv(cmnd->device->host);
2065 	struct csio_hw *hw = csio_lnode_to_hw(ln);
2066 	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
2067 	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
2068 	struct csio_ioreq *ioreq = NULL;
2069 	struct csio_scsi_qset *sqset;
2070 	unsigned long flags;
2071 	int retval;
2072 	int count, ret;
2073 	LIST_HEAD(local_q);
2074 	struct csio_scsi_level_data sld;
2075 
2076 	if (!rn)
2077 		goto fail;
2078 
2079 	csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n",
2080 		      cmnd->device->lun, rn->flowid, rn->scsi_id);
2081 
2082 	if (!csio_is_lnode_ready(ln)) {
2083 		csio_err(hw,
2084 			 "LUN reset cannot be issued on non-ready"
2085 			 " local node vnpi:0x%x (LUN:%llu)\n",
2086 			 ln->vnp_flowid, cmnd->device->lun);
2087 		goto fail;
2088 	}
2089 
2090 	/* Lnode is ready, now wait on rport node readiness */
2091 	ret = fc_block_scsi_eh(cmnd);
2092 	if (ret)
2093 		return ret;
2094 
2095 	/*
2096 	 * If we have blocked in the previous call, at this point, either the
2097 	 * remote node has come back online, or device loss timer has fired
2098 	 * and the remote node is destroyed. Allow the LUN reset only for
2099 	 * the former case, since LUN reset is a TMF I/O on the wire, and we
2100 	 * need a valid session to issue it.
2101 	 */
2102 	if (fc_remote_port_chkready(rn->rport)) {
2103 		csio_err(hw,
2104 			 "LUN reset cannot be issued on non-ready"
2105 			 " remote node ssni:0x%x (LUN:%llu)\n",
2106 			 rn->flowid, cmnd->device->lun);
2107 		goto fail;
2108 	}
2109 
2110 	/* Get a free ioreq structure - SM is already set to uninit */
2111 	ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
2112 
2113 	if (!ioreq) {
2114 		csio_err(hw, "Out of IO request elements. Active # :%d\n",
2115 			 scsim->stats.n_active);
2116 		goto fail;
2117 	}
2118 
2119 	sqset			= &hw->sqset[ln->portid][smp_processor_id()];
2120 	ioreq->nsge		= 0;
2121 	ioreq->lnode		= ln;
2122 	ioreq->rnode		= rn;
2123 	ioreq->iq_idx		= sqset->iq_idx;
2124 	ioreq->eq_idx		= sqset->eq_idx;
2125 
2126 	csio_scsi_cmnd(ioreq)	= cmnd;
2127 	cmnd->host_scribble	= (unsigned char *)ioreq;
2128 	csio_priv(cmnd)->wr_status = 0;
2129 
2130 	csio_priv(cmnd)->fc_tm_flags = FCP_TMF_LUN_RESET;
2131 	ioreq->tmo		= CSIO_SCSI_LUNRST_TMO_MS / 1000;
2132 
2133 	/*
2134 	 * FW times the LUN reset for ioreq->tmo, so we got to wait a little
2135 	 * longer (10s for now) than that to allow FW to return the timed
2136 	 * out command.
2137 	 */
2138 	count = DIV_ROUND_UP((ioreq->tmo + 10) * 1000, CSIO_SCSI_TM_POLL_MS);
2139 
2140 	/* Set cbfn */
2141 	ioreq->io_cbfn = csio_tm_cbfn;
2142 
2143 	/* Save of the ioreq info for later use */
2144 	sld.level = CSIO_LEV_LUN;
2145 	sld.lnode = ioreq->lnode;
2146 	sld.rnode = ioreq->rnode;
2147 	sld.oslun = cmnd->device->lun;
2148 
2149 	spin_lock_irqsave(&hw->lock, flags);
2150 	/* Kick off TM SM on the ioreq */
2151 	retval = csio_scsi_start_tm(ioreq);
2152 	spin_unlock_irqrestore(&hw->lock, flags);
2153 
2154 	if (retval != 0) {
2155 		csio_err(hw, "Failed to issue LUN reset, req:%p, status:%d\n",
2156 			    ioreq, retval);
2157 		goto fail_ret_ioreq;
2158 	}
2159 
2160 	csio_dbg(hw, "Waiting max %d secs for LUN reset completion\n",
2161 		    count * (CSIO_SCSI_TM_POLL_MS / 1000));
2162 	/* Wait for completion */
2163 	while ((((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd)
2164 								&& count--)
2165 		msleep(CSIO_SCSI_TM_POLL_MS);
2166 
2167 	/* LUN reset timed-out */
2168 	if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
2169 		csio_err(hw, "LUN reset (%d:%llu) timed out\n",
2170 			 cmnd->device->id, cmnd->device->lun);
2171 
2172 		spin_lock_irq(&hw->lock);
2173 		csio_scsi_drvcleanup(ioreq);
2174 		list_del_init(&ioreq->sm.sm_list);
2175 		spin_unlock_irq(&hw->lock);
2176 
2177 		goto fail_ret_ioreq;
2178 	}
2179 
2180 	/* LUN reset returned, check cached status */
2181 	if (csio_priv(cmnd)->wr_status != FW_SUCCESS) {
2182 		csio_err(hw, "LUN reset failed (%d:%llu), status: %d\n",
2183 			 cmnd->device->id, cmnd->device->lun,
2184 			 csio_priv(cmnd)->wr_status);
2185 		goto fail;
2186 	}
2187 
2188 	/* LUN reset succeeded, Start aborting affected I/Os */
2189 	/*
2190 	 * Since the host guarantees during LUN reset that there
2191 	 * will not be any more I/Os to that LUN, until the LUN reset
2192 	 * completes, we gather pending I/Os after the LUN reset.
2193 	 */
2194 	spin_lock_irq(&hw->lock);
2195 	csio_scsi_gather_active_ios(scsim, &sld, &local_q);
2196 
2197 	retval = csio_scsi_abort_io_q(scsim, &local_q, 30000);
2198 	spin_unlock_irq(&hw->lock);
2199 
2200 	/* Aborts may have timed out */
2201 	if (retval != 0) {
2202 		csio_err(hw,
2203 			 "Attempt to abort I/Os during LUN reset of %llu"
2204 			 " returned %d\n", cmnd->device->lun, retval);
2205 		/* Return I/Os back to active_q */
2206 		spin_lock_irq(&hw->lock);
2207 		list_splice_tail_init(&local_q, &scsim->active_q);
2208 		spin_unlock_irq(&hw->lock);
2209 		goto fail;
2210 	}
2211 
2212 	CSIO_INC_STATS(rn, n_lun_rst);
2213 
2214 	csio_info(hw, "LUN reset occurred (%d:%llu)\n",
2215 		  cmnd->device->id, cmnd->device->lun);
2216 
2217 	return SUCCESS;
2218 
2219 fail_ret_ioreq:
2220 	csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
2221 fail:
2222 	CSIO_INC_STATS(rn, n_lun_rst_fail);
2223 	return FAILED;
2224 }
2225 
2226 static int
2227 csio_slave_alloc(struct scsi_device *sdev)
2228 {
2229 	struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
2230 
2231 	if (!rport || fc_remote_port_chkready(rport))
2232 		return -ENXIO;
2233 
2234 	sdev->hostdata = *((struct csio_lnode **)(rport->dd_data));
2235 
2236 	return 0;
2237 }
2238 
2239 static int
2240 csio_slave_configure(struct scsi_device *sdev)
2241 {
2242 	scsi_change_queue_depth(sdev, csio_lun_qdepth);
2243 	return 0;
2244 }
2245 
2246 static void
2247 csio_slave_destroy(struct scsi_device *sdev)
2248 {
2249 	sdev->hostdata = NULL;
2250 }
2251 
2252 static int
2253 csio_scan_finished(struct Scsi_Host *shost, unsigned long time)
2254 {
2255 	struct csio_lnode *ln = shost_priv(shost);
2256 	int rv = 1;
2257 
2258 	spin_lock_irq(shost->host_lock);
2259 	if (!ln->hwp || csio_list_deleted(&ln->sm.sm_list))
2260 		goto out;
2261 
2262 	rv = csio_scan_done(ln, jiffies, time, csio_max_scan_tmo * HZ,
2263 			    csio_delta_scan_tmo * HZ);
2264 out:
2265 	spin_unlock_irq(shost->host_lock);
2266 
2267 	return rv;
2268 }
2269 
2270 struct scsi_host_template csio_fcoe_shost_template = {
2271 	.module			= THIS_MODULE,
2272 	.name			= CSIO_DRV_DESC,
2273 	.proc_name		= KBUILD_MODNAME,
2274 	.queuecommand		= csio_queuecommand,
2275 	.cmd_size		= sizeof(struct csio_cmd_priv),
2276 	.eh_timed_out		= fc_eh_timed_out,
2277 	.eh_abort_handler	= csio_eh_abort_handler,
2278 	.eh_device_reset_handler = csio_eh_lun_reset_handler,
2279 	.slave_alloc		= csio_slave_alloc,
2280 	.slave_configure	= csio_slave_configure,
2281 	.slave_destroy		= csio_slave_destroy,
2282 	.scan_finished		= csio_scan_finished,
2283 	.this_id		= -1,
2284 	.sg_tablesize		= CSIO_SCSI_MAX_SGE,
2285 	.cmd_per_lun		= CSIO_MAX_CMD_PER_LUN,
2286 	.shost_groups		= csio_fcoe_lport_groups,
2287 	.max_sectors		= CSIO_MAX_SECTOR_SIZE,
2288 };
2289 
2290 struct scsi_host_template csio_fcoe_shost_vport_template = {
2291 	.module			= THIS_MODULE,
2292 	.name			= CSIO_DRV_DESC,
2293 	.proc_name		= KBUILD_MODNAME,
2294 	.queuecommand		= csio_queuecommand,
2295 	.eh_timed_out		= fc_eh_timed_out,
2296 	.eh_abort_handler	= csio_eh_abort_handler,
2297 	.eh_device_reset_handler = csio_eh_lun_reset_handler,
2298 	.slave_alloc		= csio_slave_alloc,
2299 	.slave_configure	= csio_slave_configure,
2300 	.slave_destroy		= csio_slave_destroy,
2301 	.scan_finished		= csio_scan_finished,
2302 	.this_id		= -1,
2303 	.sg_tablesize		= CSIO_SCSI_MAX_SGE,
2304 	.cmd_per_lun		= CSIO_MAX_CMD_PER_LUN,
2305 	.shost_groups		= csio_fcoe_vport_groups,
2306 	.max_sectors		= CSIO_MAX_SECTOR_SIZE,
2307 };
2308 
2309 /*
2310  * csio_scsi_alloc_ddp_bufs - Allocate buffers for DDP of unaligned SGLs.
2311  * @scm: SCSI Module
2312  * @hw: HW device.
2313  * @buf_size: buffer size
2314  * @num_buf : Number of buffers.
2315  *
2316  * This routine allocates DMA buffers required for SCSI Data xfer, if
2317  * each SGL buffer for a SCSI Read request posted by SCSI midlayer are
2318  * not virtually contiguous.
2319  */
2320 static int
2321 csio_scsi_alloc_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw,
2322 			 int buf_size, int num_buf)
2323 {
2324 	int n = 0;
2325 	struct list_head *tmp;
2326 	struct csio_dma_buf *ddp_desc = NULL;
2327 	uint32_t unit_size = 0;
2328 
2329 	if (!num_buf)
2330 		return 0;
2331 
2332 	if (!buf_size)
2333 		return -EINVAL;
2334 
2335 	INIT_LIST_HEAD(&scm->ddp_freelist);
2336 
2337 	/* Align buf size to page size */
2338 	buf_size = (buf_size + PAGE_SIZE - 1) & PAGE_MASK;
2339 	/* Initialize dma descriptors */
2340 	for (n = 0; n < num_buf; n++) {
2341 		/* Set unit size to request size */
2342 		unit_size = buf_size;
2343 		ddp_desc = kzalloc(sizeof(struct csio_dma_buf), GFP_KERNEL);
2344 		if (!ddp_desc) {
2345 			csio_err(hw,
2346 				 "Failed to allocate ddp descriptors,"
2347 				 " Num allocated = %d.\n",
2348 				 scm->stats.n_free_ddp);
2349 			goto no_mem;
2350 		}
2351 
2352 		/* Allocate Dma buffers for DDP */
2353 		ddp_desc->vaddr = dma_alloc_coherent(&hw->pdev->dev, unit_size,
2354 				&ddp_desc->paddr, GFP_KERNEL);
2355 		if (!ddp_desc->vaddr) {
2356 			csio_err(hw,
2357 				 "SCSI response DMA buffer (ddp) allocation"
2358 				 " failed!\n");
2359 			kfree(ddp_desc);
2360 			goto no_mem;
2361 		}
2362 
2363 		ddp_desc->len = unit_size;
2364 
2365 		/* Added it to scsi ddp freelist */
2366 		list_add_tail(&ddp_desc->list, &scm->ddp_freelist);
2367 		CSIO_INC_STATS(scm, n_free_ddp);
2368 	}
2369 
2370 	return 0;
2371 no_mem:
2372 	/* release dma descs back to freelist and free dma memory */
2373 	list_for_each(tmp, &scm->ddp_freelist) {
2374 		ddp_desc = (struct csio_dma_buf *) tmp;
2375 		tmp = csio_list_prev(tmp);
2376 		dma_free_coherent(&hw->pdev->dev, ddp_desc->len,
2377 				  ddp_desc->vaddr, ddp_desc->paddr);
2378 		list_del_init(&ddp_desc->list);
2379 		kfree(ddp_desc);
2380 	}
2381 	scm->stats.n_free_ddp = 0;
2382 
2383 	return -ENOMEM;
2384 }
2385 
2386 /*
2387  * csio_scsi_free_ddp_bufs - free DDP buffers of unaligned SGLs.
2388  * @scm: SCSI Module
2389  * @hw: HW device.
2390  *
2391  * This routine frees ddp buffers.
2392  */
2393 static void
2394 csio_scsi_free_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw)
2395 {
2396 	struct list_head *tmp;
2397 	struct csio_dma_buf *ddp_desc;
2398 
2399 	/* release dma descs back to freelist and free dma memory */
2400 	list_for_each(tmp, &scm->ddp_freelist) {
2401 		ddp_desc = (struct csio_dma_buf *) tmp;
2402 		tmp = csio_list_prev(tmp);
2403 		dma_free_coherent(&hw->pdev->dev, ddp_desc->len,
2404 				  ddp_desc->vaddr, ddp_desc->paddr);
2405 		list_del_init(&ddp_desc->list);
2406 		kfree(ddp_desc);
2407 	}
2408 	scm->stats.n_free_ddp = 0;
2409 }
2410 
2411 /**
2412  * csio_scsim_init - Initialize SCSI Module
2413  * @scm:	SCSI Module
2414  * @hw:		HW module
2415  *
2416  */
2417 int
2418 csio_scsim_init(struct csio_scsim *scm, struct csio_hw *hw)
2419 {
2420 	int i;
2421 	struct csio_ioreq *ioreq;
2422 	struct csio_dma_buf *dma_buf;
2423 
2424 	INIT_LIST_HEAD(&scm->active_q);
2425 	scm->hw = hw;
2426 
2427 	scm->proto_cmd_len = sizeof(struct fcp_cmnd);
2428 	scm->proto_rsp_len = CSIO_SCSI_RSP_LEN;
2429 	scm->max_sge = CSIO_SCSI_MAX_SGE;
2430 
2431 	spin_lock_init(&scm->freelist_lock);
2432 
2433 	/* Pre-allocate ioreqs and initialize them */
2434 	INIT_LIST_HEAD(&scm->ioreq_freelist);
2435 	for (i = 0; i < csio_scsi_ioreqs; i++) {
2436 
2437 		ioreq = kzalloc(sizeof(struct csio_ioreq), GFP_KERNEL);
2438 		if (!ioreq) {
2439 			csio_err(hw,
2440 				 "I/O request element allocation failed, "
2441 				 " Num allocated = %d.\n",
2442 				 scm->stats.n_free_ioreq);
2443 
2444 			goto free_ioreq;
2445 		}
2446 
2447 		/* Allocate Dma buffers for Response Payload */
2448 		dma_buf = &ioreq->dma_buf;
2449 		dma_buf->vaddr = dma_pool_alloc(hw->scsi_dma_pool, GFP_KERNEL,
2450 						&dma_buf->paddr);
2451 		if (!dma_buf->vaddr) {
2452 			csio_err(hw,
2453 				 "SCSI response DMA buffer allocation"
2454 				 " failed!\n");
2455 			kfree(ioreq);
2456 			goto free_ioreq;
2457 		}
2458 
2459 		dma_buf->len = scm->proto_rsp_len;
2460 
2461 		/* Set state to uninit */
2462 		csio_init_state(&ioreq->sm, csio_scsis_uninit);
2463 		INIT_LIST_HEAD(&ioreq->gen_list);
2464 		init_completion(&ioreq->cmplobj);
2465 
2466 		list_add_tail(&ioreq->sm.sm_list, &scm->ioreq_freelist);
2467 		CSIO_INC_STATS(scm, n_free_ioreq);
2468 	}
2469 
2470 	if (csio_scsi_alloc_ddp_bufs(scm, hw, PAGE_SIZE, csio_ddp_descs))
2471 		goto free_ioreq;
2472 
2473 	return 0;
2474 
2475 free_ioreq:
2476 	/*
2477 	 * Free up existing allocations, since an error
2478 	 * from here means we are returning for good
2479 	 */
2480 	while (!list_empty(&scm->ioreq_freelist)) {
2481 		struct csio_sm *tmp;
2482 
2483 		tmp = list_first_entry(&scm->ioreq_freelist,
2484 				       struct csio_sm, sm_list);
2485 		list_del_init(&tmp->sm_list);
2486 		ioreq = (struct csio_ioreq *)tmp;
2487 
2488 		dma_buf = &ioreq->dma_buf;
2489 		dma_pool_free(hw->scsi_dma_pool, dma_buf->vaddr,
2490 			      dma_buf->paddr);
2491 
2492 		kfree(ioreq);
2493 	}
2494 
2495 	scm->stats.n_free_ioreq = 0;
2496 
2497 	return -ENOMEM;
2498 }
2499 
2500 /**
2501  * csio_scsim_exit: Uninitialize SCSI Module
2502  * @scm: SCSI Module
2503  *
2504  */
2505 void
2506 csio_scsim_exit(struct csio_scsim *scm)
2507 {
2508 	struct csio_ioreq *ioreq;
2509 	struct csio_dma_buf *dma_buf;
2510 
2511 	while (!list_empty(&scm->ioreq_freelist)) {
2512 		struct csio_sm *tmp;
2513 
2514 		tmp = list_first_entry(&scm->ioreq_freelist,
2515 				       struct csio_sm, sm_list);
2516 		list_del_init(&tmp->sm_list);
2517 		ioreq = (struct csio_ioreq *)tmp;
2518 
2519 		dma_buf = &ioreq->dma_buf;
2520 		dma_pool_free(scm->hw->scsi_dma_pool, dma_buf->vaddr,
2521 			      dma_buf->paddr);
2522 
2523 		kfree(ioreq);
2524 	}
2525 
2526 	scm->stats.n_free_ioreq = 0;
2527 
2528 	csio_scsi_free_ddp_bufs(scm, scm->hw);
2529 }
2530