xref: /openbmc/linux/drivers/scsi/lpfc/lpfc_nvmet.c (revision 5d0e4d78)
1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channsel Host Bus Adapters.                               *
4  * Copyright (C) 2017 Broadcom. All Rights Reserved. The term      *
5  * “Broadcom” refers to Broadcom Limited and/or its subsidiaries.  *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30 
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38 
39 #include <../drivers/nvme/host/nvme.h>
40 #include <linux/nvme-fc-driver.h>
41 
42 #include "lpfc_version.h"
43 #include "lpfc_hw4.h"
44 #include "lpfc_hw.h"
45 #include "lpfc_sli.h"
46 #include "lpfc_sli4.h"
47 #include "lpfc_nl.h"
48 #include "lpfc_disc.h"
49 #include "lpfc.h"
50 #include "lpfc_scsi.h"
51 #include "lpfc_nvme.h"
52 #include "lpfc_nvmet.h"
53 #include "lpfc_logmsg.h"
54 #include "lpfc_crtn.h"
55 #include "lpfc_vport.h"
56 #include "lpfc_debugfs.h"
57 
58 static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
59 						 struct lpfc_nvmet_rcv_ctx *,
60 						 dma_addr_t rspbuf,
61 						 uint16_t rspsize);
62 static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
63 						  struct lpfc_nvmet_rcv_ctx *);
64 static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
65 					  struct lpfc_nvmet_rcv_ctx *,
66 					  uint32_t, uint16_t);
67 static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
68 					    struct lpfc_nvmet_rcv_ctx *,
69 					    uint32_t, uint16_t);
70 static int lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *,
71 					   struct lpfc_nvmet_rcv_ctx *,
72 					   uint32_t, uint16_t);
73 
74 void
75 lpfc_nvmet_defer_release(struct lpfc_hba *phba, struct lpfc_nvmet_rcv_ctx *ctxp)
76 {
77 	unsigned long iflag;
78 
79 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
80 			"6313 NVMET Defer ctx release xri x%x flg x%x\n",
81 			ctxp->oxid, ctxp->flag);
82 
83 	spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
84 	if (ctxp->flag & LPFC_NVMET_CTX_RLS) {
85 		spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock,
86 				       iflag);
87 		return;
88 	}
89 	ctxp->flag |= LPFC_NVMET_CTX_RLS;
90 	list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
91 	spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, iflag);
92 }
93 
94 /**
95  * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
96  * @phba: Pointer to HBA context object.
97  * @cmdwqe: Pointer to driver command WQE object.
98  * @wcqe: Pointer to driver response CQE object.
99  *
100  * The function is called from SLI ring event handler with no
101  * lock held. This function is the completion handler for NVME LS commands
102  * The function frees memory resources used for the NVME commands.
103  **/
104 static void
105 lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
106 			  struct lpfc_wcqe_complete *wcqe)
107 {
108 	struct lpfc_nvmet_tgtport *tgtp;
109 	struct nvmefc_tgt_ls_req *rsp;
110 	struct lpfc_nvmet_rcv_ctx *ctxp;
111 	uint32_t status, result;
112 
113 	status = bf_get(lpfc_wcqe_c_status, wcqe);
114 	result = wcqe->parameter;
115 	ctxp = cmdwqe->context2;
116 
117 	if (ctxp->state != LPFC_NVMET_STE_LS_RSP || ctxp->entry_cnt != 2) {
118 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
119 				"6410 NVMET LS cmpl state mismatch IO x%x: "
120 				"%d %d\n",
121 				ctxp->oxid, ctxp->state, ctxp->entry_cnt);
122 	}
123 
124 	if (!phba->targetport)
125 		goto out;
126 
127 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
128 
129 	if (status)
130 		atomic_inc(&tgtp->xmt_ls_rsp_error);
131 	else
132 		atomic_inc(&tgtp->xmt_ls_rsp_cmpl);
133 
134 out:
135 	rsp = &ctxp->ctx.ls_req;
136 
137 	lpfc_nvmeio_data(phba, "NVMET LS  CMPL: xri x%x stat x%x result x%x\n",
138 			 ctxp->oxid, status, result);
139 
140 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
141 			"6038 NVMET LS rsp cmpl: %d %d oxid x%x\n",
142 			status, result, ctxp->oxid);
143 
144 	lpfc_nlp_put(cmdwqe->context1);
145 	cmdwqe->context2 = NULL;
146 	cmdwqe->context3 = NULL;
147 	lpfc_sli_release_iocbq(phba, cmdwqe);
148 	rsp->done(rsp);
149 	kfree(ctxp);
150 }
151 
152 /**
153  * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context
154  * @phba: HBA buffer is associated with
155  * @ctxp: context to clean up
156  * @mp: Buffer to free
157  *
158  * Description: Frees the given DMA buffer in the appropriate way given by
159  * reposting it to its associated RQ so it can be reused.
160  *
161  * Notes: Takes phba->hbalock.  Can be called with or without other locks held.
162  *
163  * Returns: None
164  **/
165 void
166 lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
167 {
168 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
169 	struct lpfc_nvmet_rcv_ctx *ctxp = ctx_buf->context;
170 	struct lpfc_nvmet_tgtport *tgtp;
171 	struct fc_frame_header *fc_hdr;
172 	struct rqb_dmabuf *nvmebuf;
173 	uint32_t *payload;
174 	uint32_t size, oxid, sid, rc;
175 	unsigned long iflag;
176 
177 	if (ctxp->txrdy) {
178 		pci_pool_free(phba->txrdy_payload_pool, ctxp->txrdy,
179 			      ctxp->txrdy_phys);
180 		ctxp->txrdy = NULL;
181 		ctxp->txrdy_phys = 0;
182 	}
183 
184 	if (ctxp->state == LPFC_NVMET_STE_FREE) {
185 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
186 				"6411 NVMET free, already free IO x%x: %d %d\n",
187 				ctxp->oxid, ctxp->state, ctxp->entry_cnt);
188 	}
189 	ctxp->state = LPFC_NVMET_STE_FREE;
190 
191 	spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
192 	if (phba->sli4_hba.nvmet_io_wait_cnt) {
193 		list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list,
194 				 nvmebuf, struct rqb_dmabuf,
195 				 hbuf.list);
196 		phba->sli4_hba.nvmet_io_wait_cnt--;
197 		spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
198 				       iflag);
199 
200 		fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
201 		oxid = be16_to_cpu(fc_hdr->fh_ox_id);
202 		tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
203 		payload = (uint32_t *)(nvmebuf->dbuf.virt);
204 		size = nvmebuf->bytes_recv;
205 		sid = sli4_sid_from_fc_hdr(fc_hdr);
206 
207 		ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
208 		ctxp->wqeq = NULL;
209 		ctxp->txrdy = NULL;
210 		ctxp->offset = 0;
211 		ctxp->phba = phba;
212 		ctxp->size = size;
213 		ctxp->oxid = oxid;
214 		ctxp->sid = sid;
215 		ctxp->state = LPFC_NVMET_STE_RCV;
216 		ctxp->entry_cnt = 1;
217 		ctxp->flag = 0;
218 		ctxp->ctxbuf = ctx_buf;
219 		spin_lock_init(&ctxp->ctxlock);
220 
221 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
222 		if (phba->ktime_on) {
223 			ctxp->ts_cmd_nvme = ktime_get_ns();
224 			ctxp->ts_isr_cmd = ctxp->ts_cmd_nvme;
225 			ctxp->ts_nvme_data = 0;
226 			ctxp->ts_data_wqput = 0;
227 			ctxp->ts_isr_data = 0;
228 			ctxp->ts_data_nvme = 0;
229 			ctxp->ts_nvme_status = 0;
230 			ctxp->ts_status_wqput = 0;
231 			ctxp->ts_isr_status = 0;
232 			ctxp->ts_status_nvme = 0;
233 		}
234 #endif
235 		atomic_inc(&tgtp->rcv_fcp_cmd_in);
236 		/*
237 		 * The calling sequence should be:
238 		 * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done
239 		 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
240 		 * When we return from nvmet_fc_rcv_fcp_req, all relevant info
241 		 * the NVME command / FC header is stored.
242 		 * A buffer has already been reposted for this IO, so just free
243 		 * the nvmebuf.
244 		 */
245 		rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
246 					  payload, size);
247 
248 		/* Process FCP command */
249 		if (rc == 0) {
250 			atomic_inc(&tgtp->rcv_fcp_cmd_out);
251 			nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
252 			return;
253 		}
254 
255 		atomic_inc(&tgtp->rcv_fcp_cmd_drop);
256 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
257 				"2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
258 				ctxp->oxid, rc,
259 				atomic_read(&tgtp->rcv_fcp_cmd_in),
260 				atomic_read(&tgtp->rcv_fcp_cmd_out),
261 				atomic_read(&tgtp->xmt_fcp_release));
262 
263 		lpfc_nvmet_defer_release(phba, ctxp);
264 		lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
265 		nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
266 		return;
267 	}
268 	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
269 
270 	spin_lock_irqsave(&phba->sli4_hba.nvmet_ctx_put_lock, iflag);
271 	list_add_tail(&ctx_buf->list,
272 		      &phba->sli4_hba.lpfc_nvmet_ctx_put_list);
273 	phba->sli4_hba.nvmet_ctx_put_cnt++;
274 	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_ctx_put_lock, iflag);
275 #endif
276 }
277 
278 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
279 static void
280 lpfc_nvmet_ktime(struct lpfc_hba *phba,
281 		 struct lpfc_nvmet_rcv_ctx *ctxp)
282 {
283 	uint64_t seg1, seg2, seg3, seg4, seg5;
284 	uint64_t seg6, seg7, seg8, seg9, seg10;
285 
286 	if (!phba->ktime_on)
287 		return;
288 
289 	if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
290 	    !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
291 	    !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
292 	    !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
293 	    !ctxp->ts_isr_status || !ctxp->ts_status_nvme)
294 		return;
295 
296 	if (ctxp->ts_isr_cmd  > ctxp->ts_cmd_nvme)
297 		return;
298 	if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
299 		return;
300 	if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
301 		return;
302 	if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
303 		return;
304 	if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
305 		return;
306 	if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
307 		return;
308 	if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
309 		return;
310 	if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
311 		return;
312 	if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
313 		return;
314 	/*
315 	 * Segment 1 - Time from FCP command received by MSI-X ISR
316 	 * to FCP command is passed to NVME Layer.
317 	 * Segment 2 - Time from FCP command payload handed
318 	 * off to NVME Layer to Driver receives a Command op
319 	 * from NVME Layer.
320 	 * Segment 3 - Time from Driver receives a Command op
321 	 * from NVME Layer to Command is put on WQ.
322 	 * Segment 4 - Time from Driver WQ put is done
323 	 * to MSI-X ISR for Command cmpl.
324 	 * Segment 5 - Time from MSI-X ISR for Command cmpl to
325 	 * Command cmpl is passed to NVME Layer.
326 	 * Segment 6 - Time from Command cmpl is passed to NVME
327 	 * Layer to Driver receives a RSP op from NVME Layer.
328 	 * Segment 7 - Time from Driver receives a RSP op from
329 	 * NVME Layer to WQ put is done on TRSP FCP Status.
330 	 * Segment 8 - Time from Driver WQ put is done on TRSP
331 	 * FCP Status to MSI-X ISR for TRSP cmpl.
332 	 * Segment 9 - Time from MSI-X ISR for TRSP cmpl to
333 	 * TRSP cmpl is passed to NVME Layer.
334 	 * Segment 10 - Time from FCP command received by
335 	 * MSI-X ISR to command is completed on wire.
336 	 * (Segments 1 thru 8) for READDATA / WRITEDATA
337 	 * (Segments 1 thru 4) for READDATA_RSP
338 	 */
339 	seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
340 	seg2 = (ctxp->ts_nvme_data - ctxp->ts_isr_cmd) - seg1;
341 	seg3 = (ctxp->ts_data_wqput - ctxp->ts_isr_cmd) -
342 		seg1 - seg2;
343 	seg4 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd) -
344 		seg1 - seg2 - seg3;
345 	seg5 = (ctxp->ts_data_nvme - ctxp->ts_isr_cmd) -
346 		seg1 - seg2 - seg3 - seg4;
347 
348 	/* For auto rsp commands seg6 thru seg10 will be 0 */
349 	if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
350 		seg6 = (ctxp->ts_nvme_status -
351 			ctxp->ts_isr_cmd) -
352 			seg1 - seg2 - seg3 - seg4 - seg5;
353 		seg7 = (ctxp->ts_status_wqput -
354 			ctxp->ts_isr_cmd) -
355 			seg1 - seg2 - seg3 -
356 			seg4 - seg5 - seg6;
357 		seg8 = (ctxp->ts_isr_status -
358 			ctxp->ts_isr_cmd) -
359 			seg1 - seg2 - seg3 - seg4 -
360 			seg5 - seg6 - seg7;
361 		seg9 = (ctxp->ts_status_nvme -
362 			ctxp->ts_isr_cmd) -
363 			seg1 - seg2 - seg3 - seg4 -
364 			seg5 - seg6 - seg7 - seg8;
365 		seg10 = (ctxp->ts_isr_status -
366 			ctxp->ts_isr_cmd);
367 	} else {
368 		seg6 =  0;
369 		seg7 =  0;
370 		seg8 =  0;
371 		seg9 =  0;
372 		seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
373 	}
374 
375 	phba->ktime_seg1_total += seg1;
376 	if (seg1 < phba->ktime_seg1_min)
377 		phba->ktime_seg1_min = seg1;
378 	else if (seg1 > phba->ktime_seg1_max)
379 		phba->ktime_seg1_max = seg1;
380 
381 	phba->ktime_seg2_total += seg2;
382 	if (seg2 < phba->ktime_seg2_min)
383 		phba->ktime_seg2_min = seg2;
384 	else if (seg2 > phba->ktime_seg2_max)
385 		phba->ktime_seg2_max = seg2;
386 
387 	phba->ktime_seg3_total += seg3;
388 	if (seg3 < phba->ktime_seg3_min)
389 		phba->ktime_seg3_min = seg3;
390 	else if (seg3 > phba->ktime_seg3_max)
391 		phba->ktime_seg3_max = seg3;
392 
393 	phba->ktime_seg4_total += seg4;
394 	if (seg4 < phba->ktime_seg4_min)
395 		phba->ktime_seg4_min = seg4;
396 	else if (seg4 > phba->ktime_seg4_max)
397 		phba->ktime_seg4_max = seg4;
398 
399 	phba->ktime_seg5_total += seg5;
400 	if (seg5 < phba->ktime_seg5_min)
401 		phba->ktime_seg5_min = seg5;
402 	else if (seg5 > phba->ktime_seg5_max)
403 		phba->ktime_seg5_max = seg5;
404 
405 	phba->ktime_data_samples++;
406 	if (!seg6)
407 		goto out;
408 
409 	phba->ktime_seg6_total += seg6;
410 	if (seg6 < phba->ktime_seg6_min)
411 		phba->ktime_seg6_min = seg6;
412 	else if (seg6 > phba->ktime_seg6_max)
413 		phba->ktime_seg6_max = seg6;
414 
415 	phba->ktime_seg7_total += seg7;
416 	if (seg7 < phba->ktime_seg7_min)
417 		phba->ktime_seg7_min = seg7;
418 	else if (seg7 > phba->ktime_seg7_max)
419 		phba->ktime_seg7_max = seg7;
420 
421 	phba->ktime_seg8_total += seg8;
422 	if (seg8 < phba->ktime_seg8_min)
423 		phba->ktime_seg8_min = seg8;
424 	else if (seg8 > phba->ktime_seg8_max)
425 		phba->ktime_seg8_max = seg8;
426 
427 	phba->ktime_seg9_total += seg9;
428 	if (seg9 < phba->ktime_seg9_min)
429 		phba->ktime_seg9_min = seg9;
430 	else if (seg9 > phba->ktime_seg9_max)
431 		phba->ktime_seg9_max = seg9;
432 out:
433 	phba->ktime_seg10_total += seg10;
434 	if (seg10 < phba->ktime_seg10_min)
435 		phba->ktime_seg10_min = seg10;
436 	else if (seg10 > phba->ktime_seg10_max)
437 		phba->ktime_seg10_max = seg10;
438 	phba->ktime_status_samples++;
439 }
440 #endif
441 
442 /**
443  * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
444  * @phba: Pointer to HBA context object.
445  * @cmdwqe: Pointer to driver command WQE object.
446  * @wcqe: Pointer to driver response CQE object.
447  *
448  * The function is called from SLI ring event handler with no
449  * lock held. This function is the completion handler for NVME FCP commands
450  * The function frees memory resources used for the NVME commands.
451  **/
452 static void
453 lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
454 			  struct lpfc_wcqe_complete *wcqe)
455 {
456 	struct lpfc_nvmet_tgtport *tgtp;
457 	struct nvmefc_tgt_fcp_req *rsp;
458 	struct lpfc_nvmet_rcv_ctx *ctxp;
459 	uint32_t status, result, op, start_clean;
460 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
461 	uint32_t id;
462 #endif
463 
464 	ctxp = cmdwqe->context2;
465 	ctxp->flag &= ~LPFC_NVMET_IO_INP;
466 
467 	rsp = &ctxp->ctx.fcp_req;
468 	op = rsp->op;
469 
470 	status = bf_get(lpfc_wcqe_c_status, wcqe);
471 	result = wcqe->parameter;
472 
473 	if (phba->targetport)
474 		tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
475 	else
476 		tgtp = NULL;
477 
478 	lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
479 			 ctxp->oxid, op, status);
480 
481 	if (status) {
482 		rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
483 		rsp->transferred_length = 0;
484 		if (tgtp)
485 			atomic_inc(&tgtp->xmt_fcp_rsp_error);
486 
487 		/* pick up SLI4 exhange busy condition */
488 		if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
489 			ctxp->flag |= LPFC_NVMET_XBUSY;
490 
491 			lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
492 					"6315 IO Cmpl XBUSY: xri x%x: %x/%x\n",
493 					ctxp->oxid, status, result);
494 		} else {
495 			ctxp->flag &= ~LPFC_NVMET_XBUSY;
496 		}
497 
498 	} else {
499 		rsp->fcp_error = NVME_SC_SUCCESS;
500 		if (op == NVMET_FCOP_RSP)
501 			rsp->transferred_length = rsp->rsplen;
502 		else
503 			rsp->transferred_length = rsp->transfer_length;
504 		if (tgtp)
505 			atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);
506 	}
507 
508 	if ((op == NVMET_FCOP_READDATA_RSP) ||
509 	    (op == NVMET_FCOP_RSP)) {
510 		/* Sanity check */
511 		ctxp->state = LPFC_NVMET_STE_DONE;
512 		ctxp->entry_cnt++;
513 
514 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
515 		if (phba->ktime_on) {
516 			if (rsp->op == NVMET_FCOP_READDATA_RSP) {
517 				ctxp->ts_isr_data =
518 					cmdwqe->isr_timestamp;
519 				ctxp->ts_data_nvme =
520 					ktime_get_ns();
521 				ctxp->ts_nvme_status =
522 					ctxp->ts_data_nvme;
523 				ctxp->ts_status_wqput =
524 					ctxp->ts_data_nvme;
525 				ctxp->ts_isr_status =
526 					ctxp->ts_data_nvme;
527 				ctxp->ts_status_nvme =
528 					ctxp->ts_data_nvme;
529 			} else {
530 				ctxp->ts_isr_status =
531 					cmdwqe->isr_timestamp;
532 				ctxp->ts_status_nvme =
533 					ktime_get_ns();
534 			}
535 		}
536 		if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
537 			id = smp_processor_id();
538 			if (ctxp->cpu != id)
539 				lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
540 						"6703 CPU Check cmpl: "
541 						"cpu %d expect %d\n",
542 						id, ctxp->cpu);
543 			if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
544 				phba->cpucheck_cmpl_io[id]++;
545 		}
546 #endif
547 		rsp->done(rsp);
548 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
549 		if (phba->ktime_on)
550 			lpfc_nvmet_ktime(phba, ctxp);
551 #endif
552 		/* lpfc_nvmet_xmt_fcp_release() will recycle the context */
553 	} else {
554 		ctxp->entry_cnt++;
555 		start_clean = offsetof(struct lpfc_iocbq, wqe);
556 		memset(((char *)cmdwqe) + start_clean, 0,
557 		       (sizeof(struct lpfc_iocbq) - start_clean));
558 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
559 		if (phba->ktime_on) {
560 			ctxp->ts_isr_data = cmdwqe->isr_timestamp;
561 			ctxp->ts_data_nvme = ktime_get_ns();
562 		}
563 		if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
564 			id = smp_processor_id();
565 			if (ctxp->cpu != id)
566 				lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
567 						"6704 CPU Check cmdcmpl: "
568 						"cpu %d expect %d\n",
569 						id, ctxp->cpu);
570 			if (ctxp->cpu < LPFC_CHECK_CPU_CNT)
571 				phba->cpucheck_ccmpl_io[id]++;
572 		}
573 #endif
574 		rsp->done(rsp);
575 	}
576 }
577 
578 static int
579 lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
580 		      struct nvmefc_tgt_ls_req *rsp)
581 {
582 	struct lpfc_nvmet_rcv_ctx *ctxp =
583 		container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.ls_req);
584 	struct lpfc_hba *phba = ctxp->phba;
585 	struct hbq_dmabuf *nvmebuf =
586 		(struct hbq_dmabuf *)ctxp->rqb_buffer;
587 	struct lpfc_iocbq *nvmewqeq;
588 	struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
589 	struct lpfc_dmabuf dmabuf;
590 	struct ulp_bde64 bpl;
591 	int rc;
592 
593 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
594 			"6023 NVMET LS rsp oxid x%x\n", ctxp->oxid);
595 
596 	if ((ctxp->state != LPFC_NVMET_STE_LS_RCV) ||
597 	    (ctxp->entry_cnt != 1)) {
598 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
599 				"6412 NVMET LS rsp state mismatch "
600 				"oxid x%x: %d %d\n",
601 				ctxp->oxid, ctxp->state, ctxp->entry_cnt);
602 	}
603 	ctxp->state = LPFC_NVMET_STE_LS_RSP;
604 	ctxp->entry_cnt++;
605 
606 	nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, ctxp, rsp->rspdma,
607 				      rsp->rsplen);
608 	if (nvmewqeq == NULL) {
609 		atomic_inc(&nvmep->xmt_ls_drop);
610 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
611 				"6150 LS Drop IO x%x: Prep\n",
612 				ctxp->oxid);
613 		lpfc_in_buf_free(phba, &nvmebuf->dbuf);
614 		atomic_inc(&nvmep->xmt_ls_abort);
615 		lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp,
616 						ctxp->sid, ctxp->oxid);
617 		return -ENOMEM;
618 	}
619 
620 	/* Save numBdes for bpl2sgl */
621 	nvmewqeq->rsvd2 = 1;
622 	nvmewqeq->hba_wqidx = 0;
623 	nvmewqeq->context3 = &dmabuf;
624 	dmabuf.virt = &bpl;
625 	bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
626 	bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
627 	bpl.tus.f.bdeSize = rsp->rsplen;
628 	bpl.tus.f.bdeFlags = 0;
629 	bpl.tus.w = le32_to_cpu(bpl.tus.w);
630 
631 	nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_rsp_cmp;
632 	nvmewqeq->iocb_cmpl = NULL;
633 	nvmewqeq->context2 = ctxp;
634 
635 	lpfc_nvmeio_data(phba, "NVMET LS  RESP: xri x%x wqidx x%x len x%x\n",
636 			 ctxp->oxid, nvmewqeq->hba_wqidx, rsp->rsplen);
637 
638 	rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, nvmewqeq);
639 	if (rc == WQE_SUCCESS) {
640 		/*
641 		 * Okay to repost buffer here, but wait till cmpl
642 		 * before freeing ctxp and iocbq.
643 		 */
644 		lpfc_in_buf_free(phba, &nvmebuf->dbuf);
645 		ctxp->rqb_buffer = 0;
646 		atomic_inc(&nvmep->xmt_ls_rsp);
647 		return 0;
648 	}
649 	/* Give back resources */
650 	atomic_inc(&nvmep->xmt_ls_drop);
651 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
652 			"6151 LS Drop IO x%x: Issue %d\n",
653 			ctxp->oxid, rc);
654 
655 	lpfc_nlp_put(nvmewqeq->context1);
656 
657 	lpfc_in_buf_free(phba, &nvmebuf->dbuf);
658 	atomic_inc(&nvmep->xmt_ls_abort);
659 	lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
660 	return -ENXIO;
661 }
662 
663 static int
664 lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
665 		      struct nvmefc_tgt_fcp_req *rsp)
666 {
667 	struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
668 	struct lpfc_nvmet_rcv_ctx *ctxp =
669 		container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
670 	struct lpfc_hba *phba = ctxp->phba;
671 	struct lpfc_iocbq *nvmewqeq;
672 	int rc;
673 
674 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
675 	if (phba->ktime_on) {
676 		if (rsp->op == NVMET_FCOP_RSP)
677 			ctxp->ts_nvme_status = ktime_get_ns();
678 		else
679 			ctxp->ts_nvme_data = ktime_get_ns();
680 	}
681 	if (phba->cpucheck_on & LPFC_CHECK_NVMET_IO) {
682 		int id = smp_processor_id();
683 		ctxp->cpu = id;
684 		if (id < LPFC_CHECK_CPU_CNT)
685 			phba->cpucheck_xmt_io[id]++;
686 		if (rsp->hwqid != id) {
687 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
688 					"6705 CPU Check OP: "
689 					"cpu %d expect %d\n",
690 					id, rsp->hwqid);
691 			ctxp->cpu = rsp->hwqid;
692 		}
693 	}
694 #endif
695 
696 	/* Sanity check */
697 	if ((ctxp->flag & LPFC_NVMET_ABTS_RCV) ||
698 	    (ctxp->state == LPFC_NVMET_STE_ABORT)) {
699 		atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
700 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
701 				"6102 IO xri x%x aborted\n",
702 				ctxp->oxid);
703 		rc = -ENXIO;
704 		goto aerr;
705 	}
706 
707 	nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
708 	if (nvmewqeq == NULL) {
709 		atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
710 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
711 				"6152 FCP Drop IO x%x: Prep\n",
712 				ctxp->oxid);
713 		rc = -ENXIO;
714 		goto aerr;
715 	}
716 
717 	nvmewqeq->wqe_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
718 	nvmewqeq->iocb_cmpl = NULL;
719 	nvmewqeq->context2 = ctxp;
720 	nvmewqeq->iocb_flag |=  LPFC_IO_NVMET;
721 	ctxp->wqeq->hba_wqidx = rsp->hwqid;
722 
723 	lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
724 			 ctxp->oxid, rsp->op, rsp->rsplen);
725 
726 	ctxp->flag |= LPFC_NVMET_IO_INP;
727 	rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, nvmewqeq);
728 	if (rc == WQE_SUCCESS) {
729 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
730 		if (!phba->ktime_on)
731 			return 0;
732 		if (rsp->op == NVMET_FCOP_RSP)
733 			ctxp->ts_status_wqput = ktime_get_ns();
734 		else
735 			ctxp->ts_data_wqput = ktime_get_ns();
736 #endif
737 		return 0;
738 	}
739 
740 	/* Give back resources */
741 	atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
742 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
743 			"6153 FCP Drop IO x%x: Issue: %d\n",
744 			ctxp->oxid, rc);
745 
746 	ctxp->wqeq->hba_wqidx = 0;
747 	nvmewqeq->context2 = NULL;
748 	nvmewqeq->context3 = NULL;
749 	rc = -EBUSY;
750 aerr:
751 	return rc;
752 }
753 
754 static void
755 lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
756 {
757 	struct lpfc_nvmet_tgtport *tport = targetport->private;
758 
759 	/* release any threads waiting for the unreg to complete */
760 	complete(&tport->tport_unreg_done);
761 }
762 
763 static void
764 lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
765 			 struct nvmefc_tgt_fcp_req *req)
766 {
767 	struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
768 	struct lpfc_nvmet_rcv_ctx *ctxp =
769 		container_of(req, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
770 	struct lpfc_hba *phba = ctxp->phba;
771 	unsigned long flags;
772 
773 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
774 			"6103 NVMET Abort op: oxri x%x flg x%x ste %d\n",
775 			ctxp->oxid, ctxp->flag, ctxp->state);
776 
777 	lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n",
778 			 ctxp->oxid, ctxp->flag, ctxp->state);
779 
780 	atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
781 
782 	spin_lock_irqsave(&ctxp->ctxlock, flags);
783 
784 	/* Since iaab/iaar are NOT set, we need to check
785 	 * if the firmware is in process of aborting IO
786 	 */
787 	if (ctxp->flag & LPFC_NVMET_XBUSY) {
788 		spin_unlock_irqrestore(&ctxp->ctxlock, flags);
789 		return;
790 	}
791 	ctxp->flag |= LPFC_NVMET_ABORT_OP;
792 
793 	/* An state of LPFC_NVMET_STE_RCV means we have just received
794 	 * the NVME command and have not started processing it.
795 	 * (by issuing any IO WQEs on this exchange yet)
796 	 */
797 	if (ctxp->state == LPFC_NVMET_STE_RCV)
798 		lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
799 						 ctxp->oxid);
800 	else
801 		lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
802 					       ctxp->oxid);
803 	spin_unlock_irqrestore(&ctxp->ctxlock, flags);
804 }
805 
806 static void
807 lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
808 			   struct nvmefc_tgt_fcp_req *rsp)
809 {
810 	struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
811 	struct lpfc_nvmet_rcv_ctx *ctxp =
812 		container_of(rsp, struct lpfc_nvmet_rcv_ctx, ctx.fcp_req);
813 	struct lpfc_hba *phba = ctxp->phba;
814 	unsigned long flags;
815 	bool aborting = false;
816 
817 	if (ctxp->state != LPFC_NVMET_STE_DONE &&
818 	    ctxp->state != LPFC_NVMET_STE_ABORT) {
819 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
820 				"6413 NVMET release bad state %d %d oxid x%x\n",
821 				ctxp->state, ctxp->entry_cnt, ctxp->oxid);
822 	}
823 
824 	spin_lock_irqsave(&ctxp->ctxlock, flags);
825 	if ((ctxp->flag & LPFC_NVMET_ABORT_OP) ||
826 	    (ctxp->flag & LPFC_NVMET_XBUSY)) {
827 		aborting = true;
828 		/* let the abort path do the real release */
829 		lpfc_nvmet_defer_release(phba, ctxp);
830 	}
831 	spin_unlock_irqrestore(&ctxp->ctxlock, flags);
832 
833 	lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid,
834 			 ctxp->state, aborting);
835 
836 	atomic_inc(&lpfc_nvmep->xmt_fcp_release);
837 
838 	if (aborting)
839 		return;
840 
841 	lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
842 }
843 
844 static struct nvmet_fc_target_template lpfc_tgttemplate = {
845 	.targetport_delete = lpfc_nvmet_targetport_delete,
846 	.xmt_ls_rsp     = lpfc_nvmet_xmt_ls_rsp,
847 	.fcp_op         = lpfc_nvmet_xmt_fcp_op,
848 	.fcp_abort      = lpfc_nvmet_xmt_fcp_abort,
849 	.fcp_req_release = lpfc_nvmet_xmt_fcp_release,
850 
851 	.max_hw_queues  = 1,
852 	.max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
853 	.max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
854 	.dma_boundary = 0xFFFFFFFF,
855 
856 	/* optional features */
857 	.target_features = 0,
858 	/* sizes of additional private data for data structures */
859 	.target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
860 };
861 
862 static void
863 lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
864 {
865 	struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;
866 	unsigned long flags;
867 
868 	spin_lock_irqsave(&phba->sli4_hba.nvmet_ctx_get_lock, flags);
869 	spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
870 	list_for_each_entry_safe(ctx_buf, next_ctx_buf,
871 			&phba->sli4_hba.lpfc_nvmet_ctx_get_list, list) {
872 		spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
873 		list_del_init(&ctx_buf->list);
874 		spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
875 		__lpfc_clear_active_sglq(phba,
876 					 ctx_buf->sglq->sli4_lxritag);
877 		ctx_buf->sglq->state = SGL_FREED;
878 		ctx_buf->sglq->ndlp = NULL;
879 
880 		spin_lock(&phba->sli4_hba.sgl_list_lock);
881 		list_add_tail(&ctx_buf->sglq->list,
882 			      &phba->sli4_hba.lpfc_nvmet_sgl_list);
883 		spin_unlock(&phba->sli4_hba.sgl_list_lock);
884 
885 		lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
886 		kfree(ctx_buf->context);
887 	}
888 	list_for_each_entry_safe(ctx_buf, next_ctx_buf,
889 			&phba->sli4_hba.lpfc_nvmet_ctx_put_list, list) {
890 		spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
891 		list_del_init(&ctx_buf->list);
892 		spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
893 		__lpfc_clear_active_sglq(phba,
894 					 ctx_buf->sglq->sli4_lxritag);
895 		ctx_buf->sglq->state = SGL_FREED;
896 		ctx_buf->sglq->ndlp = NULL;
897 
898 		spin_lock(&phba->sli4_hba.sgl_list_lock);
899 		list_add_tail(&ctx_buf->sglq->list,
900 			      &phba->sli4_hba.lpfc_nvmet_sgl_list);
901 		spin_unlock(&phba->sli4_hba.sgl_list_lock);
902 
903 		lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
904 		kfree(ctx_buf->context);
905 	}
906 	spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
907 	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_ctx_get_lock, flags);
908 }
909 
910 static int
911 lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
912 {
913 	struct lpfc_nvmet_ctxbuf *ctx_buf;
914 	struct lpfc_iocbq *nvmewqe;
915 	union lpfc_wqe128 *wqe;
916 	int i;
917 
918 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
919 			"6403 Allocate NVMET resources for %d XRIs\n",
920 			phba->sli4_hba.nvmet_xri_cnt);
921 
922 	/* For all nvmet xris, allocate resources needed to process a
923 	 * received command on a per xri basis.
924 	 */
925 	for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
926 		ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
927 		if (!ctx_buf) {
928 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
929 					"6404 Ran out of memory for NVMET\n");
930 			return -ENOMEM;
931 		}
932 
933 		ctx_buf->context = kzalloc(sizeof(*ctx_buf->context),
934 					   GFP_KERNEL);
935 		if (!ctx_buf->context) {
936 			kfree(ctx_buf);
937 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
938 					"6405 Ran out of NVMET "
939 					"context memory\n");
940 			return -ENOMEM;
941 		}
942 		ctx_buf->context->ctxbuf = ctx_buf;
943 		ctx_buf->context->state = LPFC_NVMET_STE_FREE;
944 
945 		ctx_buf->iocbq = lpfc_sli_get_iocbq(phba);
946 		if (!ctx_buf->iocbq) {
947 			kfree(ctx_buf->context);
948 			kfree(ctx_buf);
949 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
950 					"6406 Ran out of NVMET iocb/WQEs\n");
951 			return -ENOMEM;
952 		}
953 		ctx_buf->iocbq->iocb_flag = LPFC_IO_NVMET;
954 		nvmewqe = ctx_buf->iocbq;
955 		wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
956 		/* Initialize WQE */
957 		memset(wqe, 0, sizeof(union lpfc_wqe));
958 		/* Word 7 */
959 		bf_set(wqe_ct, &wqe->generic.wqe_com, SLI4_CT_RPI);
960 		bf_set(wqe_class, &wqe->generic.wqe_com, CLASS3);
961 		bf_set(wqe_pu, &wqe->generic.wqe_com, 1);
962 		/* Word 10 */
963 		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
964 		bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0);
965 		bf_set(wqe_qosd, &wqe->generic.wqe_com, 0);
966 
967 		ctx_buf->iocbq->context1 = NULL;
968 		spin_lock(&phba->sli4_hba.sgl_list_lock);
969 		ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq);
970 		spin_unlock(&phba->sli4_hba.sgl_list_lock);
971 		if (!ctx_buf->sglq) {
972 			lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
973 			kfree(ctx_buf->context);
974 			kfree(ctx_buf);
975 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
976 					"6407 Ran out of NVMET XRIs\n");
977 			return -ENOMEM;
978 		}
979 		spin_lock(&phba->sli4_hba.nvmet_ctx_get_lock);
980 		list_add_tail(&ctx_buf->list,
981 			      &phba->sli4_hba.lpfc_nvmet_ctx_get_list);
982 		spin_unlock(&phba->sli4_hba.nvmet_ctx_get_lock);
983 	}
984 	phba->sli4_hba.nvmet_ctx_get_cnt = phba->sli4_hba.nvmet_xri_cnt;
985 	return 0;
986 }
987 
988 int
989 lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
990 {
991 	struct lpfc_vport  *vport = phba->pport;
992 	struct lpfc_nvmet_tgtport *tgtp;
993 	struct nvmet_fc_port_info pinfo;
994 	int error;
995 
996 	if (phba->targetport)
997 		return 0;
998 
999 	error = lpfc_nvmet_setup_io_context(phba);
1000 	if (error)
1001 		return error;
1002 
1003 	memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
1004 	pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
1005 	pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
1006 	pinfo.port_id = vport->fc_myDID;
1007 
1008 	/* Limit to LPFC_MAX_NVME_SEG_CNT.
1009 	 * For now need + 1 to get around NVME transport logic.
1010 	 */
1011 	if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) {
1012 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
1013 				"6400 Reducing sg segment cnt to %d\n",
1014 				LPFC_MAX_NVME_SEG_CNT);
1015 		phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT;
1016 	} else {
1017 		phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt;
1018 	}
1019 	lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
1020 	lpfc_tgttemplate.max_hw_queues = phba->cfg_nvme_io_channel;
1021 	lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP |
1022 					   NVMET_FCTGTFEAT_CMD_IN_ISR |
1023 					   NVMET_FCTGTFEAT_OPDONE_IN_ISR;
1024 
1025 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1026 	error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
1027 					     &phba->pcidev->dev,
1028 					     &phba->targetport);
1029 #else
1030 	error = -ENOENT;
1031 #endif
1032 	if (error) {
1033 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1034 				"6025 Cannot register NVME targetport "
1035 				"x%x\n", error);
1036 		phba->targetport = NULL;
1037 
1038 		lpfc_nvmet_cleanup_io_context(phba);
1039 
1040 	} else {
1041 		tgtp = (struct lpfc_nvmet_tgtport *)
1042 			phba->targetport->private;
1043 		tgtp->phba = phba;
1044 
1045 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1046 				"6026 Registered NVME "
1047 				"targetport: %p, private %p "
1048 				"portnm %llx nodenm %llx\n",
1049 				phba->targetport, tgtp,
1050 				pinfo.port_name, pinfo.node_name);
1051 
1052 		atomic_set(&tgtp->rcv_ls_req_in, 0);
1053 		atomic_set(&tgtp->rcv_ls_req_out, 0);
1054 		atomic_set(&tgtp->rcv_ls_req_drop, 0);
1055 		atomic_set(&tgtp->xmt_ls_abort, 0);
1056 		atomic_set(&tgtp->xmt_ls_abort_cmpl, 0);
1057 		atomic_set(&tgtp->xmt_ls_rsp, 0);
1058 		atomic_set(&tgtp->xmt_ls_drop, 0);
1059 		atomic_set(&tgtp->xmt_ls_rsp_error, 0);
1060 		atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
1061 		atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
1062 		atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
1063 		atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
1064 		atomic_set(&tgtp->xmt_fcp_drop, 0);
1065 		atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
1066 		atomic_set(&tgtp->xmt_fcp_read, 0);
1067 		atomic_set(&tgtp->xmt_fcp_write, 0);
1068 		atomic_set(&tgtp->xmt_fcp_rsp, 0);
1069 		atomic_set(&tgtp->xmt_fcp_release, 0);
1070 		atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
1071 		atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
1072 		atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
1073 		atomic_set(&tgtp->xmt_fcp_abort, 0);
1074 		atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0);
1075 		atomic_set(&tgtp->xmt_abort_unsol, 0);
1076 		atomic_set(&tgtp->xmt_abort_sol, 0);
1077 		atomic_set(&tgtp->xmt_abort_rsp, 0);
1078 		atomic_set(&tgtp->xmt_abort_rsp_error, 0);
1079 	}
1080 	return error;
1081 }
1082 
1083 int
1084 lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
1085 {
1086 	struct lpfc_vport  *vport = phba->pport;
1087 
1088 	if (!phba->targetport)
1089 		return 0;
1090 
1091 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1092 			 "6007 Update NVMET port %p did x%x\n",
1093 			 phba->targetport, vport->fc_myDID);
1094 
1095 	phba->targetport->port_id = vport->fc_myDID;
1096 	return 0;
1097 }
1098 
1099 /**
1100  * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort
1101  * @phba: pointer to lpfc hba data structure.
1102  * @axri: pointer to the nvmet xri abort wcqe structure.
1103  *
1104  * This routine is invoked by the worker thread to process a SLI4 fast-path
1105  * NVMET aborted xri.
1106  **/
1107 void
1108 lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,
1109 			    struct sli4_wcqe_xri_aborted *axri)
1110 {
1111 	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
1112 	uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
1113 	struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
1114 	struct lpfc_nodelist *ndlp;
1115 	unsigned long iflag = 0;
1116 	int rrq_empty = 0;
1117 	bool released = false;
1118 
1119 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1120 			"6317 XB aborted xri x%x rxid x%x\n", xri, rxid);
1121 
1122 	if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
1123 		return;
1124 	spin_lock_irqsave(&phba->hbalock, iflag);
1125 	spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1126 	list_for_each_entry_safe(ctxp, next_ctxp,
1127 				 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1128 				 list) {
1129 		if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1130 			continue;
1131 
1132 		/* Check if we already received a free context call
1133 		 * and we have completed processing an abort situation.
1134 		 */
1135 		if (ctxp->flag & LPFC_NVMET_CTX_RLS &&
1136 		    !(ctxp->flag & LPFC_NVMET_ABORT_OP)) {
1137 			list_del(&ctxp->list);
1138 			released = true;
1139 		}
1140 		ctxp->flag &= ~LPFC_NVMET_XBUSY;
1141 		spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1142 
1143 		rrq_empty = list_empty(&phba->active_rrq_list);
1144 		spin_unlock_irqrestore(&phba->hbalock, iflag);
1145 		ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1146 		if (ndlp && NLP_CHK_NODE_ACT(ndlp) &&
1147 		    (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||
1148 		     ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {
1149 			lpfc_set_rrq_active(phba, ndlp,
1150 				ctxp->ctxbuf->sglq->sli4_lxritag,
1151 				rxid, 1);
1152 			lpfc_sli4_abts_err_handler(phba, ndlp, axri);
1153 		}
1154 
1155 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1156 				"6318 XB aborted oxid %x flg x%x (%x)\n",
1157 				ctxp->oxid, ctxp->flag, released);
1158 		if (released)
1159 			lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
1160 
1161 		if (rrq_empty)
1162 			lpfc_worker_wake_up(phba);
1163 		return;
1164 	}
1165 	spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1166 	spin_unlock_irqrestore(&phba->hbalock, iflag);
1167 }
1168 
1169 int
1170 lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
1171 			   struct fc_frame_header *fc_hdr)
1172 
1173 {
1174 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1175 	struct lpfc_hba *phba = vport->phba;
1176 	struct lpfc_nvmet_rcv_ctx *ctxp, *next_ctxp;
1177 	struct nvmefc_tgt_fcp_req *rsp;
1178 	uint16_t xri;
1179 	unsigned long iflag = 0;
1180 
1181 	xri = be16_to_cpu(fc_hdr->fh_ox_id);
1182 
1183 	spin_lock_irqsave(&phba->hbalock, iflag);
1184 	spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1185 	list_for_each_entry_safe(ctxp, next_ctxp,
1186 				 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
1187 				 list) {
1188 		if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
1189 			continue;
1190 
1191 		spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1192 		spin_unlock_irqrestore(&phba->hbalock, iflag);
1193 
1194 		spin_lock_irqsave(&ctxp->ctxlock, iflag);
1195 		ctxp->flag |= LPFC_NVMET_ABTS_RCV;
1196 		spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
1197 
1198 		lpfc_nvmeio_data(phba,
1199 			"NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1200 			xri, smp_processor_id(), 0);
1201 
1202 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1203 				"6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
1204 
1205 		rsp = &ctxp->ctx.fcp_req;
1206 		nvmet_fc_rcv_fcp_abort(phba->targetport, rsp);
1207 
1208 		/* Respond with BA_ACC accordingly */
1209 		lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
1210 		return 0;
1211 	}
1212 	spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock);
1213 	spin_unlock_irqrestore(&phba->hbalock, iflag);
1214 
1215 	lpfc_nvmeio_data(phba, "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
1216 			 xri, smp_processor_id(), 1);
1217 
1218 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
1219 			"6320 NVMET Rcv ABTS:rjt xri x%x\n", xri);
1220 
1221 	/* Respond with BA_RJT accordingly */
1222 	lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0);
1223 #endif
1224 	return 0;
1225 }
1226 
1227 void
1228 lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
1229 {
1230 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1231 	struct lpfc_nvmet_tgtport *tgtp;
1232 
1233 	if (phba->nvmet_support == 0)
1234 		return;
1235 	if (phba->targetport) {
1236 		tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1237 		init_completion(&tgtp->tport_unreg_done);
1238 		nvmet_fc_unregister_targetport(phba->targetport);
1239 		wait_for_completion_timeout(&tgtp->tport_unreg_done, 5);
1240 		lpfc_nvmet_cleanup_io_context(phba);
1241 	}
1242 	phba->targetport = NULL;
1243 #endif
1244 }
1245 
1246 /**
1247  * lpfc_nvmet_unsol_ls_buffer - Process an unsolicited event data buffer
1248  * @phba: pointer to lpfc hba data structure.
1249  * @pring: pointer to a SLI ring.
1250  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1251  *
1252  * This routine is used for processing the WQE associated with a unsolicited
1253  * event. It first determines whether there is an existing ndlp that matches
1254  * the DID from the unsolicited WQE. If not, it will create a new one with
1255  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1256  * WQE is then used to invoke the proper routine and to set up proper state
1257  * of the discovery state machine.
1258  **/
1259 static void
1260 lpfc_nvmet_unsol_ls_buffer(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1261 			   struct hbq_dmabuf *nvmebuf)
1262 {
1263 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1264 	struct lpfc_nvmet_tgtport *tgtp;
1265 	struct fc_frame_header *fc_hdr;
1266 	struct lpfc_nvmet_rcv_ctx *ctxp;
1267 	uint32_t *payload;
1268 	uint32_t size, oxid, sid, rc;
1269 
1270 	if (!nvmebuf || !phba->targetport) {
1271 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1272 				"6154 LS Drop IO\n");
1273 		oxid = 0;
1274 		size = 0;
1275 		sid = 0;
1276 		ctxp = NULL;
1277 		goto dropit;
1278 	}
1279 
1280 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1281 	payload = (uint32_t *)(nvmebuf->dbuf.virt);
1282 	fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1283 	size = bf_get(lpfc_rcqe_length,  &nvmebuf->cq_event.cqe.rcqe_cmpl);
1284 	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1285 	sid = sli4_sid_from_fc_hdr(fc_hdr);
1286 
1287 	ctxp = kzalloc(sizeof(struct lpfc_nvmet_rcv_ctx), GFP_ATOMIC);
1288 	if (ctxp == NULL) {
1289 		atomic_inc(&tgtp->rcv_ls_req_drop);
1290 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1291 				"6155 LS Drop IO x%x: Alloc\n",
1292 				oxid);
1293 dropit:
1294 		lpfc_nvmeio_data(phba, "NVMET LS  DROP: "
1295 				 "xri x%x sz %d from %06x\n",
1296 				 oxid, size, sid);
1297 		if (nvmebuf)
1298 			lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1299 		return;
1300 	}
1301 	ctxp->phba = phba;
1302 	ctxp->size = size;
1303 	ctxp->oxid = oxid;
1304 	ctxp->sid = sid;
1305 	ctxp->wqeq = NULL;
1306 	ctxp->state = LPFC_NVMET_STE_LS_RCV;
1307 	ctxp->entry_cnt = 1;
1308 	ctxp->rqb_buffer = (void *)nvmebuf;
1309 
1310 	lpfc_nvmeio_data(phba, "NVMET LS   RCV: xri x%x sz %d from %06x\n",
1311 			 oxid, size, sid);
1312 	/*
1313 	 * The calling sequence should be:
1314 	 * nvmet_fc_rcv_ls_req -> lpfc_nvmet_xmt_ls_rsp/cmp ->_req->done
1315 	 * lpfc_nvmet_xmt_ls_rsp_cmp should free the allocated ctxp.
1316 	 */
1317 	atomic_inc(&tgtp->rcv_ls_req_in);
1318 	rc = nvmet_fc_rcv_ls_req(phba->targetport, &ctxp->ctx.ls_req,
1319 				 payload, size);
1320 
1321 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1322 			"6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x "
1323 			"%08x %08x %08x\n", size, rc,
1324 			*payload, *(payload+1), *(payload+2),
1325 			*(payload+3), *(payload+4), *(payload+5));
1326 
1327 	if (rc == 0) {
1328 		atomic_inc(&tgtp->rcv_ls_req_out);
1329 		return;
1330 	}
1331 
1332 	lpfc_nvmeio_data(phba, "NVMET LS  DROP: xri x%x sz %d from %06x\n",
1333 			 oxid, size, sid);
1334 
1335 	atomic_inc(&tgtp->rcv_ls_req_drop);
1336 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1337 			"6156 LS Drop IO x%x: nvmet_fc_rcv_ls_req %d\n",
1338 			ctxp->oxid, rc);
1339 
1340 	/* We assume a rcv'ed cmd ALWAYs fits into 1 buffer */
1341 	if (nvmebuf)
1342 		lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1343 
1344 	atomic_inc(&tgtp->xmt_ls_abort);
1345 	lpfc_nvmet_unsol_ls_issue_abort(phba, ctxp, sid, oxid);
1346 #endif
1347 }
1348 
1349 /**
1350  * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
1351  * @phba: pointer to lpfc hba data structure.
1352  * @pring: pointer to a SLI ring.
1353  * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
1354  *
1355  * This routine is used for processing the WQE associated with a unsolicited
1356  * event. It first determines whether there is an existing ndlp that matches
1357  * the DID from the unsolicited WQE. If not, it will create a new one with
1358  * the DID from the unsolicited WQE. The ELS command from the unsolicited
1359  * WQE is then used to invoke the proper routine and to set up proper state
1360  * of the discovery state machine.
1361  **/
1362 static void
1363 lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
1364 			    struct lpfc_sli_ring *pring,
1365 			    struct rqb_dmabuf *nvmebuf,
1366 			    uint64_t isr_timestamp)
1367 {
1368 #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
1369 	struct lpfc_nvmet_rcv_ctx *ctxp;
1370 	struct lpfc_nvmet_tgtport *tgtp;
1371 	struct fc_frame_header *fc_hdr;
1372 	struct lpfc_nvmet_ctxbuf *ctx_buf;
1373 	uint32_t *payload;
1374 	uint32_t size, oxid, sid, rc, qno;
1375 	unsigned long iflag;
1376 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1377 	uint32_t id;
1378 #endif
1379 
1380 	ctx_buf = NULL;
1381 	if (!nvmebuf || !phba->targetport) {
1382 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1383 				"6157 NVMET FCP Drop IO\n");
1384 		oxid = 0;
1385 		size = 0;
1386 		sid = 0;
1387 		ctxp = NULL;
1388 		goto dropit;
1389 	}
1390 
1391 	spin_lock_irqsave(&phba->sli4_hba.nvmet_ctx_get_lock, iflag);
1392 	if (phba->sli4_hba.nvmet_ctx_get_cnt) {
1393 		list_remove_head(&phba->sli4_hba.lpfc_nvmet_ctx_get_list,
1394 				 ctx_buf, struct lpfc_nvmet_ctxbuf, list);
1395 		phba->sli4_hba.nvmet_ctx_get_cnt--;
1396 	} else {
1397 		spin_lock(&phba->sli4_hba.nvmet_ctx_put_lock);
1398 		if (phba->sli4_hba.nvmet_ctx_put_cnt) {
1399 			list_splice(&phba->sli4_hba.lpfc_nvmet_ctx_put_list,
1400 				    &phba->sli4_hba.lpfc_nvmet_ctx_get_list);
1401 			INIT_LIST_HEAD(&phba->sli4_hba.lpfc_nvmet_ctx_put_list);
1402 			phba->sli4_hba.nvmet_ctx_get_cnt =
1403 				phba->sli4_hba.nvmet_ctx_put_cnt;
1404 			phba->sli4_hba.nvmet_ctx_put_cnt = 0;
1405 			spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
1406 
1407 			list_remove_head(
1408 				&phba->sli4_hba.lpfc_nvmet_ctx_get_list,
1409 				ctx_buf, struct lpfc_nvmet_ctxbuf, list);
1410 			phba->sli4_hba.nvmet_ctx_get_cnt--;
1411 		} else {
1412 			spin_unlock(&phba->sli4_hba.nvmet_ctx_put_lock);
1413 		}
1414 	}
1415 	spin_unlock_irqrestore(&phba->sli4_hba.nvmet_ctx_get_lock, iflag);
1416 
1417 	fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
1418 	oxid = be16_to_cpu(fc_hdr->fh_ox_id);
1419 	size = nvmebuf->bytes_recv;
1420 
1421 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1422 	if (phba->cpucheck_on & LPFC_CHECK_NVMET_RCV) {
1423 		id = smp_processor_id();
1424 		if (id < LPFC_CHECK_CPU_CNT)
1425 			phba->cpucheck_rcv_io[id]++;
1426 	}
1427 #endif
1428 
1429 	lpfc_nvmeio_data(phba, "NVMET FCP  RCV: xri x%x sz %d CPU %02x\n",
1430 			 oxid, size, smp_processor_id());
1431 
1432 	if (!ctx_buf) {
1433 		/* Queue this NVME IO to process later */
1434 		spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
1435 		list_add_tail(&nvmebuf->hbuf.list,
1436 			      &phba->sli4_hba.lpfc_nvmet_io_wait_list);
1437 		phba->sli4_hba.nvmet_io_wait_cnt++;
1438 		phba->sli4_hba.nvmet_io_wait_total++;
1439 		spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
1440 				       iflag);
1441 
1442 		/* Post a brand new DMA buffer to RQ */
1443 		qno = nvmebuf->idx;
1444 		lpfc_post_rq_buffer(
1445 			phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
1446 			phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
1447 		return;
1448 	}
1449 
1450 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1451 	payload = (uint32_t *)(nvmebuf->dbuf.virt);
1452 	sid = sli4_sid_from_fc_hdr(fc_hdr);
1453 
1454 	ctxp = (struct lpfc_nvmet_rcv_ctx *)ctx_buf->context;
1455 	if (ctxp->state != LPFC_NVMET_STE_FREE) {
1456 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1457 				"6414 NVMET Context corrupt %d %d oxid x%x\n",
1458 				ctxp->state, ctxp->entry_cnt, ctxp->oxid);
1459 	}
1460 	ctxp->wqeq = NULL;
1461 	ctxp->txrdy = NULL;
1462 	ctxp->offset = 0;
1463 	ctxp->phba = phba;
1464 	ctxp->size = size;
1465 	ctxp->oxid = oxid;
1466 	ctxp->sid = sid;
1467 	ctxp->state = LPFC_NVMET_STE_RCV;
1468 	ctxp->entry_cnt = 1;
1469 	ctxp->flag = 0;
1470 	ctxp->ctxbuf = ctx_buf;
1471 	spin_lock_init(&ctxp->ctxlock);
1472 
1473 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1474 	if (phba->ktime_on) {
1475 		ctxp->ts_isr_cmd = isr_timestamp;
1476 		ctxp->ts_cmd_nvme = ktime_get_ns();
1477 		ctxp->ts_nvme_data = 0;
1478 		ctxp->ts_data_wqput = 0;
1479 		ctxp->ts_isr_data = 0;
1480 		ctxp->ts_data_nvme = 0;
1481 		ctxp->ts_nvme_status = 0;
1482 		ctxp->ts_status_wqput = 0;
1483 		ctxp->ts_isr_status = 0;
1484 		ctxp->ts_status_nvme = 0;
1485 	}
1486 #endif
1487 
1488 	atomic_inc(&tgtp->rcv_fcp_cmd_in);
1489 	/*
1490 	 * The calling sequence should be:
1491 	 * nvmet_fc_rcv_fcp_req -> lpfc_nvmet_xmt_fcp_op/cmp -> req->done
1492 	 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
1493 	 * When we return from nvmet_fc_rcv_fcp_req, all relevant info in
1494 	 * the NVME command / FC header is stored, so we are free to repost
1495 	 * the buffer.
1496 	 */
1497 	rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->ctx.fcp_req,
1498 				  payload, size);
1499 
1500 	/* Process FCP command */
1501 	if (rc == 0) {
1502 		atomic_inc(&tgtp->rcv_fcp_cmd_out);
1503 		lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
1504 		return;
1505 	}
1506 
1507 	atomic_inc(&tgtp->rcv_fcp_cmd_drop);
1508 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1509 			"6159 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
1510 			ctxp->oxid, rc,
1511 			atomic_read(&tgtp->rcv_fcp_cmd_in),
1512 			atomic_read(&tgtp->rcv_fcp_cmd_out),
1513 			atomic_read(&tgtp->xmt_fcp_release));
1514 dropit:
1515 	lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
1516 			 oxid, size, sid);
1517 	if (oxid) {
1518 		lpfc_nvmet_defer_release(phba, ctxp);
1519 		lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
1520 		lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
1521 		return;
1522 	}
1523 
1524 	if (ctx_buf)
1525 		lpfc_nvmet_ctxbuf_post(phba, ctx_buf);
1526 
1527 	if (nvmebuf)
1528 		lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
1529 #endif
1530 }
1531 
1532 /**
1533  * lpfc_nvmet_unsol_ls_event - Process an unsolicited event from an nvme nport
1534  * @phba: pointer to lpfc hba data structure.
1535  * @pring: pointer to a SLI ring.
1536  * @nvmebuf: pointer to received nvme data structure.
1537  *
1538  * This routine is used to process an unsolicited event received from a SLI
1539  * (Service Level Interface) ring. The actual processing of the data buffer
1540  * associated with the unsolicited event is done by invoking the routine
1541  * lpfc_nvmet_unsol_ls_buffer() after properly set up the buffer from the
1542  * SLI RQ on which the unsolicited event was received.
1543  **/
1544 void
1545 lpfc_nvmet_unsol_ls_event(struct lpfc_hba *phba, struct lpfc_sli_ring *pring,
1546 			  struct lpfc_iocbq *piocb)
1547 {
1548 	struct lpfc_dmabuf *d_buf;
1549 	struct hbq_dmabuf *nvmebuf;
1550 
1551 	d_buf = piocb->context2;
1552 	nvmebuf = container_of(d_buf, struct hbq_dmabuf, dbuf);
1553 
1554 	if (phba->nvmet_support == 0) {
1555 		lpfc_in_buf_free(phba, &nvmebuf->dbuf);
1556 		return;
1557 	}
1558 	lpfc_nvmet_unsol_ls_buffer(phba, pring, nvmebuf);
1559 }
1560 
1561 /**
1562  * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
1563  * @phba: pointer to lpfc hba data structure.
1564  * @pring: pointer to a SLI ring.
1565  * @nvmebuf: pointer to received nvme data structure.
1566  *
1567  * This routine is used to process an unsolicited event received from a SLI
1568  * (Service Level Interface) ring. The actual processing of the data buffer
1569  * associated with the unsolicited event is done by invoking the routine
1570  * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
1571  * SLI RQ on which the unsolicited event was received.
1572  **/
1573 void
1574 lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
1575 			   struct lpfc_sli_ring *pring,
1576 			   struct rqb_dmabuf *nvmebuf,
1577 			   uint64_t isr_timestamp)
1578 {
1579 	if (phba->nvmet_support == 0) {
1580 		lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
1581 		return;
1582 	}
1583 	lpfc_nvmet_unsol_fcp_buffer(phba, pring, nvmebuf,
1584 				    isr_timestamp);
1585 }
1586 
1587 /**
1588  * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
1589  * @phba: pointer to a host N_Port data structure.
1590  * @ctxp: Context info for NVME LS Request
1591  * @rspbuf: DMA buffer of NVME command.
1592  * @rspsize: size of the NVME command.
1593  *
1594  * This routine is used for allocating a lpfc-WQE data structure from
1595  * the driver lpfc-WQE free-list and prepare the WQE with the parameters
1596  * passed into the routine for discovery state machine to issue an Extended
1597  * Link Service (NVME) commands. It is a generic lpfc-WQE allocation
1598  * and preparation routine that is used by all the discovery state machine
1599  * routines and the NVME command-specific fields will be later set up by
1600  * the individual discovery machine routines after calling this routine
1601  * allocating and preparing a generic WQE data structure. It fills in the
1602  * Buffer Descriptor Entries (BDEs), allocates buffers for both command
1603  * payload and response payload (if expected). The reference count on the
1604  * ndlp is incremented by 1 and the reference to the ndlp is put into
1605  * context1 of the WQE data structure for this WQE to hold the ndlp
1606  * reference for the command's callback function to access later.
1607  *
1608  * Return code
1609  *   Pointer to the newly allocated/prepared nvme wqe data structure
1610  *   NULL - when nvme wqe data structure allocation/preparation failed
1611  **/
1612 static struct lpfc_iocbq *
1613 lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
1614 		       struct lpfc_nvmet_rcv_ctx *ctxp,
1615 		       dma_addr_t rspbuf, uint16_t rspsize)
1616 {
1617 	struct lpfc_nodelist *ndlp;
1618 	struct lpfc_iocbq *nvmewqe;
1619 	union lpfc_wqe *wqe;
1620 
1621 	if (!lpfc_is_link_up(phba)) {
1622 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1623 				"6104 NVMET prep LS wqe: link err: "
1624 				"NPORT x%x oxid:x%x ste %d\n",
1625 				ctxp->sid, ctxp->oxid, ctxp->state);
1626 		return NULL;
1627 	}
1628 
1629 	/* Allocate buffer for  command wqe */
1630 	nvmewqe = lpfc_sli_get_iocbq(phba);
1631 	if (nvmewqe == NULL) {
1632 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1633 				"6105 NVMET prep LS wqe: No WQE: "
1634 				"NPORT x%x oxid x%x ste %d\n",
1635 				ctxp->sid, ctxp->oxid, ctxp->state);
1636 		return NULL;
1637 	}
1638 
1639 	ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1640 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1641 	    ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1642 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1643 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_DISC,
1644 				"6106 NVMET prep LS wqe: No ndlp: "
1645 				"NPORT x%x oxid x%x ste %d\n",
1646 				ctxp->sid, ctxp->oxid, ctxp->state);
1647 		goto nvme_wqe_free_wqeq_exit;
1648 	}
1649 	ctxp->wqeq = nvmewqe;
1650 
1651 	/* prevent preparing wqe with NULL ndlp reference */
1652 	nvmewqe->context1 = lpfc_nlp_get(ndlp);
1653 	if (nvmewqe->context1 == NULL)
1654 		goto nvme_wqe_free_wqeq_exit;
1655 	nvmewqe->context2 = ctxp;
1656 
1657 	wqe = &nvmewqe->wqe;
1658 	memset(wqe, 0, sizeof(union lpfc_wqe));
1659 
1660 	/* Words 0 - 2 */
1661 	wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1662 	wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
1663 	wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
1664 	wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
1665 
1666 	/* Word 3 */
1667 
1668 	/* Word 4 */
1669 
1670 	/* Word 5 */
1671 	bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
1672 	bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
1673 	bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
1674 	bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);
1675 	bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
1676 
1677 	/* Word 6 */
1678 	bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
1679 	       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1680 	bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
1681 
1682 	/* Word 7 */
1683 	bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
1684 	       CMD_XMIT_SEQUENCE64_WQE);
1685 	bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
1686 	bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
1687 	bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
1688 
1689 	/* Word 8 */
1690 	wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
1691 
1692 	/* Word 9 */
1693 	bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
1694 	/* Needs to be set by caller */
1695 	bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
1696 
1697 	/* Word 10 */
1698 	bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
1699 	bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
1700 	bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
1701 	       LPFC_WQE_LENLOC_WORD12);
1702 	bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
1703 
1704 	/* Word 11 */
1705 	bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
1706 	       LPFC_WQE_CQ_ID_DEFAULT);
1707 	bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
1708 	       OTHER_COMMAND);
1709 
1710 	/* Word 12 */
1711 	wqe->xmit_sequence.xmit_len = rspsize;
1712 
1713 	nvmewqe->retry = 1;
1714 	nvmewqe->vport = phba->pport;
1715 	nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
1716 	nvmewqe->iocb_flag |= LPFC_IO_NVME_LS;
1717 
1718 	/* Xmit NVMET response to remote NPORT <did> */
1719 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
1720 			"6039 Xmit NVMET LS response to remote "
1721 			"NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
1722 			ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
1723 			rspsize);
1724 	return nvmewqe;
1725 
1726 nvme_wqe_free_wqeq_exit:
1727 	nvmewqe->context2 = NULL;
1728 	nvmewqe->context3 = NULL;
1729 	lpfc_sli_release_iocbq(phba, nvmewqe);
1730 	return NULL;
1731 }
1732 
1733 
1734 static struct lpfc_iocbq *
1735 lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
1736 			struct lpfc_nvmet_rcv_ctx *ctxp)
1737 {
1738 	struct nvmefc_tgt_fcp_req *rsp = &ctxp->ctx.fcp_req;
1739 	struct lpfc_nvmet_tgtport *tgtp;
1740 	struct sli4_sge *sgl;
1741 	struct lpfc_nodelist *ndlp;
1742 	struct lpfc_iocbq *nvmewqe;
1743 	struct scatterlist *sgel;
1744 	union lpfc_wqe128 *wqe;
1745 	uint32_t *txrdy;
1746 	dma_addr_t physaddr;
1747 	int i, cnt;
1748 	int xc = 1;
1749 
1750 	if (!lpfc_is_link_up(phba)) {
1751 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1752 				"6107 NVMET prep FCP wqe: link err:"
1753 				"NPORT x%x oxid x%x ste %d\n",
1754 				ctxp->sid, ctxp->oxid, ctxp->state);
1755 		return NULL;
1756 	}
1757 
1758 	ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
1759 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
1760 	    ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
1761 	     (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
1762 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1763 				"6108 NVMET prep FCP wqe: no ndlp: "
1764 				"NPORT x%x oxid x%x ste %d\n",
1765 				ctxp->sid, ctxp->oxid, ctxp->state);
1766 		return NULL;
1767 	}
1768 
1769 	if (rsp->sg_cnt > phba->cfg_nvme_seg_cnt) {
1770 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1771 				"6109 NVMET prep FCP wqe: seg cnt err: "
1772 				"NPORT x%x oxid x%x ste %d cnt %d\n",
1773 				ctxp->sid, ctxp->oxid, ctxp->state,
1774 				phba->cfg_nvme_seg_cnt);
1775 		return NULL;
1776 	}
1777 
1778 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
1779 	nvmewqe = ctxp->wqeq;
1780 	if (nvmewqe == NULL) {
1781 		/* Allocate buffer for  command wqe */
1782 		nvmewqe = ctxp->ctxbuf->iocbq;
1783 		if (nvmewqe == NULL) {
1784 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1785 					"6110 NVMET prep FCP wqe: No "
1786 					"WQE: NPORT x%x oxid x%x ste %d\n",
1787 					ctxp->sid, ctxp->oxid, ctxp->state);
1788 			return NULL;
1789 		}
1790 		ctxp->wqeq = nvmewqe;
1791 		xc = 0; /* create new XRI */
1792 		nvmewqe->sli4_lxritag = NO_XRI;
1793 		nvmewqe->sli4_xritag = NO_XRI;
1794 	}
1795 
1796 	/* Sanity check */
1797 	if (((ctxp->state == LPFC_NVMET_STE_RCV) &&
1798 	    (ctxp->entry_cnt == 1)) ||
1799 	    (ctxp->state == LPFC_NVMET_STE_DATA)) {
1800 		wqe = (union lpfc_wqe128 *)&nvmewqe->wqe;
1801 	} else {
1802 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1803 				"6111 Wrong state NVMET FCP: %d  cnt %d\n",
1804 				ctxp->state, ctxp->entry_cnt);
1805 		return NULL;
1806 	}
1807 
1808 	sgl  = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl;
1809 	switch (rsp->op) {
1810 	case NVMET_FCOP_READDATA:
1811 	case NVMET_FCOP_READDATA_RSP:
1812 		/* Words 0 - 2 : The first sg segment */
1813 		sgel = &rsp->sg[0];
1814 		physaddr = sg_dma_address(sgel);
1815 		wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1816 		wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
1817 		wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
1818 		wqe->fcp_tsend.bde.addrHigh =
1819 			cpu_to_le32(putPaddrHigh(physaddr));
1820 
1821 		/* Word 3 */
1822 		wqe->fcp_tsend.payload_offset_len = 0;
1823 
1824 		/* Word 4 */
1825 		wqe->fcp_tsend.relative_offset = ctxp->offset;
1826 
1827 		/* Word 5 */
1828 
1829 		/* Word 6 */
1830 		bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
1831 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1832 		bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
1833 		       nvmewqe->sli4_xritag);
1834 
1835 		/* Word 7 */
1836 		bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
1837 
1838 		/* Word 8 */
1839 		wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
1840 
1841 		/* Word 9 */
1842 		bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
1843 		bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
1844 
1845 		/* Word 10 */
1846 		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
1847 		bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
1848 		bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
1849 		bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com,
1850 		       LPFC_WQE_LENLOC_WORD12);
1851 		bf_set(wqe_ebde_cnt, &wqe->fcp_tsend.wqe_com, 0);
1852 		bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, xc);
1853 		bf_set(wqe_nvme, &wqe->fcp_tsend.wqe_com, 1);
1854 		if (phba->cfg_nvme_oas)
1855 			bf_set(wqe_oas, &wqe->fcp_tsend.wqe_com, 1);
1856 
1857 		/* Word 11 */
1858 		bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com,
1859 		       LPFC_WQE_CQ_ID_DEFAULT);
1860 		bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com,
1861 		       FCP_COMMAND_TSEND);
1862 
1863 		/* Word 12 */
1864 		wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
1865 
1866 		/* Setup 2 SKIP SGEs */
1867 		sgl->addr_hi = 0;
1868 		sgl->addr_lo = 0;
1869 		sgl->word2 = 0;
1870 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1871 		sgl->word2 = cpu_to_le32(sgl->word2);
1872 		sgl->sge_len = 0;
1873 		sgl++;
1874 		sgl->addr_hi = 0;
1875 		sgl->addr_lo = 0;
1876 		sgl->word2 = 0;
1877 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1878 		sgl->word2 = cpu_to_le32(sgl->word2);
1879 		sgl->sge_len = 0;
1880 		sgl++;
1881 		if (rsp->op == NVMET_FCOP_READDATA_RSP) {
1882 			atomic_inc(&tgtp->xmt_fcp_read_rsp);
1883 			bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
1884 			if ((ndlp->nlp_flag & NLP_SUPPRESS_RSP) &&
1885 			    (rsp->rsplen == 12)) {
1886 				bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 1);
1887 				bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
1888 				bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
1889 				bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
1890 			} else {
1891 				bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1892 				bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
1893 				bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
1894 				bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
1895 				       ((rsp->rsplen >> 2) - 1));
1896 				memcpy(&wqe->words[16], rsp->rspaddr,
1897 				       rsp->rsplen);
1898 			}
1899 		} else {
1900 			atomic_inc(&tgtp->xmt_fcp_read);
1901 
1902 			bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1903 			bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
1904 			bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
1905 			bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
1906 			bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
1907 		}
1908 		break;
1909 
1910 	case NVMET_FCOP_WRITEDATA:
1911 		/* Words 0 - 2 : The first sg segment */
1912 		txrdy = pci_pool_alloc(phba->txrdy_payload_pool,
1913 				       GFP_KERNEL, &physaddr);
1914 		if (!txrdy) {
1915 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1916 					"6041 Bad txrdy buffer: oxid x%x\n",
1917 					ctxp->oxid);
1918 			return NULL;
1919 		}
1920 		ctxp->txrdy = txrdy;
1921 		ctxp->txrdy_phys = physaddr;
1922 		wqe->fcp_treceive.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1923 		wqe->fcp_treceive.bde.tus.f.bdeSize = TXRDY_PAYLOAD_LEN;
1924 		wqe->fcp_treceive.bde.addrLow =
1925 			cpu_to_le32(putPaddrLow(physaddr));
1926 		wqe->fcp_treceive.bde.addrHigh =
1927 			cpu_to_le32(putPaddrHigh(physaddr));
1928 
1929 		/* Word 3 */
1930 		wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
1931 
1932 		/* Word 4 */
1933 		wqe->fcp_treceive.relative_offset = ctxp->offset;
1934 
1935 		/* Word 5 */
1936 
1937 		/* Word 6 */
1938 		bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
1939 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
1940 		bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
1941 		       nvmewqe->sli4_xritag);
1942 
1943 		/* Word 7 */
1944 		bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
1945 		bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com,
1946 		       CMD_FCP_TRECEIVE64_WQE);
1947 
1948 		/* Word 8 */
1949 		wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
1950 
1951 		/* Word 9 */
1952 		bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
1953 		bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
1954 
1955 		/* Word 10 */
1956 		bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
1957 		bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
1958 		bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
1959 		bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com,
1960 		       LPFC_WQE_LENLOC_WORD12);
1961 		bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, xc);
1962 		bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
1963 		bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
1964 		bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
1965 		bf_set(wqe_nvme, &wqe->fcp_treceive.wqe_com, 1);
1966 		if (phba->cfg_nvme_oas)
1967 			bf_set(wqe_oas, &wqe->fcp_treceive.wqe_com, 1);
1968 
1969 		/* Word 11 */
1970 		bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com,
1971 		       LPFC_WQE_CQ_ID_DEFAULT);
1972 		bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com,
1973 		       FCP_COMMAND_TRECEIVE);
1974 		bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
1975 
1976 		/* Word 12 */
1977 		wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
1978 
1979 		/* Setup 1 TXRDY and 1 SKIP SGE */
1980 		txrdy[0] = 0;
1981 		txrdy[1] = cpu_to_be32(rsp->transfer_length);
1982 		txrdy[2] = 0;
1983 
1984 		sgl->addr_hi = putPaddrHigh(physaddr);
1985 		sgl->addr_lo = putPaddrLow(physaddr);
1986 		sgl->word2 = 0;
1987 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
1988 		sgl->word2 = cpu_to_le32(sgl->word2);
1989 		sgl->sge_len = cpu_to_le32(TXRDY_PAYLOAD_LEN);
1990 		sgl++;
1991 		sgl->addr_hi = 0;
1992 		sgl->addr_lo = 0;
1993 		sgl->word2 = 0;
1994 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
1995 		sgl->word2 = cpu_to_le32(sgl->word2);
1996 		sgl->sge_len = 0;
1997 		sgl++;
1998 		atomic_inc(&tgtp->xmt_fcp_write);
1999 		break;
2000 
2001 	case NVMET_FCOP_RSP:
2002 		/* Words 0 - 2 */
2003 		physaddr = rsp->rspdma;
2004 		wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
2005 		wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
2006 		wqe->fcp_trsp.bde.addrLow =
2007 			cpu_to_le32(putPaddrLow(physaddr));
2008 		wqe->fcp_trsp.bde.addrHigh =
2009 			cpu_to_le32(putPaddrHigh(physaddr));
2010 
2011 		/* Word 3 */
2012 		wqe->fcp_trsp.response_len = rsp->rsplen;
2013 
2014 		/* Word 4 */
2015 		wqe->fcp_trsp.rsvd_4_5[0] = 0;
2016 
2017 
2018 		/* Word 5 */
2019 
2020 		/* Word 6 */
2021 		bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
2022 		       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2023 		bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
2024 		       nvmewqe->sli4_xritag);
2025 
2026 		/* Word 7 */
2027 		bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1);
2028 		bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
2029 
2030 		/* Word 8 */
2031 		wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
2032 
2033 		/* Word 9 */
2034 		bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
2035 		bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
2036 
2037 		/* Word 10 */
2038 		bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
2039 		bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 0);
2040 		bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_WRITE);
2041 		bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com,
2042 		       LPFC_WQE_LENLOC_WORD3);
2043 		bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, xc);
2044 		bf_set(wqe_nvme, &wqe->fcp_trsp.wqe_com, 1);
2045 		if (phba->cfg_nvme_oas)
2046 			bf_set(wqe_oas, &wqe->fcp_trsp.wqe_com, 1);
2047 
2048 		/* Word 11 */
2049 		bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com,
2050 		       LPFC_WQE_CQ_ID_DEFAULT);
2051 		bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com,
2052 		       FCP_COMMAND_TRSP);
2053 		bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
2054 
2055 		if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
2056 			/* Good response - all zero's on wire */
2057 			bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
2058 			bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
2059 			bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
2060 		} else {
2061 			bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
2062 			bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
2063 			bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
2064 			       ((rsp->rsplen >> 2) - 1));
2065 			memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
2066 		}
2067 
2068 		/* Use rspbuf, NOT sg list */
2069 		rsp->sg_cnt = 0;
2070 		sgl->word2 = 0;
2071 		atomic_inc(&tgtp->xmt_fcp_rsp);
2072 		break;
2073 
2074 	default:
2075 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2076 				"6064 Unknown Rsp Op %d\n",
2077 				rsp->op);
2078 		return NULL;
2079 	}
2080 
2081 	nvmewqe->retry = 1;
2082 	nvmewqe->vport = phba->pport;
2083 	nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
2084 	nvmewqe->context1 = ndlp;
2085 
2086 	for (i = 0; i < rsp->sg_cnt; i++) {
2087 		sgel = &rsp->sg[i];
2088 		physaddr = sg_dma_address(sgel);
2089 		cnt = sg_dma_len(sgel);
2090 		sgl->addr_hi = putPaddrHigh(physaddr);
2091 		sgl->addr_lo = putPaddrLow(physaddr);
2092 		sgl->word2 = 0;
2093 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
2094 		bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
2095 		if ((i+1) == rsp->sg_cnt)
2096 			bf_set(lpfc_sli4_sge_last, sgl, 1);
2097 		sgl->word2 = cpu_to_le32(sgl->word2);
2098 		sgl->sge_len = cpu_to_le32(cnt);
2099 		sgl++;
2100 		ctxp->offset += cnt;
2101 	}
2102 	ctxp->state = LPFC_NVMET_STE_DATA;
2103 	ctxp->entry_cnt++;
2104 	return nvmewqe;
2105 }
2106 
2107 /**
2108  * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
2109  * @phba: Pointer to HBA context object.
2110  * @cmdwqe: Pointer to driver command WQE object.
2111  * @wcqe: Pointer to driver response CQE object.
2112  *
2113  * The function is called from SLI ring event handler with no
2114  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
2115  * The function frees memory resources used for the NVME commands.
2116  **/
2117 static void
2118 lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2119 			     struct lpfc_wcqe_complete *wcqe)
2120 {
2121 	struct lpfc_nvmet_rcv_ctx *ctxp;
2122 	struct lpfc_nvmet_tgtport *tgtp;
2123 	uint32_t status, result;
2124 	unsigned long flags;
2125 	bool released = false;
2126 
2127 	ctxp = cmdwqe->context2;
2128 	status = bf_get(lpfc_wcqe_c_status, wcqe);
2129 	result = wcqe->parameter;
2130 
2131 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2132 	if (ctxp->flag & LPFC_NVMET_ABORT_OP)
2133 		atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
2134 
2135 	ctxp->state = LPFC_NVMET_STE_DONE;
2136 
2137 	/* Check if we already received a free context call
2138 	 * and we have completed processing an abort situation.
2139 	 */
2140 	spin_lock_irqsave(&ctxp->ctxlock, flags);
2141 	if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
2142 	    !(ctxp->flag & LPFC_NVMET_XBUSY)) {
2143 		list_del(&ctxp->list);
2144 		released = true;
2145 	}
2146 	ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2147 	spin_unlock_irqrestore(&ctxp->ctxlock, flags);
2148 	atomic_inc(&tgtp->xmt_abort_rsp);
2149 
2150 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2151 			"6165 ABORT cmpl: xri x%x flg x%x (%d) "
2152 			"WCQE: %08x %08x %08x %08x\n",
2153 			ctxp->oxid, ctxp->flag, released,
2154 			wcqe->word0, wcqe->total_data_placed,
2155 			result, wcqe->word3);
2156 
2157 	cmdwqe->context2 = NULL;
2158 	cmdwqe->context3 = NULL;
2159 	/*
2160 	 * if transport has released ctx, then can reuse it. Otherwise,
2161 	 * will be recycled by transport release call.
2162 	 */
2163 	if (released)
2164 		lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
2165 
2166 	/* This is the iocbq for the abort, not the command */
2167 	lpfc_sli_release_iocbq(phba, cmdwqe);
2168 
2169 	/* Since iaab/iaar are NOT set, there is no work left.
2170 	 * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
2171 	 * should have been called already.
2172 	 */
2173 }
2174 
2175 /**
2176  * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS
2177  * @phba: Pointer to HBA context object.
2178  * @cmdwqe: Pointer to driver command WQE object.
2179  * @wcqe: Pointer to driver response CQE object.
2180  *
2181  * The function is called from SLI ring event handler with no
2182  * lock held. This function is the completion handler for NVME ABTS for FCP cmds
2183  * The function frees memory resources used for the NVME commands.
2184  **/
2185 static void
2186 lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2187 			       struct lpfc_wcqe_complete *wcqe)
2188 {
2189 	struct lpfc_nvmet_rcv_ctx *ctxp;
2190 	struct lpfc_nvmet_tgtport *tgtp;
2191 	unsigned long flags;
2192 	uint32_t status, result;
2193 	bool released = false;
2194 
2195 	ctxp = cmdwqe->context2;
2196 	status = bf_get(lpfc_wcqe_c_status, wcqe);
2197 	result = wcqe->parameter;
2198 
2199 	if (!ctxp) {
2200 		/* if context is clear, related io alrady complete */
2201 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2202 				"6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",
2203 				wcqe->word0, wcqe->total_data_placed,
2204 				result, wcqe->word3);
2205 		return;
2206 	}
2207 
2208 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2209 	if (ctxp->flag & LPFC_NVMET_ABORT_OP)
2210 		atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
2211 
2212 	/* Sanity check */
2213 	if (ctxp->state != LPFC_NVMET_STE_ABORT) {
2214 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2215 				"6112 ABTS Wrong state:%d oxid x%x\n",
2216 				ctxp->state, ctxp->oxid);
2217 	}
2218 
2219 	/* Check if we already received a free context call
2220 	 * and we have completed processing an abort situation.
2221 	 */
2222 	ctxp->state = LPFC_NVMET_STE_DONE;
2223 	spin_lock_irqsave(&ctxp->ctxlock, flags);
2224 	if ((ctxp->flag & LPFC_NVMET_CTX_RLS) &&
2225 	    !(ctxp->flag & LPFC_NVMET_XBUSY)) {
2226 		list_del(&ctxp->list);
2227 		released = true;
2228 	}
2229 	ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2230 	spin_unlock_irqrestore(&ctxp->ctxlock, flags);
2231 	atomic_inc(&tgtp->xmt_abort_rsp);
2232 
2233 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2234 			"6316 ABTS cmpl xri x%x flg x%x (%x) "
2235 			"WCQE: %08x %08x %08x %08x\n",
2236 			ctxp->oxid, ctxp->flag, released,
2237 			wcqe->word0, wcqe->total_data_placed,
2238 			result, wcqe->word3);
2239 
2240 	cmdwqe->context2 = NULL;
2241 	cmdwqe->context3 = NULL;
2242 	/*
2243 	 * if transport has released ctx, then can reuse it. Otherwise,
2244 	 * will be recycled by transport release call.
2245 	 */
2246 	if (released)
2247 		lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
2248 
2249 	/* Since iaab/iaar are NOT set, there is no work left.
2250 	 * For LPFC_NVMET_XBUSY, lpfc_sli4_nvmet_xri_aborted
2251 	 * should have been called already.
2252 	 */
2253 }
2254 
2255 /**
2256  * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
2257  * @phba: Pointer to HBA context object.
2258  * @cmdwqe: Pointer to driver command WQE object.
2259  * @wcqe: Pointer to driver response CQE object.
2260  *
2261  * The function is called from SLI ring event handler with no
2262  * lock held. This function is the completion handler for NVME ABTS for LS cmds
2263  * The function frees memory resources used for the NVME commands.
2264  **/
2265 static void
2266 lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
2267 			    struct lpfc_wcqe_complete *wcqe)
2268 {
2269 	struct lpfc_nvmet_rcv_ctx *ctxp;
2270 	struct lpfc_nvmet_tgtport *tgtp;
2271 	uint32_t status, result;
2272 
2273 	ctxp = cmdwqe->context2;
2274 	status = bf_get(lpfc_wcqe_c_status, wcqe);
2275 	result = wcqe->parameter;
2276 
2277 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2278 	atomic_inc(&tgtp->xmt_ls_abort_cmpl);
2279 
2280 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2281 			"6083 Abort cmpl: ctx %p WCQE:%08x %08x %08x %08x\n",
2282 			ctxp, wcqe->word0, wcqe->total_data_placed,
2283 			result, wcqe->word3);
2284 
2285 	if (!ctxp) {
2286 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2287 				"6415 NVMET LS Abort No ctx: WCQE: "
2288 				 "%08x %08x %08x %08x\n",
2289 				wcqe->word0, wcqe->total_data_placed,
2290 				result, wcqe->word3);
2291 
2292 		lpfc_sli_release_iocbq(phba, cmdwqe);
2293 		return;
2294 	}
2295 
2296 	if (ctxp->state != LPFC_NVMET_STE_LS_ABORT) {
2297 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2298 				"6416 NVMET LS abort cmpl state mismatch: "
2299 				"oxid x%x: %d %d\n",
2300 				ctxp->oxid, ctxp->state, ctxp->entry_cnt);
2301 	}
2302 
2303 	cmdwqe->context2 = NULL;
2304 	cmdwqe->context3 = NULL;
2305 	lpfc_sli_release_iocbq(phba, cmdwqe);
2306 	kfree(ctxp);
2307 }
2308 
2309 static int
2310 lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
2311 			     struct lpfc_nvmet_rcv_ctx *ctxp,
2312 			     uint32_t sid, uint16_t xri)
2313 {
2314 	struct lpfc_nvmet_tgtport *tgtp;
2315 	struct lpfc_iocbq *abts_wqeq;
2316 	union lpfc_wqe *wqe_abts;
2317 	struct lpfc_nodelist *ndlp;
2318 
2319 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2320 			"6067 ABTS: sid %x xri x%x/x%x\n",
2321 			sid, xri, ctxp->wqeq->sli4_xritag);
2322 
2323 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2324 
2325 	ndlp = lpfc_findnode_did(phba->pport, sid);
2326 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2327 	    ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2328 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2329 		atomic_inc(&tgtp->xmt_abort_rsp_error);
2330 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2331 				"6134 Drop ABTS - wrong NDLP state x%x.\n",
2332 				(ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2333 
2334 		/* No failure to an ABTS request. */
2335 		return 0;
2336 	}
2337 
2338 	abts_wqeq = ctxp->wqeq;
2339 	wqe_abts = &abts_wqeq->wqe;
2340 
2341 	/*
2342 	 * Since we zero the whole WQE, we need to ensure we set the WQE fields
2343 	 * that were initialized in lpfc_sli4_nvmet_alloc.
2344 	 */
2345 	memset(wqe_abts, 0, sizeof(union lpfc_wqe));
2346 
2347 	/* Word 5 */
2348 	bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
2349 	bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
2350 	bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
2351 	bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
2352 	bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
2353 
2354 	/* Word 6 */
2355 	bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
2356 	       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
2357 	bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
2358 	       abts_wqeq->sli4_xritag);
2359 
2360 	/* Word 7 */
2361 	bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
2362 	       CMD_XMIT_SEQUENCE64_WQE);
2363 	bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
2364 	bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
2365 	bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
2366 
2367 	/* Word 8 */
2368 	wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
2369 
2370 	/* Word 9 */
2371 	bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
2372 	/* Needs to be set by caller */
2373 	bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
2374 
2375 	/* Word 10 */
2376 	bf_set(wqe_dbde, &wqe_abts->xmit_sequence.wqe_com, 1);
2377 	bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
2378 	bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
2379 	       LPFC_WQE_LENLOC_WORD12);
2380 	bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
2381 	bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
2382 
2383 	/* Word 11 */
2384 	bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
2385 	       LPFC_WQE_CQ_ID_DEFAULT);
2386 	bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
2387 	       OTHER_COMMAND);
2388 
2389 	abts_wqeq->vport = phba->pport;
2390 	abts_wqeq->context1 = ndlp;
2391 	abts_wqeq->context2 = ctxp;
2392 	abts_wqeq->context3 = NULL;
2393 	abts_wqeq->rsvd2 = 0;
2394 	/* hba_wqidx should already be setup from command we are aborting */
2395 	abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
2396 	abts_wqeq->iocb.ulpLe = 1;
2397 
2398 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2399 			"6069 Issue ABTS to xri x%x reqtag x%x\n",
2400 			xri, abts_wqeq->iotag);
2401 	return 1;
2402 }
2403 
2404 static int
2405 lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
2406 			       struct lpfc_nvmet_rcv_ctx *ctxp,
2407 			       uint32_t sid, uint16_t xri)
2408 {
2409 	struct lpfc_nvmet_tgtport *tgtp;
2410 	struct lpfc_iocbq *abts_wqeq;
2411 	union lpfc_wqe *abts_wqe;
2412 	struct lpfc_nodelist *ndlp;
2413 	unsigned long flags;
2414 	int rc;
2415 
2416 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2417 	if (!ctxp->wqeq) {
2418 		ctxp->wqeq = ctxp->ctxbuf->iocbq;
2419 		ctxp->wqeq->hba_wqidx = 0;
2420 	}
2421 
2422 	ndlp = lpfc_findnode_did(phba->pport, sid);
2423 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp) ||
2424 	    ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
2425 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
2426 		atomic_inc(&tgtp->xmt_abort_rsp_error);
2427 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2428 				"6160 Drop ABORT - wrong NDLP state x%x.\n",
2429 				(ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
2430 
2431 		/* No failure to an ABTS request. */
2432 		ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2433 		return 0;
2434 	}
2435 
2436 	/* Issue ABTS for this WQE based on iotag */
2437 	ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
2438 	if (!ctxp->abort_wqeq) {
2439 		atomic_inc(&tgtp->xmt_abort_rsp_error);
2440 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2441 				"6161 ABORT failed: No wqeqs: "
2442 				"xri: x%x\n", ctxp->oxid);
2443 		/* No failure to an ABTS request. */
2444 		ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2445 		return 0;
2446 	}
2447 	abts_wqeq = ctxp->abort_wqeq;
2448 	abts_wqe = &abts_wqeq->wqe;
2449 	ctxp->state = LPFC_NVMET_STE_ABORT;
2450 
2451 	/* Announce entry to new IO submit field. */
2452 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2453 			"6162 ABORT Request to rport DID x%06x "
2454 			"for xri x%x x%x\n",
2455 			ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
2456 
2457 	/* If the hba is getting reset, this flag is set.  It is
2458 	 * cleared when the reset is complete and rings reestablished.
2459 	 */
2460 	spin_lock_irqsave(&phba->hbalock, flags);
2461 	/* driver queued commands are in process of being flushed */
2462 	if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) {
2463 		spin_unlock_irqrestore(&phba->hbalock, flags);
2464 		atomic_inc(&tgtp->xmt_abort_rsp_error);
2465 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
2466 				"6163 Driver in reset cleanup - flushing "
2467 				"NVME Req now. hba_flag x%x oxid x%x\n",
2468 				phba->hba_flag, ctxp->oxid);
2469 		lpfc_sli_release_iocbq(phba, abts_wqeq);
2470 		ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2471 		return 0;
2472 	}
2473 
2474 	/* Outstanding abort is in progress */
2475 	if (abts_wqeq->iocb_flag & LPFC_DRIVER_ABORTED) {
2476 		spin_unlock_irqrestore(&phba->hbalock, flags);
2477 		atomic_inc(&tgtp->xmt_abort_rsp_error);
2478 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
2479 				"6164 Outstanding NVME I/O Abort Request "
2480 				"still pending on oxid x%x\n",
2481 				ctxp->oxid);
2482 		lpfc_sli_release_iocbq(phba, abts_wqeq);
2483 		ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2484 		return 0;
2485 	}
2486 
2487 	/* Ready - mark outstanding as aborted by driver. */
2488 	abts_wqeq->iocb_flag |= LPFC_DRIVER_ABORTED;
2489 
2490 	/* WQEs are reused.  Clear stale data and set key fields to
2491 	 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
2492 	 */
2493 	memset(abts_wqe, 0, sizeof(union lpfc_wqe));
2494 
2495 	/* word 3 */
2496 	bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
2497 
2498 	/* word 7 */
2499 	bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0);
2500 	bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
2501 
2502 	/* word 8 - tell the FW to abort the IO associated with this
2503 	 * outstanding exchange ID.
2504 	 */
2505 	abts_wqe->abort_cmd.wqe_com.abort_tag = ctxp->wqeq->sli4_xritag;
2506 
2507 	/* word 9 - this is the iotag for the abts_wqe completion. */
2508 	bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
2509 	       abts_wqeq->iotag);
2510 
2511 	/* word 10 */
2512 	bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
2513 	bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
2514 
2515 	/* word 11 */
2516 	bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
2517 	bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
2518 	bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
2519 
2520 	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
2521 	abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
2522 	abts_wqeq->wqe_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
2523 	abts_wqeq->iocb_cmpl = 0;
2524 	abts_wqeq->iocb_flag |= LPFC_IO_NVME;
2525 	abts_wqeq->context2 = ctxp;
2526 	abts_wqeq->vport = phba->pport;
2527 	rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
2528 	spin_unlock_irqrestore(&phba->hbalock, flags);
2529 	if (rc == WQE_SUCCESS) {
2530 		atomic_inc(&tgtp->xmt_abort_sol);
2531 		return 0;
2532 	}
2533 
2534 	atomic_inc(&tgtp->xmt_abort_rsp_error);
2535 	ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2536 	lpfc_sli_release_iocbq(phba, abts_wqeq);
2537 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2538 			"6166 Failed ABORT issue_wqe with status x%x "
2539 			"for oxid x%x.\n",
2540 			rc, ctxp->oxid);
2541 	return 1;
2542 }
2543 
2544 
2545 static int
2546 lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
2547 				 struct lpfc_nvmet_rcv_ctx *ctxp,
2548 				 uint32_t sid, uint16_t xri)
2549 {
2550 	struct lpfc_nvmet_tgtport *tgtp;
2551 	struct lpfc_iocbq *abts_wqeq;
2552 	unsigned long flags;
2553 	int rc;
2554 
2555 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2556 	if (!ctxp->wqeq) {
2557 		ctxp->wqeq = ctxp->ctxbuf->iocbq;
2558 		ctxp->wqeq->hba_wqidx = 0;
2559 	}
2560 
2561 	if (ctxp->state == LPFC_NVMET_STE_FREE) {
2562 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2563 				"6417 NVMET ABORT ctx freed %d %d oxid x%x\n",
2564 				ctxp->state, ctxp->entry_cnt, ctxp->oxid);
2565 		rc = WQE_BUSY;
2566 		goto aerr;
2567 	}
2568 	ctxp->state = LPFC_NVMET_STE_ABORT;
2569 	ctxp->entry_cnt++;
2570 	rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
2571 	if (rc == 0)
2572 		goto aerr;
2573 
2574 	spin_lock_irqsave(&phba->hbalock, flags);
2575 	abts_wqeq = ctxp->wqeq;
2576 	abts_wqeq->wqe_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;
2577 	abts_wqeq->iocb_cmpl = NULL;
2578 	abts_wqeq->iocb_flag |= LPFC_IO_NVMET;
2579 	rc = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_wqeq);
2580 	spin_unlock_irqrestore(&phba->hbalock, flags);
2581 	if (rc == WQE_SUCCESS) {
2582 		return 0;
2583 	}
2584 
2585 aerr:
2586 	ctxp->flag &= ~LPFC_NVMET_ABORT_OP;
2587 	atomic_inc(&tgtp->xmt_abort_rsp_error);
2588 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2589 			"6135 Failed to Issue ABTS for oxid x%x. Status x%x\n",
2590 			ctxp->oxid, rc);
2591 	return 1;
2592 }
2593 
2594 static int
2595 lpfc_nvmet_unsol_ls_issue_abort(struct lpfc_hba *phba,
2596 				struct lpfc_nvmet_rcv_ctx *ctxp,
2597 				uint32_t sid, uint16_t xri)
2598 {
2599 	struct lpfc_nvmet_tgtport *tgtp;
2600 	struct lpfc_iocbq *abts_wqeq;
2601 	union lpfc_wqe *wqe_abts;
2602 	unsigned long flags;
2603 	int rc;
2604 
2605 	if ((ctxp->state == LPFC_NVMET_STE_LS_RCV && ctxp->entry_cnt == 1) ||
2606 	    (ctxp->state == LPFC_NVMET_STE_LS_RSP && ctxp->entry_cnt == 2)) {
2607 		ctxp->state = LPFC_NVMET_STE_LS_ABORT;
2608 		ctxp->entry_cnt++;
2609 	} else {
2610 		lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2611 				"6418 NVMET LS abort state mismatch "
2612 				"IO x%x: %d %d\n",
2613 				ctxp->oxid, ctxp->state, ctxp->entry_cnt);
2614 		ctxp->state = LPFC_NVMET_STE_LS_ABORT;
2615 	}
2616 
2617 	tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
2618 	if (!ctxp->wqeq) {
2619 		/* Issue ABTS for this WQE based on iotag */
2620 		ctxp->wqeq = lpfc_sli_get_iocbq(phba);
2621 		if (!ctxp->wqeq) {
2622 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2623 					"6068 Abort failed: No wqeqs: "
2624 					"xri: x%x\n", xri);
2625 			/* No failure to an ABTS request. */
2626 			kfree(ctxp);
2627 			return 0;
2628 		}
2629 	}
2630 	abts_wqeq = ctxp->wqeq;
2631 	wqe_abts = &abts_wqeq->wqe;
2632 
2633 	if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {
2634 		rc = WQE_BUSY;
2635 		goto out;
2636 	}
2637 
2638 	spin_lock_irqsave(&phba->hbalock, flags);
2639 	abts_wqeq->wqe_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
2640 	abts_wqeq->iocb_cmpl = 0;
2641 	abts_wqeq->iocb_flag |=  LPFC_IO_NVME_LS;
2642 	rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, abts_wqeq);
2643 	spin_unlock_irqrestore(&phba->hbalock, flags);
2644 	if (rc == WQE_SUCCESS) {
2645 		atomic_inc(&tgtp->xmt_abort_unsol);
2646 		return 0;
2647 	}
2648 out:
2649 	atomic_inc(&tgtp->xmt_abort_rsp_error);
2650 	abts_wqeq->context2 = NULL;
2651 	abts_wqeq->context3 = NULL;
2652 	lpfc_sli_release_iocbq(phba, abts_wqeq);
2653 	kfree(ctxp);
2654 	lpfc_printf_log(phba, KERN_ERR, LOG_NVME_ABTS,
2655 			"6056 Failed to Issue ABTS. Status x%x\n", rc);
2656 	return 0;
2657 }
2658