xref: /openbmc/linux/drivers/scsi/lpfc/lpfc_nvme.c (revision add48ba4)
1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channel Host Bus Adapters.                                *
4  * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30 
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38 
39 #include "lpfc_version.h"
40 #include "lpfc_hw4.h"
41 #include "lpfc_hw.h"
42 #include "lpfc_sli.h"
43 #include "lpfc_sli4.h"
44 #include "lpfc_nl.h"
45 #include "lpfc_disc.h"
46 #include "lpfc.h"
47 #include "lpfc_nvme.h"
48 #include "lpfc_scsi.h"
49 #include "lpfc_logmsg.h"
50 #include "lpfc_crtn.h"
51 #include "lpfc_vport.h"
52 #include "lpfc_debugfs.h"
53 
54 /* NVME initiator-based functions */
55 
56 static struct lpfc_io_buf *
57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 		  int idx, int expedite);
59 
60 static void
61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62 
63 static struct nvme_fc_port_template lpfc_nvme_template;
64 
65 static union lpfc_wqe128 lpfc_iread_cmd_template;
66 static union lpfc_wqe128 lpfc_iwrite_cmd_template;
67 static union lpfc_wqe128 lpfc_icmnd_cmd_template;
68 
69 /* Setup WQE templates for NVME IOs */
70 void
71 lpfc_nvme_cmd_template(void)
72 {
73 	union lpfc_wqe128 *wqe;
74 
75 	/* IREAD template */
76 	wqe = &lpfc_iread_cmd_template;
77 	memset(wqe, 0, sizeof(union lpfc_wqe128));
78 
79 	/* Word 0, 1, 2 - BDE is variable */
80 
81 	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
82 
83 	/* Word 4 - total_xfer_len is variable */
84 
85 	/* Word 5 - is zero */
86 
87 	/* Word 6 - ctxt_tag, xri_tag is variable */
88 
89 	/* Word 7 */
90 	bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
91 	bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
92 	bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
93 	bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
94 
95 	/* Word 8 - abort_tag is variable */
96 
97 	/* Word 9  - reqtag is variable */
98 
99 	/* Word 10 - dbde, wqes is variable */
100 	bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
101 	bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1);
102 	bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
103 	bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
104 	bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
105 	bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
106 
107 	/* Word 11 - pbde is variable */
108 	bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD);
109 	bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
110 	bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
111 
112 	/* Word 12 - is zero */
113 
114 	/* Word 13, 14, 15 - PBDE is variable */
115 
116 	/* IWRITE template */
117 	wqe = &lpfc_iwrite_cmd_template;
118 	memset(wqe, 0, sizeof(union lpfc_wqe128));
119 
120 	/* Word 0, 1, 2 - BDE is variable */
121 
122 	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
123 
124 	/* Word 4 - total_xfer_len is variable */
125 
126 	/* Word 5 - initial_xfer_len is variable */
127 
128 	/* Word 6 - ctxt_tag, xri_tag is variable */
129 
130 	/* Word 7 */
131 	bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
132 	bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
133 	bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
134 	bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
135 
136 	/* Word 8 - abort_tag is variable */
137 
138 	/* Word 9  - reqtag is variable */
139 
140 	/* Word 10 - dbde, wqes is variable */
141 	bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
142 	bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1);
143 	bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
144 	bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
145 	bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
146 	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
147 
148 	/* Word 11 - pbde is variable */
149 	bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD);
150 	bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
151 	bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
152 
153 	/* Word 12 - is zero */
154 
155 	/* Word 13, 14, 15 - PBDE is variable */
156 
157 	/* ICMND template */
158 	wqe = &lpfc_icmnd_cmd_template;
159 	memset(wqe, 0, sizeof(union lpfc_wqe128));
160 
161 	/* Word 0, 1, 2 - BDE is variable */
162 
163 	/* Word 3 - payload_offset_len is variable */
164 
165 	/* Word 4, 5 - is zero */
166 
167 	/* Word 6 - ctxt_tag, xri_tag is variable */
168 
169 	/* Word 7 */
170 	bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
171 	bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
172 	bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
173 	bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
174 
175 	/* Word 8 - abort_tag is variable */
176 
177 	/* Word 9  - reqtag is variable */
178 
179 	/* Word 10 - dbde, wqes is variable */
180 	bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
181 	bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
182 	bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
183 	bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
184 	bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
185 	bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
186 
187 	/* Word 11 */
188 	bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND);
189 	bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
190 	bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
191 
192 	/* Word 12, 13, 14, 15 - is zero */
193 }
194 
195 /**
196  * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry.
197  * @pwqeq: Pointer to command iocb.
198  * @xritag: Tag that  uniqely identifies the local exchange resource.
199  * @opt: Option bits -
200  *		bit 0 = inhibit sending abts on the link
201  *
202  * This function is called with hbalock held.
203  **/
204 void
205 lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt)
206 {
207 	union lpfc_wqe128 *wqe = &pwqeq->wqe;
208 
209 	/* WQEs are reused.  Clear stale data and set key fields to
210 	 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
211 	 */
212 	memset(wqe, 0, sizeof(*wqe));
213 
214 	if (opt & INHIBIT_ABORT)
215 		bf_set(abort_cmd_ia, &wqe->abort_cmd, 1);
216 	/* Abort specified xri tag, with the mask deliberately zeroed */
217 	bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG);
218 
219 	bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
220 
221 	/* Abort the IO associated with this outstanding exchange ID. */
222 	wqe->abort_cmd.wqe_com.abort_tag = xritag;
223 
224 	/* iotag for the wqe completion. */
225 	bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag);
226 
227 	bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1);
228 	bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
229 
230 	bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND);
231 	bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1);
232 	bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
233 }
234 
235 /**
236  * lpfc_nvme_create_queue -
237  * @lpfc_pnvme: Pointer to the driver's nvme instance data
238  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
239  * @handle: An opaque driver handle used in follow-up calls.
240  *
241  * Driver registers this routine to preallocate and initialize any
242  * internal data structures to bind the @qidx to its internal IO queues.
243  * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
244  *
245  * Return value :
246  *   0 - Success
247  *   -EINVAL - Unsupported input value.
248  *   -ENOMEM - Could not alloc necessary memory
249  **/
250 static int
251 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
252 		       unsigned int qidx, u16 qsize,
253 		       void **handle)
254 {
255 	struct lpfc_nvme_lport *lport;
256 	struct lpfc_vport *vport;
257 	struct lpfc_nvme_qhandle *qhandle;
258 	char *str;
259 
260 	if (!pnvme_lport->private)
261 		return -ENOMEM;
262 
263 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
264 	vport = lport->vport;
265 	qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
266 	if (qhandle == NULL)
267 		return -ENOMEM;
268 
269 	qhandle->cpu_id = raw_smp_processor_id();
270 	qhandle->qidx = qidx;
271 	/*
272 	 * NVME qidx == 0 is the admin queue, so both admin queue
273 	 * and first IO queue will use MSI-X vector and associated
274 	 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
275 	 */
276 	if (qidx) {
277 		str = "IO ";  /* IO queue */
278 		qhandle->index = ((qidx - 1) %
279 			lpfc_nvme_template.max_hw_queues);
280 	} else {
281 		str = "ADM";  /* Admin queue */
282 		qhandle->index = qidx;
283 	}
284 
285 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
286 			 "6073 Binding %s HdwQueue %d  (cpu %d) to "
287 			 "hdw_queue %d qhandle x%px\n", str,
288 			 qidx, qhandle->cpu_id, qhandle->index, qhandle);
289 	*handle = (void *)qhandle;
290 	return 0;
291 }
292 
293 /**
294  * lpfc_nvme_delete_queue -
295  * @lpfc_pnvme: Pointer to the driver's nvme instance data
296  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
297  * @handle: An opaque driver handle from lpfc_nvme_create_queue
298  *
299  * Driver registers this routine to free
300  * any internal data structures to bind the @qidx to its internal
301  * IO queues.
302  *
303  * Return value :
304  *   0 - Success
305  *   TODO:  What are the failure codes.
306  **/
307 static void
308 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
309 		       unsigned int qidx,
310 		       void *handle)
311 {
312 	struct lpfc_nvme_lport *lport;
313 	struct lpfc_vport *vport;
314 
315 	if (!pnvme_lport->private)
316 		return;
317 
318 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
319 	vport = lport->vport;
320 
321 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
322 			"6001 ENTER.  lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
323 			lport, qidx, handle);
324 	kfree(handle);
325 }
326 
327 static void
328 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
329 {
330 	struct lpfc_nvme_lport *lport = localport->private;
331 
332 	lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
333 			 "6173 localport x%px delete complete\n",
334 			 lport);
335 
336 	/* release any threads waiting for the unreg to complete */
337 	if (lport->vport->localport)
338 		complete(lport->lport_unreg_cmp);
339 }
340 
341 /* lpfc_nvme_remoteport_delete
342  *
343  * @remoteport: Pointer to an nvme transport remoteport instance.
344  *
345  * This is a template downcall.  NVME transport calls this function
346  * when it has completed the unregistration of a previously
347  * registered remoteport.
348  *
349  * Return value :
350  * None
351  */
352 static void
353 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
354 {
355 	struct lpfc_nvme_rport *rport = remoteport->private;
356 	struct lpfc_vport *vport;
357 	struct lpfc_nodelist *ndlp;
358 
359 	ndlp = rport->ndlp;
360 	if (!ndlp)
361 		goto rport_err;
362 
363 	vport = ndlp->vport;
364 	if (!vport)
365 		goto rport_err;
366 
367 	/* Remove this rport from the lport's list - memory is owned by the
368 	 * transport. Remove the ndlp reference for the NVME transport before
369 	 * calling state machine to remove the node.
370 	 */
371 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
372 			"6146 remoteport delete of remoteport x%px\n",
373 			remoteport);
374 	spin_lock_irq(&vport->phba->hbalock);
375 
376 	/* The register rebind might have occurred before the delete
377 	 * downcall.  Guard against this race.
378 	 */
379 	if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
380 		ndlp->nrport = NULL;
381 		ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
382 		spin_unlock_irq(&vport->phba->hbalock);
383 
384 		/* Remove original register reference. The host transport
385 		 * won't reference this rport/remoteport any further.
386 		 */
387 		lpfc_nlp_put(ndlp);
388 	} else {
389 		spin_unlock_irq(&vport->phba->hbalock);
390 	}
391 
392  rport_err:
393 	return;
394 }
395 
396 /**
397  * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
398  * @phba: pointer to lpfc hba data structure.
399  * @axchg: pointer to exchange context for the NVME LS request
400  *
401  * This routine is used for processing an asychronously received NVME LS
402  * request. Any remaining validation is done and the LS is then forwarded
403  * to the nvme-fc transport via nvme_fc_rcv_ls_req().
404  *
405  * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
406  * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
407  * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
408  *
409  * Returns 0 if LS was handled and delivered to the transport
410  * Returns 1 if LS failed to be handled and should be dropped
411  */
412 int
413 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
414 			struct lpfc_async_xchg_ctx *axchg)
415 {
416 #if (IS_ENABLED(CONFIG_NVME_FC))
417 	struct lpfc_vport *vport;
418 	struct lpfc_nvme_rport *lpfc_rport;
419 	struct nvme_fc_remote_port *remoteport;
420 	struct lpfc_nvme_lport *lport;
421 	uint32_t *payload = axchg->payload;
422 	int rc;
423 
424 	vport = axchg->ndlp->vport;
425 	lpfc_rport = axchg->ndlp->nrport;
426 	if (!lpfc_rport)
427 		return -EINVAL;
428 
429 	remoteport = lpfc_rport->remoteport;
430 	if (!vport->localport)
431 		return -EINVAL;
432 
433 	lport = vport->localport->private;
434 	if (!lport)
435 		return -EINVAL;
436 
437 	rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
438 				axchg->size);
439 
440 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
441 			"6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
442 			"%08x %08x %08x\n",
443 			axchg->size, rc,
444 			*payload, *(payload+1), *(payload+2),
445 			*(payload+3), *(payload+4), *(payload+5));
446 
447 	if (!rc)
448 		return 0;
449 #endif
450 	return 1;
451 }
452 
453 /**
454  * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
455  *        LS request.
456  * @phba: Pointer to HBA context object
457  * @vport: The local port that issued the LS
458  * @cmdwqe: Pointer to driver command WQE object.
459  * @wcqe: Pointer to driver response CQE object.
460  *
461  * This function is the generic completion handler for NVME LS requests.
462  * The function updates any states and statistics, calls the transport
463  * ls_req done() routine, then tears down the command and buffers used
464  * for the LS request.
465  **/
466 void
467 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba,  struct lpfc_vport *vport,
468 			struct lpfc_iocbq *cmdwqe,
469 			struct lpfc_wcqe_complete *wcqe)
470 {
471 	struct nvmefc_ls_req *pnvme_lsreq;
472 	struct lpfc_dmabuf *buf_ptr;
473 	struct lpfc_nodelist *ndlp;
474 	uint32_t status;
475 
476 	pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
477 	ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
478 	status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
479 
480 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
481 			 "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x "
482 			 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
483 			 "ndlp:x%px\n",
484 			 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
485 			 cmdwqe->sli4_xritag, status,
486 			 (wcqe->parameter & 0xffff),
487 			 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
488 
489 	lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
490 			 cmdwqe->sli4_xritag, status, wcqe->parameter);
491 
492 	if (cmdwqe->context3) {
493 		buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
494 		lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
495 		kfree(buf_ptr);
496 		cmdwqe->context3 = NULL;
497 	}
498 	if (pnvme_lsreq->done)
499 		pnvme_lsreq->done(pnvme_lsreq, status);
500 	else
501 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
502 				 "6046 NVMEx cmpl without done call back? "
503 				 "Data %px DID %x Xri: %x status %x\n",
504 				pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
505 				cmdwqe->sli4_xritag, status);
506 	if (ndlp) {
507 		lpfc_nlp_put(ndlp);
508 		cmdwqe->context1 = NULL;
509 	}
510 	lpfc_sli_release_iocbq(phba, cmdwqe);
511 }
512 
513 static void
514 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
515 		       struct lpfc_wcqe_complete *wcqe)
516 {
517 	struct lpfc_vport *vport = cmdwqe->vport;
518 	struct lpfc_nvme_lport *lport;
519 	uint32_t status;
520 
521 	status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
522 
523 	if (vport->localport) {
524 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
525 		if (lport) {
526 			atomic_inc(&lport->fc4NvmeLsCmpls);
527 			if (status) {
528 				if (bf_get(lpfc_wcqe_c_xb, wcqe))
529 					atomic_inc(&lport->cmpl_ls_xb);
530 				atomic_inc(&lport->cmpl_ls_err);
531 			}
532 		}
533 	}
534 
535 	__lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
536 }
537 
538 static int
539 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
540 		  struct lpfc_dmabuf *inp,
541 		  struct nvmefc_ls_req *pnvme_lsreq,
542 		  void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
543 			       struct lpfc_wcqe_complete *),
544 		  struct lpfc_nodelist *ndlp, uint32_t num_entry,
545 		  uint32_t tmo, uint8_t retry)
546 {
547 	struct lpfc_hba *phba = vport->phba;
548 	union lpfc_wqe128 *wqe;
549 	struct lpfc_iocbq *genwqe;
550 	struct ulp_bde64 *bpl;
551 	struct ulp_bde64 bde;
552 	int i, rc, xmit_len, first_len;
553 
554 	/* Allocate buffer for  command WQE */
555 	genwqe = lpfc_sli_get_iocbq(phba);
556 	if (genwqe == NULL)
557 		return 1;
558 
559 	wqe = &genwqe->wqe;
560 	/* Initialize only 64 bytes */
561 	memset(wqe, 0, sizeof(union lpfc_wqe));
562 
563 	genwqe->context3 = (uint8_t *)bmp;
564 	genwqe->iocb_flag |= LPFC_IO_NVME_LS;
565 
566 	/* Save for completion so we can release these resources */
567 	genwqe->context1 = lpfc_nlp_get(ndlp);
568 	genwqe->context2 = (uint8_t *)pnvme_lsreq;
569 	/* Fill in payload, bp points to frame payload */
570 
571 	if (!tmo)
572 		/* FC spec states we need 3 * ratov for CT requests */
573 		tmo = (3 * phba->fc_ratov);
574 
575 	/* For this command calculate the xmit length of the request bde. */
576 	xmit_len = 0;
577 	first_len = 0;
578 	bpl = (struct ulp_bde64 *)bmp->virt;
579 	for (i = 0; i < num_entry; i++) {
580 		bde.tus.w = bpl[i].tus.w;
581 		if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
582 			break;
583 		xmit_len += bde.tus.f.bdeSize;
584 		if (i == 0)
585 			first_len = xmit_len;
586 	}
587 
588 	genwqe->rsvd2 = num_entry;
589 	genwqe->hba_wqidx = 0;
590 
591 	/* Words 0 - 2 */
592 	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
593 	wqe->generic.bde.tus.f.bdeSize = first_len;
594 	wqe->generic.bde.addrLow = bpl[0].addrLow;
595 	wqe->generic.bde.addrHigh = bpl[0].addrHigh;
596 
597 	/* Word 3 */
598 	wqe->gen_req.request_payload_len = first_len;
599 
600 	/* Word 4 */
601 
602 	/* Word 5 */
603 	bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
604 	bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
605 	bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
606 	bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
607 	bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
608 
609 	/* Word 6 */
610 	bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
611 	       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
612 	bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
613 
614 	/* Word 7 */
615 	bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1));
616 	bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
617 	bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
618 	bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
619 
620 	/* Word 8 */
621 	wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
622 
623 	/* Word 9 */
624 	bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
625 
626 	/* Word 10 */
627 	bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
628 	bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
629 	bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
630 	bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
631 	bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
632 
633 	/* Word 11 */
634 	bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
635 	bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
636 
637 
638 	/* Issue GEN REQ WQE for NPORT <did> */
639 	genwqe->wqe_cmpl = cmpl;
640 	genwqe->iocb_cmpl = NULL;
641 	genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
642 	genwqe->vport = vport;
643 	genwqe->retry = retry;
644 
645 	lpfc_nvmeio_data(phba, "NVME LS  XMIT: xri x%x iotag x%x to x%06x\n",
646 			 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
647 
648 	rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
649 	if (rc) {
650 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC | LOG_ELS,
651 				 "6045 Issue GEN REQ WQE to NPORT x%x "
652 				 "Data: x%x x%x  rc x%x\n",
653 				 ndlp->nlp_DID, genwqe->iotag,
654 				 vport->port_state, rc);
655 		lpfc_sli_release_iocbq(phba, genwqe);
656 		return 1;
657 	}
658 
659 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
660 			 "6050 Issue GEN REQ WQE to NPORT x%x "
661 			 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
662 			 "bmp:x%px xmit:%d 1st:%d\n",
663 			 ndlp->nlp_DID, genwqe->sli4_xritag,
664 			 vport->port_state,
665 			 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
666 	return 0;
667 }
668 
669 
670 /**
671  * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
672  * @vport: The local port issuing the LS
673  * @ndlp: The remote port to send the LS to
674  * @pnvme_lsreq: Pointer to LS request structure from the transport
675  *
676  * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
677  * WQE to perform the LS operation.
678  *
679  * Return value :
680  *   0 - Success
681  *   non-zero: various error codes, in form of -Exxx
682  **/
683 int
684 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
685 		      struct nvmefc_ls_req *pnvme_lsreq,
686 		      void (*gen_req_cmp)(struct lpfc_hba *phba,
687 				struct lpfc_iocbq *cmdwqe,
688 				struct lpfc_wcqe_complete *wcqe))
689 {
690 	struct lpfc_dmabuf *bmp;
691 	struct ulp_bde64 *bpl;
692 	int ret;
693 	uint16_t ntype, nstate;
694 
695 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
696 		lpfc_printf_vlog(vport, KERN_ERR,
697 				 LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR,
698 				 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
699 				 "LS Req\n",
700 				 ndlp);
701 		return -ENODEV;
702 	}
703 
704 	ntype = ndlp->nlp_type;
705 	nstate = ndlp->nlp_state;
706 	if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
707 	    (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
708 		lpfc_printf_vlog(vport, KERN_ERR,
709 				 LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR,
710 				 "6088 NVMEx LS REQ: Fail DID x%06x not "
711 				 "ready for IO. Type x%x, State x%x\n",
712 				 ndlp->nlp_DID, ntype, nstate);
713 		return -ENODEV;
714 	}
715 
716 	/*
717 	 * there are two dma buf in the request, actually there is one and
718 	 * the second one is just the start address + cmd size.
719 	 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
720 	 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
721 	 * because the nvem layer owns the data bufs.
722 	 * We do not have to break these packets open, we don't care what is
723 	 * in them. And we do not have to look at the resonse data, we only
724 	 * care that we got a response. All of the caring is going to happen
725 	 * in the nvme-fc layer.
726 	 */
727 
728 	bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
729 	if (!bmp) {
730 
731 		lpfc_printf_vlog(vport, KERN_ERR,
732 				 LOG_NVME_DISC | LOG_NVME_IOERR,
733 				 "6044 NVMEx LS REQ: Could not alloc LS buf "
734 				 "for DID %x\n",
735 				 ndlp->nlp_DID);
736 		return -ENOMEM;
737 	}
738 
739 	bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
740 	if (!bmp->virt) {
741 		lpfc_printf_vlog(vport, KERN_ERR,
742 				 LOG_NVME_DISC | LOG_NVME_IOERR,
743 				 "6042 NVMEx LS REQ: Could not alloc mbuf "
744 				 "for DID %x\n",
745 				 ndlp->nlp_DID);
746 		kfree(bmp);
747 		return -ENOMEM;
748 	}
749 
750 	INIT_LIST_HEAD(&bmp->list);
751 
752 	bpl = (struct ulp_bde64 *)bmp->virt;
753 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
754 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
755 	bpl->tus.f.bdeFlags = 0;
756 	bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
757 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
758 	bpl++;
759 
760 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
761 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
762 	bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
763 	bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
764 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
765 
766 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
767 			"6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
768 			"rqstlen:%d rsplen:%d %pad %pad\n",
769 			ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
770 			pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
771 			&pnvme_lsreq->rspdma);
772 
773 	ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
774 				pnvme_lsreq, gen_req_cmp, ndlp, 2,
775 				LPFC_NVME_LS_TIMEOUT, 0);
776 	if (ret != WQE_SUCCESS) {
777 		lpfc_printf_vlog(vport, KERN_ERR,
778 				 LOG_NVME_DISC | LOG_NVME_IOERR,
779 				 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
780 				 "lsreq x%px Status %x DID %x\n",
781 				 pnvme_lsreq, ret, ndlp->nlp_DID);
782 		lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
783 		kfree(bmp);
784 		return -EIO;
785 	}
786 
787 	return 0;
788 }
789 
790 /**
791  * lpfc_nvme_ls_req - Issue an NVME Link Service request
792  * @lpfc_nvme_lport: Transport localport that LS is to be issued from.
793  * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to.
794  * @pnvme_lsreq - the transport nvme_ls_req structure for the LS
795  *
796  * Driver registers this routine to handle any link service request
797  * from the nvme_fc transport to a remote nvme-aware port.
798  *
799  * Return value :
800  *   0 - Success
801  *   non-zero: various error codes, in form of -Exxx
802  **/
803 static int
804 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
805 		 struct nvme_fc_remote_port *pnvme_rport,
806 		 struct nvmefc_ls_req *pnvme_lsreq)
807 {
808 	struct lpfc_nvme_lport *lport;
809 	struct lpfc_nvme_rport *rport;
810 	struct lpfc_vport *vport;
811 	int ret;
812 
813 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
814 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
815 	if (unlikely(!lport) || unlikely(!rport))
816 		return -EINVAL;
817 
818 	vport = lport->vport;
819 	if (vport->load_flag & FC_UNLOADING)
820 		return -ENODEV;
821 
822 	atomic_inc(&lport->fc4NvmeLsRequests);
823 
824 	ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
825 				 lpfc_nvme_ls_req_cmp);
826 	if (ret)
827 		atomic_inc(&lport->xmt_ls_err);
828 
829 	return ret;
830 }
831 
832 /**
833  * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
834  *         NVME LS request
835  * @vport: The local port that issued the LS
836  * @ndlp: The remote port the LS was sent to
837  * @pnvme_lsreq: Pointer to LS request structure from the transport
838  *
839  * The driver validates the ndlp, looks for the LS, and aborts the
840  * LS if found.
841  *
842  * Returns:
843  * 0 : if LS found and aborted
844  * non-zero: various error conditions in form -Exxx
845  **/
846 int
847 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
848 			struct nvmefc_ls_req *pnvme_lsreq)
849 {
850 	struct lpfc_hba *phba = vport->phba;
851 	struct lpfc_sli_ring *pring;
852 	struct lpfc_iocbq *wqe, *next_wqe;
853 	bool foundit = false;
854 
855 	if (!ndlp) {
856 		lpfc_printf_log(phba, KERN_ERR,
857 				LOG_NVME_DISC | LOG_NODE |
858 					LOG_NVME_IOERR | LOG_NVME_ABTS,
859 				"6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
860 				"x%06x, Failing LS Req\n",
861 				ndlp, ndlp ? ndlp->nlp_DID : 0);
862 		return -EINVAL;
863 	}
864 
865 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
866 			 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
867 			 "x%p rqstlen:%d rsplen:%d %pad %pad\n",
868 			 pnvme_lsreq, pnvme_lsreq->rqstlen,
869 			 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
870 			 &pnvme_lsreq->rspdma);
871 
872 	/*
873 	 * Lock the ELS ring txcmplq and look for the wqe that matches
874 	 * this ELS. If found, issue an abort on the wqe.
875 	 */
876 	pring = phba->sli4_hba.nvmels_wq->pring;
877 	spin_lock_irq(&phba->hbalock);
878 	spin_lock(&pring->ring_lock);
879 	list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
880 		if (wqe->context2 == pnvme_lsreq) {
881 			wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
882 			foundit = true;
883 			break;
884 		}
885 	}
886 	spin_unlock(&pring->ring_lock);
887 
888 	if (foundit)
889 		lpfc_sli_issue_abort_iotag(phba, pring, wqe);
890 	spin_unlock_irq(&phba->hbalock);
891 
892 	if (foundit)
893 		return 0;
894 
895 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
896 			 "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n",
897 			 pnvme_lsreq);
898 	return -EINVAL;
899 }
900 
901 static int
902 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
903 		     struct nvme_fc_remote_port *remoteport,
904 		     struct nvmefc_ls_rsp *ls_rsp)
905 {
906 	struct lpfc_async_xchg_ctx *axchg =
907 		container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
908 	struct lpfc_nvme_lport *lport;
909 	int rc;
910 
911 	if (axchg->phba->pport->load_flag & FC_UNLOADING)
912 		return -ENODEV;
913 
914 	lport = (struct lpfc_nvme_lport *)localport->private;
915 
916 	rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
917 
918 	if (rc) {
919 		/*
920 		 * unless the failure is due to having already sent
921 		 * the response, an abort will be generated for the
922 		 * exchange if the rsp can't be sent.
923 		 */
924 		if (rc != -EALREADY)
925 			atomic_inc(&lport->xmt_ls_abort);
926 		return rc;
927 	}
928 
929 	return 0;
930 }
931 
932 /**
933  * lpfc_nvme_ls_abort - Abort a prior NVME LS request
934  * @lpfc_nvme_lport: Transport localport that LS is to be issued from.
935  * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to.
936  * @pnvme_lsreq - the transport nvme_ls_req structure for the LS
937  *
938  * Driver registers this routine to abort a NVME LS request that is
939  * in progress (from the transports perspective).
940  **/
941 static void
942 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
943 		   struct nvme_fc_remote_port *pnvme_rport,
944 		   struct nvmefc_ls_req *pnvme_lsreq)
945 {
946 	struct lpfc_nvme_lport *lport;
947 	struct lpfc_vport *vport;
948 	struct lpfc_hba *phba;
949 	struct lpfc_nodelist *ndlp;
950 	int ret;
951 
952 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
953 	if (unlikely(!lport))
954 		return;
955 	vport = lport->vport;
956 	phba = vport->phba;
957 
958 	if (vport->load_flag & FC_UNLOADING)
959 		return;
960 
961 	ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
962 
963 	ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
964 	if (!ret)
965 		atomic_inc(&lport->xmt_ls_abort);
966 }
967 
968 /* Fix up the existing sgls for NVME IO. */
969 static inline void
970 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
971 		       struct lpfc_io_buf *lpfc_ncmd,
972 		       struct nvmefc_fcp_req *nCmd)
973 {
974 	struct lpfc_hba  *phba = vport->phba;
975 	struct sli4_sge *sgl;
976 	union lpfc_wqe128 *wqe;
977 	uint32_t *wptr, *dptr;
978 
979 	/*
980 	 * Get a local pointer to the built-in wqe and correct
981 	 * the cmd size to match NVME's 96 bytes and fix
982 	 * the dma address.
983 	 */
984 
985 	wqe = &lpfc_ncmd->cur_iocbq.wqe;
986 
987 	/*
988 	 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
989 	 * match NVME.  NVME sends 96 bytes. Also, use the
990 	 * nvme commands command and response dma addresses
991 	 * rather than the virtual memory to ease the restore
992 	 * operation.
993 	 */
994 	sgl = lpfc_ncmd->dma_sgl;
995 	sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
996 	if (phba->cfg_nvme_embed_cmd) {
997 		sgl->addr_hi = 0;
998 		sgl->addr_lo = 0;
999 
1000 		/* Word 0-2 - NVME CMND IU (embedded payload) */
1001 		wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
1002 		wqe->generic.bde.tus.f.bdeSize = 56;
1003 		wqe->generic.bde.addrHigh = 0;
1004 		wqe->generic.bde.addrLow =  64;  /* Word 16 */
1005 
1006 		/* Word 10  - dbde is 0, wqes is 1 in template */
1007 
1008 		/*
1009 		 * Embed the payload in the last half of the WQE
1010 		 * WQE words 16-30 get the NVME CMD IU payload
1011 		 *
1012 		 * WQE words 16-19 get payload Words 1-4
1013 		 * WQE words 20-21 get payload Words 6-7
1014 		 * WQE words 22-29 get payload Words 16-23
1015 		 */
1016 		wptr = &wqe->words[16];  /* WQE ptr */
1017 		dptr = (uint32_t *)nCmd->cmdaddr;  /* payload ptr */
1018 		dptr++;			/* Skip Word 0 in payload */
1019 
1020 		*wptr++ = *dptr++;	/* Word 1 */
1021 		*wptr++ = *dptr++;	/* Word 2 */
1022 		*wptr++ = *dptr++;	/* Word 3 */
1023 		*wptr++ = *dptr++;	/* Word 4 */
1024 		dptr++;			/* Skip Word 5 in payload */
1025 		*wptr++ = *dptr++;	/* Word 6 */
1026 		*wptr++ = *dptr++;	/* Word 7 */
1027 		dptr += 8;		/* Skip Words 8-15 in payload */
1028 		*wptr++ = *dptr++;	/* Word 16 */
1029 		*wptr++ = *dptr++;	/* Word 17 */
1030 		*wptr++ = *dptr++;	/* Word 18 */
1031 		*wptr++ = *dptr++;	/* Word 19 */
1032 		*wptr++ = *dptr++;	/* Word 20 */
1033 		*wptr++ = *dptr++;	/* Word 21 */
1034 		*wptr++ = *dptr++;	/* Word 22 */
1035 		*wptr   = *dptr;	/* Word 23 */
1036 	} else {
1037 		sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
1038 		sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
1039 
1040 		/* Word 0-2 - NVME CMND IU Inline BDE */
1041 		wqe->generic.bde.tus.f.bdeFlags =  BUFF_TYPE_BDE_64;
1042 		wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
1043 		wqe->generic.bde.addrHigh = sgl->addr_hi;
1044 		wqe->generic.bde.addrLow =  sgl->addr_lo;
1045 
1046 		/* Word 10 */
1047 		bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
1048 		bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
1049 	}
1050 
1051 	sgl++;
1052 
1053 	/* Setup the physical region for the FCP RSP */
1054 	sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
1055 	sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
1056 	sgl->word2 = le32_to_cpu(sgl->word2);
1057 	if (nCmd->sg_cnt)
1058 		bf_set(lpfc_sli4_sge_last, sgl, 0);
1059 	else
1060 		bf_set(lpfc_sli4_sge_last, sgl, 1);
1061 	sgl->word2 = cpu_to_le32(sgl->word2);
1062 	sgl->sge_len = cpu_to_le32(nCmd->rsplen);
1063 }
1064 
1065 
1066 /**
1067  * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
1068  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1069  * @lpfc_nvme_lport: Pointer to the driver's local port data
1070  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1071  *
1072  * Driver registers this routine as it io request handler.  This
1073  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1074  * data structure to the rport indicated in @lpfc_nvme_rport.
1075  *
1076  * Return value :
1077  *   0 - Success
1078  *   TODO: What are the failure codes.
1079  **/
1080 static void
1081 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
1082 			  struct lpfc_wcqe_complete *wcqe)
1083 {
1084 	struct lpfc_io_buf *lpfc_ncmd =
1085 		(struct lpfc_io_buf *)pwqeIn->context1;
1086 	struct lpfc_vport *vport = pwqeIn->vport;
1087 	struct nvmefc_fcp_req *nCmd;
1088 	struct nvme_fc_ersp_iu *ep;
1089 	struct nvme_fc_cmd_iu *cp;
1090 	struct lpfc_nodelist *ndlp;
1091 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1092 	struct lpfc_nvme_lport *lport;
1093 	uint32_t code, status, idx;
1094 	uint16_t cid, sqhd, data;
1095 	uint32_t *ptr;
1096 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1097 	int cpu;
1098 #endif
1099 
1100 	/* Sanity check on return of outstanding command */
1101 	if (!lpfc_ncmd) {
1102 		lpfc_printf_vlog(vport, KERN_ERR,
1103 				 LOG_NODE | LOG_NVME_IOERR,
1104 				 "6071 Null lpfc_ncmd pointer. No "
1105 				 "release, skip completion\n");
1106 		return;
1107 	}
1108 
1109 	/* Guard against abort handler being called at same time */
1110 	spin_lock(&lpfc_ncmd->buf_lock);
1111 
1112 	if (!lpfc_ncmd->nvmeCmd) {
1113 		spin_unlock(&lpfc_ncmd->buf_lock);
1114 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
1115 				 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
1116 				 "nvmeCmd x%px\n",
1117 				 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
1118 
1119 		/* Release the lpfc_ncmd regardless of the missing elements. */
1120 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
1121 		return;
1122 	}
1123 	nCmd = lpfc_ncmd->nvmeCmd;
1124 	status = bf_get(lpfc_wcqe_c_status, wcqe);
1125 
1126 	idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
1127 	phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
1128 
1129 	if (unlikely(status && vport->localport)) {
1130 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
1131 		if (lport) {
1132 			if (bf_get(lpfc_wcqe_c_xb, wcqe))
1133 				atomic_inc(&lport->cmpl_fcp_xb);
1134 			atomic_inc(&lport->cmpl_fcp_err);
1135 		}
1136 	}
1137 
1138 	lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
1139 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
1140 			 status, wcqe->parameter);
1141 	/*
1142 	 * Catch race where our node has transitioned, but the
1143 	 * transport is still transitioning.
1144 	 */
1145 	ndlp = lpfc_ncmd->ndlp;
1146 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1147 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
1148 				 "6062 Ignoring NVME cmpl.  No ndlp\n");
1149 		goto out_err;
1150 	}
1151 
1152 	code = bf_get(lpfc_wcqe_c_code, wcqe);
1153 	if (code == CQE_CODE_NVME_ERSP) {
1154 		/* For this type of CQE, we need to rebuild the rsp */
1155 		ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1156 
1157 		/*
1158 		 * Get Command Id from cmd to plug into response. This
1159 		 * code is not needed in the next NVME Transport drop.
1160 		 */
1161 		cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1162 		cid = cp->sqe.common.command_id;
1163 
1164 		/*
1165 		 * RSN is in CQE word 2
1166 		 * SQHD is in CQE Word 3 bits 15:0
1167 		 * Cmd Specific info is in CQE Word 1
1168 		 * and in CQE Word 0 bits 15:0
1169 		 */
1170 		sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1171 
1172 		/* Now lets build the NVME ERSP IU */
1173 		ep->iu_len = cpu_to_be16(8);
1174 		ep->rsn = wcqe->parameter;
1175 		ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1176 		ep->rsvd12 = 0;
1177 		ptr = (uint32_t *)&ep->cqe.result.u64;
1178 		*ptr++ = wcqe->total_data_placed;
1179 		data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1180 		*ptr = (uint32_t)data;
1181 		ep->cqe.sq_head = sqhd;
1182 		ep->cqe.sq_id =  nCmd->sqid;
1183 		ep->cqe.command_id = cid;
1184 		ep->cqe.status = 0;
1185 
1186 		lpfc_ncmd->status = IOSTAT_SUCCESS;
1187 		lpfc_ncmd->result = 0;
1188 		nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1189 		nCmd->transferred_length = nCmd->payload_length;
1190 	} else {
1191 		lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);
1192 		lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1193 
1194 		/* For NVME, the only failure path that results in an
1195 		 * IO error is when the adapter rejects it.  All other
1196 		 * conditions are a success case and resolved by the
1197 		 * transport.
1198 		 * IOSTAT_FCP_RSP_ERROR means:
1199 		 * 1. Length of data received doesn't match total
1200 		 *    transfer length in WQE
1201 		 * 2. If the RSP payload does NOT match these cases:
1202 		 *    a. RSP length 12/24 bytes and all zeros
1203 		 *    b. NVME ERSP
1204 		 */
1205 		switch (lpfc_ncmd->status) {
1206 		case IOSTAT_SUCCESS:
1207 			nCmd->transferred_length = wcqe->total_data_placed;
1208 			nCmd->rcv_rsplen = 0;
1209 			nCmd->status = 0;
1210 			break;
1211 		case IOSTAT_FCP_RSP_ERROR:
1212 			nCmd->transferred_length = wcqe->total_data_placed;
1213 			nCmd->rcv_rsplen = wcqe->parameter;
1214 			nCmd->status = 0;
1215 			/* Sanity check */
1216 			if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
1217 				break;
1218 			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
1219 					 "6081 NVME Completion Protocol Error: "
1220 					 "xri %x status x%x result x%x "
1221 					 "placed x%x\n",
1222 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1223 					 lpfc_ncmd->status, lpfc_ncmd->result,
1224 					 wcqe->total_data_placed);
1225 			break;
1226 		case IOSTAT_LOCAL_REJECT:
1227 			/* Let fall through to set command final state. */
1228 			if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1229 				lpfc_printf_vlog(vport, KERN_INFO,
1230 					 LOG_NVME_IOERR,
1231 					 "6032 Delay Aborted cmd x%px "
1232 					 "nvme cmd x%px, xri x%x, "
1233 					 "xb %d\n",
1234 					 lpfc_ncmd, nCmd,
1235 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1236 					 bf_get(lpfc_wcqe_c_xb, wcqe));
1237 			/* fall through */
1238 		default:
1239 out_err:
1240 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1241 					 "6072 NVME Completion Error: xri %x "
1242 					 "status x%x result x%x [x%x] "
1243 					 "placed x%x\n",
1244 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1245 					 lpfc_ncmd->status, lpfc_ncmd->result,
1246 					 wcqe->parameter,
1247 					 wcqe->total_data_placed);
1248 			nCmd->transferred_length = 0;
1249 			nCmd->rcv_rsplen = 0;
1250 			nCmd->status = NVME_SC_INTERNAL;
1251 		}
1252 	}
1253 
1254 	/* pick up SLI4 exhange busy condition */
1255 	if (bf_get(lpfc_wcqe_c_xb, wcqe))
1256 		lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1257 	else
1258 		lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1259 
1260 	/* Update stats and complete the IO.  There is
1261 	 * no need for dma unprep because the nvme_transport
1262 	 * owns the dma address.
1263 	 */
1264 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1265 	if (lpfc_ncmd->ts_cmd_start) {
1266 		lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1267 		lpfc_ncmd->ts_data_io = ktime_get_ns();
1268 		phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1269 		lpfc_io_ktime(phba, lpfc_ncmd);
1270 	}
1271 	if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1272 		cpu = raw_smp_processor_id();
1273 		this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1274 		if (lpfc_ncmd->cpu != cpu)
1275 			lpfc_printf_vlog(vport,
1276 					 KERN_INFO, LOG_NVME_IOERR,
1277 					 "6701 CPU Check cmpl: "
1278 					 "cpu %d expect %d\n",
1279 					 cpu, lpfc_ncmd->cpu);
1280 	}
1281 #endif
1282 
1283 	/* NVME targets need completion held off until the abort exchange
1284 	 * completes unless the NVME Rport is getting unregistered.
1285 	 */
1286 
1287 	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1288 		freqpriv = nCmd->private;
1289 		freqpriv->nvme_buf = NULL;
1290 		lpfc_ncmd->nvmeCmd = NULL;
1291 		spin_unlock(&lpfc_ncmd->buf_lock);
1292 		nCmd->done(nCmd);
1293 	} else
1294 		spin_unlock(&lpfc_ncmd->buf_lock);
1295 
1296 	/* Call release with XB=1 to queue the IO into the abort list. */
1297 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1298 }
1299 
1300 
1301 /**
1302  * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1303  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1304  * @lpfc_nvme_lport: Pointer to the driver's local port data
1305  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1306  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1307  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1308  *
1309  * Driver registers this routine as it io request handler.  This
1310  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1311  * data structure to the rport indicated in @lpfc_nvme_rport.
1312  *
1313  * Return value :
1314  *   0 - Success
1315  *   TODO: What are the failure codes.
1316  **/
1317 static int
1318 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1319 		      struct lpfc_io_buf *lpfc_ncmd,
1320 		      struct lpfc_nodelist *pnode,
1321 		      struct lpfc_fc4_ctrl_stat *cstat)
1322 {
1323 	struct lpfc_hba *phba = vport->phba;
1324 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1325 	struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
1326 	union lpfc_wqe128 *wqe = &pwqeq->wqe;
1327 	uint32_t req_len;
1328 
1329 	if (!NLP_CHK_NODE_ACT(pnode))
1330 		return -EINVAL;
1331 
1332 	/*
1333 	 * There are three possibilities here - use scatter-gather segment, use
1334 	 * the single mapping, or neither.
1335 	 */
1336 	if (nCmd->sg_cnt) {
1337 		if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1338 			/* From the iwrite template, initialize words 7 - 11 */
1339 			memcpy(&wqe->words[7],
1340 			       &lpfc_iwrite_cmd_template.words[7],
1341 			       sizeof(uint32_t) * 5);
1342 
1343 			/* Word 4 */
1344 			wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1345 
1346 			/* Word 5 */
1347 			if ((phba->cfg_nvme_enable_fb) &&
1348 			    (pnode->nlp_flag & NLP_FIRSTBURST)) {
1349 				req_len = lpfc_ncmd->nvmeCmd->payload_length;
1350 				if (req_len < pnode->nvme_fb_size)
1351 					wqe->fcp_iwrite.initial_xfer_len =
1352 						req_len;
1353 				else
1354 					wqe->fcp_iwrite.initial_xfer_len =
1355 						pnode->nvme_fb_size;
1356 			} else {
1357 				wqe->fcp_iwrite.initial_xfer_len = 0;
1358 			}
1359 			cstat->output_requests++;
1360 		} else {
1361 			/* From the iread template, initialize words 7 - 11 */
1362 			memcpy(&wqe->words[7],
1363 			       &lpfc_iread_cmd_template.words[7],
1364 			       sizeof(uint32_t) * 5);
1365 
1366 			/* Word 4 */
1367 			wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1368 
1369 			/* Word 5 */
1370 			wqe->fcp_iread.rsrvd5 = 0;
1371 
1372 			cstat->input_requests++;
1373 		}
1374 	} else {
1375 		/* From the icmnd template, initialize words 4 - 11 */
1376 		memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1377 		       sizeof(uint32_t) * 8);
1378 		cstat->control_requests++;
1379 	}
1380 
1381 	if (pnode->nlp_nvme_info & NLP_NVME_NSLER)
1382 		bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1383 	/*
1384 	 * Finish initializing those WQE fields that are independent
1385 	 * of the nvme_cmnd request_buffer
1386 	 */
1387 
1388 	/* Word 3 */
1389 	bf_set(payload_offset_len, &wqe->fcp_icmd,
1390 	       (nCmd->rsplen + nCmd->cmdlen));
1391 
1392 	/* Word 6 */
1393 	bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1394 	       phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1395 	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1396 
1397 	/* Word 8 */
1398 	wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1399 
1400 	/* Word 9 */
1401 	bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1402 
1403 	/* Words 13 14 15 are for PBDE support */
1404 
1405 	pwqeq->vport = vport;
1406 	return 0;
1407 }
1408 
1409 
1410 /**
1411  * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1412  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1413  * @lpfc_nvme_lport: Pointer to the driver's local port data
1414  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1415  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1416  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1417  *
1418  * Driver registers this routine as it io request handler.  This
1419  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1420  * data structure to the rport indicated in @lpfc_nvme_rport.
1421  *
1422  * Return value :
1423  *   0 - Success
1424  *   TODO: What are the failure codes.
1425  **/
1426 static int
1427 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1428 		      struct lpfc_io_buf *lpfc_ncmd)
1429 {
1430 	struct lpfc_hba *phba = vport->phba;
1431 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1432 	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1433 	struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1434 	struct sli4_hybrid_sgl *sgl_xtra = NULL;
1435 	struct scatterlist *data_sg;
1436 	struct sli4_sge *first_data_sgl;
1437 	struct ulp_bde64 *bde;
1438 	dma_addr_t physaddr = 0;
1439 	uint32_t num_bde = 0;
1440 	uint32_t dma_len = 0;
1441 	uint32_t dma_offset = 0;
1442 	int nseg, i, j;
1443 	bool lsp_just_set = false;
1444 
1445 	/* Fix up the command and response DMA stuff. */
1446 	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1447 
1448 	/*
1449 	 * There are three possibilities here - use scatter-gather segment, use
1450 	 * the single mapping, or neither.
1451 	 */
1452 	if (nCmd->sg_cnt) {
1453 		/*
1454 		 * Jump over the cmd and rsp SGEs.  The fix routine
1455 		 * has already adjusted for this.
1456 		 */
1457 		sgl += 2;
1458 
1459 		first_data_sgl = sgl;
1460 		lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1461 		if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1462 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1463 					"6058 Too many sg segments from "
1464 					"NVME Transport.  Max %d, "
1465 					"nvmeIO sg_cnt %d\n",
1466 					phba->cfg_nvme_seg_cnt + 1,
1467 					lpfc_ncmd->seg_cnt);
1468 			lpfc_ncmd->seg_cnt = 0;
1469 			return 1;
1470 		}
1471 
1472 		/*
1473 		 * The driver established a maximum scatter-gather segment count
1474 		 * during probe that limits the number of sg elements in any
1475 		 * single nvme command.  Just run through the seg_cnt and format
1476 		 * the sge's.
1477 		 */
1478 		nseg = nCmd->sg_cnt;
1479 		data_sg = nCmd->first_sgl;
1480 
1481 		/* for tracking the segment boundaries */
1482 		j = 2;
1483 		for (i = 0; i < nseg; i++) {
1484 			if (data_sg == NULL) {
1485 				lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1486 						"6059 dptr err %d, nseg %d\n",
1487 						i, nseg);
1488 				lpfc_ncmd->seg_cnt = 0;
1489 				return 1;
1490 			}
1491 
1492 			sgl->word2 = 0;
1493 			if ((num_bde + 1) == nseg) {
1494 				bf_set(lpfc_sli4_sge_last, sgl, 1);
1495 				bf_set(lpfc_sli4_sge_type, sgl,
1496 				       LPFC_SGE_TYPE_DATA);
1497 			} else {
1498 				bf_set(lpfc_sli4_sge_last, sgl, 0);
1499 
1500 				/* expand the segment */
1501 				if (!lsp_just_set &&
1502 				    !((j + 1) % phba->border_sge_num) &&
1503 				    ((nseg - 1) != i)) {
1504 					/* set LSP type */
1505 					bf_set(lpfc_sli4_sge_type, sgl,
1506 					       LPFC_SGE_TYPE_LSP);
1507 
1508 					sgl_xtra = lpfc_get_sgl_per_hdwq(
1509 							phba, lpfc_ncmd);
1510 
1511 					if (unlikely(!sgl_xtra)) {
1512 						lpfc_ncmd->seg_cnt = 0;
1513 						return 1;
1514 					}
1515 					sgl->addr_lo = cpu_to_le32(putPaddrLow(
1516 						       sgl_xtra->dma_phys_sgl));
1517 					sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1518 						       sgl_xtra->dma_phys_sgl));
1519 
1520 				} else {
1521 					bf_set(lpfc_sli4_sge_type, sgl,
1522 					       LPFC_SGE_TYPE_DATA);
1523 				}
1524 			}
1525 
1526 			if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1527 				     LPFC_SGE_TYPE_LSP)) {
1528 				if ((nseg - 1) == i)
1529 					bf_set(lpfc_sli4_sge_last, sgl, 1);
1530 
1531 				physaddr = data_sg->dma_address;
1532 				dma_len = data_sg->length;
1533 				sgl->addr_lo = cpu_to_le32(
1534 							 putPaddrLow(physaddr));
1535 				sgl->addr_hi = cpu_to_le32(
1536 							putPaddrHigh(physaddr));
1537 
1538 				bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1539 				sgl->word2 = cpu_to_le32(sgl->word2);
1540 				sgl->sge_len = cpu_to_le32(dma_len);
1541 
1542 				dma_offset += dma_len;
1543 				data_sg = sg_next(data_sg);
1544 
1545 				sgl++;
1546 
1547 				lsp_just_set = false;
1548 			} else {
1549 				sgl->word2 = cpu_to_le32(sgl->word2);
1550 
1551 				sgl->sge_len = cpu_to_le32(
1552 						     phba->cfg_sg_dma_buf_size);
1553 
1554 				sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1555 				i = i - 1;
1556 
1557 				lsp_just_set = true;
1558 			}
1559 
1560 			j++;
1561 		}
1562 		if (phba->cfg_enable_pbde) {
1563 			/* Use PBDE support for first SGL only, offset == 0 */
1564 			/* Words 13-15 */
1565 			bde = (struct ulp_bde64 *)
1566 				&wqe->words[13];
1567 			bde->addrLow = first_data_sgl->addr_lo;
1568 			bde->addrHigh = first_data_sgl->addr_hi;
1569 			bde->tus.f.bdeSize =
1570 				le32_to_cpu(first_data_sgl->sge_len);
1571 			bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1572 			bde->tus.w = cpu_to_le32(bde->tus.w);
1573 			/* wqe_pbde is 1 in template */
1574 		} else {
1575 			memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1576 			bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
1577 		}
1578 
1579 	} else {
1580 		lpfc_ncmd->seg_cnt = 0;
1581 
1582 		/* For this clause to be valid, the payload_length
1583 		 * and sg_cnt must zero.
1584 		 */
1585 		if (nCmd->payload_length != 0) {
1586 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1587 					"6063 NVME DMA Prep Err: sg_cnt %d "
1588 					"payload_length x%x\n",
1589 					nCmd->sg_cnt, nCmd->payload_length);
1590 			return 1;
1591 		}
1592 	}
1593 	return 0;
1594 }
1595 
1596 /**
1597  * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1598  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1599  * @lpfc_nvme_lport: Pointer to the driver's local port data
1600  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1601  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1602  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1603  *
1604  * Driver registers this routine as it io request handler.  This
1605  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1606  * data structure to the rport
1607  indicated in @lpfc_nvme_rport.
1608  *
1609  * Return value :
1610  *   0 - Success
1611  *   TODO: What are the failure codes.
1612  **/
1613 static int
1614 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1615 			struct nvme_fc_remote_port *pnvme_rport,
1616 			void *hw_queue_handle,
1617 			struct nvmefc_fcp_req *pnvme_fcreq)
1618 {
1619 	int ret = 0;
1620 	int expedite = 0;
1621 	int idx, cpu;
1622 	struct lpfc_nvme_lport *lport;
1623 	struct lpfc_fc4_ctrl_stat *cstat;
1624 	struct lpfc_vport *vport;
1625 	struct lpfc_hba *phba;
1626 	struct lpfc_nodelist *ndlp;
1627 	struct lpfc_io_buf *lpfc_ncmd;
1628 	struct lpfc_nvme_rport *rport;
1629 	struct lpfc_nvme_qhandle *lpfc_queue_info;
1630 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1631 	struct nvme_common_command *sqe;
1632 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1633 	uint64_t start = 0;
1634 #endif
1635 
1636 	/* Validate pointers. LLDD fault handling with transport does
1637 	 * have timing races.
1638 	 */
1639 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1640 	if (unlikely(!lport)) {
1641 		ret = -EINVAL;
1642 		goto out_fail;
1643 	}
1644 
1645 	vport = lport->vport;
1646 
1647 	if (unlikely(!hw_queue_handle)) {
1648 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1649 				 "6117 Fail IO, NULL hw_queue_handle\n");
1650 		atomic_inc(&lport->xmt_fcp_err);
1651 		ret = -EBUSY;
1652 		goto out_fail;
1653 	}
1654 
1655 	phba = vport->phba;
1656 
1657 	if (unlikely(vport->load_flag & FC_UNLOADING)) {
1658 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1659 				 "6124 Fail IO, Driver unload\n");
1660 		atomic_inc(&lport->xmt_fcp_err);
1661 		ret = -ENODEV;
1662 		goto out_fail;
1663 	}
1664 
1665 	freqpriv = pnvme_fcreq->private;
1666 	if (unlikely(!freqpriv)) {
1667 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1668 				 "6158 Fail IO, NULL request data\n");
1669 		atomic_inc(&lport->xmt_fcp_err);
1670 		ret = -EINVAL;
1671 		goto out_fail;
1672 	}
1673 
1674 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1675 	if (phba->ktime_on)
1676 		start = ktime_get_ns();
1677 #endif
1678 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1679 	lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1680 
1681 	/*
1682 	 * Catch race where our node has transitioned, but the
1683 	 * transport is still transitioning.
1684 	 */
1685 	ndlp = rport->ndlp;
1686 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1687 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1688 				 "6053 Busy IO, ndlp not ready: rport x%px "
1689 				  "ndlp x%px, DID x%06x\n",
1690 				 rport, ndlp, pnvme_rport->port_id);
1691 		atomic_inc(&lport->xmt_fcp_err);
1692 		ret = -EBUSY;
1693 		goto out_fail;
1694 	}
1695 
1696 	/* The remote node has to be a mapped target or it's an error. */
1697 	if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1698 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1699 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1700 				 "6036 Fail IO, DID x%06x not ready for "
1701 				 "IO. State x%x, Type x%x Flg x%x\n",
1702 				 pnvme_rport->port_id,
1703 				 ndlp->nlp_state, ndlp->nlp_type,
1704 				 ndlp->upcall_flags);
1705 		atomic_inc(&lport->xmt_fcp_bad_ndlp);
1706 		ret = -EBUSY;
1707 		goto out_fail;
1708 
1709 	}
1710 
1711 	/* Currently only NVME Keep alive commands should be expedited
1712 	 * if the driver runs out of a resource. These should only be
1713 	 * issued on the admin queue, qidx 0
1714 	 */
1715 	if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1716 		sqe = &((struct nvme_fc_cmd_iu *)
1717 			pnvme_fcreq->cmdaddr)->sqe.common;
1718 		if (sqe->opcode == nvme_admin_keep_alive)
1719 			expedite = 1;
1720 	}
1721 
1722 	/* The node is shared with FCP IO, make sure the IO pending count does
1723 	 * not exceed the programmed depth.
1724 	 */
1725 	if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1726 		if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1727 		    !expedite) {
1728 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1729 					 "6174 Fail IO, ndlp qdepth exceeded: "
1730 					 "idx %d DID %x pend %d qdepth %d\n",
1731 					 lpfc_queue_info->index, ndlp->nlp_DID,
1732 					 atomic_read(&ndlp->cmd_pending),
1733 					 ndlp->cmd_qdepth);
1734 			atomic_inc(&lport->xmt_fcp_qdepth);
1735 			ret = -EBUSY;
1736 			goto out_fail;
1737 		}
1738 	}
1739 
1740 	/* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1741 	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1742 		idx = lpfc_queue_info->index;
1743 	} else {
1744 		cpu = raw_smp_processor_id();
1745 		idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1746 	}
1747 
1748 	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1749 	if (lpfc_ncmd == NULL) {
1750 		atomic_inc(&lport->xmt_fcp_noxri);
1751 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1752 				 "6065 Fail IO, driver buffer pool is empty: "
1753 				 "idx %d DID %x\n",
1754 				 lpfc_queue_info->index, ndlp->nlp_DID);
1755 		ret = -EBUSY;
1756 		goto out_fail;
1757 	}
1758 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1759 	if (start) {
1760 		lpfc_ncmd->ts_cmd_start = start;
1761 		lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1762 	} else {
1763 		lpfc_ncmd->ts_cmd_start = 0;
1764 	}
1765 #endif
1766 
1767 	/*
1768 	 * Store the data needed by the driver to issue, abort, and complete
1769 	 * an IO.
1770 	 * Do not let the IO hang out forever.  There is no midlayer issuing
1771 	 * an abort so inform the FW of the maximum IO pending time.
1772 	 */
1773 	freqpriv->nvme_buf = lpfc_ncmd;
1774 	lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1775 	lpfc_ncmd->ndlp = ndlp;
1776 	lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1777 
1778 	/*
1779 	 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1780 	 * This identfier was create in our hardware queue create callback
1781 	 * routine. The driver now is dependent on the IO queue steering from
1782 	 * the transport.  We are trusting the upper NVME layers know which
1783 	 * index to use and that they have affinitized a CPU to this hardware
1784 	 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1785 	 */
1786 	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1787 	cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1788 
1789 	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1790 	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1791 	if (ret) {
1792 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1793 				 "6175 Fail IO, Prep DMA: "
1794 				 "idx %d DID %x\n",
1795 				 lpfc_queue_info->index, ndlp->nlp_DID);
1796 		atomic_inc(&lport->xmt_fcp_err);
1797 		ret = -ENOMEM;
1798 		goto out_free_nvme_buf;
1799 	}
1800 
1801 	lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1802 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
1803 			 lpfc_queue_info->index, ndlp->nlp_DID);
1804 
1805 	ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1806 	if (ret) {
1807 		atomic_inc(&lport->xmt_fcp_wqerr);
1808 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1809 				 "6113 Fail IO, Could not issue WQE err %x "
1810 				 "sid: x%x did: x%x oxid: x%x\n",
1811 				 ret, vport->fc_myDID, ndlp->nlp_DID,
1812 				 lpfc_ncmd->cur_iocbq.sli4_xritag);
1813 		goto out_free_nvme_buf;
1814 	}
1815 
1816 	if (phba->cfg_xri_rebalancing)
1817 		lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1818 
1819 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1820 	if (lpfc_ncmd->ts_cmd_start)
1821 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1822 
1823 	if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1824 		cpu = raw_smp_processor_id();
1825 		this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1826 		lpfc_ncmd->cpu = cpu;
1827 		if (idx != cpu)
1828 			lpfc_printf_vlog(vport,
1829 					 KERN_INFO, LOG_NVME_IOERR,
1830 					"6702 CPU Check cmd: "
1831 					"cpu %d wq %d\n",
1832 					lpfc_ncmd->cpu,
1833 					lpfc_queue_info->index);
1834 	}
1835 #endif
1836 	return 0;
1837 
1838  out_free_nvme_buf:
1839 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1840 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1841 			cstat->output_requests--;
1842 		else
1843 			cstat->input_requests--;
1844 	} else
1845 		cstat->control_requests--;
1846 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1847  out_fail:
1848 	return ret;
1849 }
1850 
1851 /**
1852  * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1853  * @phba: Pointer to HBA context object
1854  * @cmdiocb: Pointer to command iocb object.
1855  * @rspiocb: Pointer to response iocb object.
1856  *
1857  * This is the callback function for any NVME FCP IO that was aborted.
1858  *
1859  * Return value:
1860  *   None
1861  **/
1862 void
1863 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1864 			   struct lpfc_wcqe_complete *abts_cmpl)
1865 {
1866 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1867 			"6145 ABORT_XRI_CN completing on rpi x%x "
1868 			"original iotag x%x, abort cmd iotag x%x "
1869 			"req_tag x%x, status x%x, hwstatus x%x\n",
1870 			cmdiocb->iocb.un.acxri.abortContextTag,
1871 			cmdiocb->iocb.un.acxri.abortIoTag,
1872 			cmdiocb->iotag,
1873 			bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1874 			bf_get(lpfc_wcqe_c_status, abts_cmpl),
1875 			bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1876 	lpfc_sli_release_iocbq(phba, cmdiocb);
1877 }
1878 
1879 /**
1880  * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1881  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1882  * @lpfc_nvme_lport: Pointer to the driver's local port data
1883  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1884  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1885  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1886  *
1887  * Driver registers this routine as its nvme request io abort handler.  This
1888  * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1889  * data structure to the rport indicated in @lpfc_nvme_rport.  This routine
1890  * is executed asynchronously - one the target is validated as "MAPPED" and
1891  * ready for IO, the driver issues the abort request and returns.
1892  *
1893  * Return value:
1894  *   None
1895  **/
1896 static void
1897 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1898 		    struct nvme_fc_remote_port *pnvme_rport,
1899 		    void *hw_queue_handle,
1900 		    struct nvmefc_fcp_req *pnvme_fcreq)
1901 {
1902 	struct lpfc_nvme_lport *lport;
1903 	struct lpfc_vport *vport;
1904 	struct lpfc_hba *phba;
1905 	struct lpfc_io_buf *lpfc_nbuf;
1906 	struct lpfc_iocbq *abts_buf;
1907 	struct lpfc_iocbq *nvmereq_wqe;
1908 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1909 	unsigned long flags;
1910 	int ret_val;
1911 
1912 	/* Validate pointers. LLDD fault handling with transport does
1913 	 * have timing races.
1914 	 */
1915 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1916 	if (unlikely(!lport))
1917 		return;
1918 
1919 	vport = lport->vport;
1920 
1921 	if (unlikely(!hw_queue_handle)) {
1922 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1923 				 "6129 Fail Abort, HW Queue Handle NULL.\n");
1924 		return;
1925 	}
1926 
1927 	phba = vport->phba;
1928 	freqpriv = pnvme_fcreq->private;
1929 
1930 	if (unlikely(!freqpriv))
1931 		return;
1932 	if (vport->load_flag & FC_UNLOADING)
1933 		return;
1934 
1935 	/* Announce entry to new IO submit field. */
1936 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1937 			 "6002 Abort Request to rport DID x%06x "
1938 			 "for nvme_fc_req x%px\n",
1939 			 pnvme_rport->port_id,
1940 			 pnvme_fcreq);
1941 
1942 	/* If the hba is getting reset, this flag is set.  It is
1943 	 * cleared when the reset is complete and rings reestablished.
1944 	 */
1945 	spin_lock_irqsave(&phba->hbalock, flags);
1946 	/* driver queued commands are in process of being flushed */
1947 	if (phba->hba_flag & HBA_IOQ_FLUSH) {
1948 		spin_unlock_irqrestore(&phba->hbalock, flags);
1949 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1950 				 "6139 Driver in reset cleanup - flushing "
1951 				 "NVME Req now.  hba_flag x%x\n",
1952 				 phba->hba_flag);
1953 		return;
1954 	}
1955 
1956 	lpfc_nbuf = freqpriv->nvme_buf;
1957 	if (!lpfc_nbuf) {
1958 		spin_unlock_irqrestore(&phba->hbalock, flags);
1959 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1960 				 "6140 NVME IO req has no matching lpfc nvme "
1961 				 "io buffer.  Skipping abort req.\n");
1962 		return;
1963 	} else if (!lpfc_nbuf->nvmeCmd) {
1964 		spin_unlock_irqrestore(&phba->hbalock, flags);
1965 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1966 				 "6141 lpfc NVME IO req has no nvme_fcreq "
1967 				 "io buffer.  Skipping abort req.\n");
1968 		return;
1969 	}
1970 	nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1971 
1972 	/* Guard against IO completion being called at same time */
1973 	spin_lock(&lpfc_nbuf->buf_lock);
1974 
1975 	/*
1976 	 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1977 	 * state must match the nvme_fcreq passed by the nvme
1978 	 * transport.  If they don't match, it is likely the driver
1979 	 * has already completed the NVME IO and the nvme transport
1980 	 * has not seen it yet.
1981 	 */
1982 	if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1983 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1984 				 "6143 NVME req mismatch: "
1985 				 "lpfc_nbuf x%px nvmeCmd x%px, "
1986 				 "pnvme_fcreq x%px.  Skipping Abort xri x%x\n",
1987 				 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1988 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1989 		goto out_unlock;
1990 	}
1991 
1992 	/* Don't abort IOs no longer on the pending queue. */
1993 	if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
1994 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1995 				 "6142 NVME IO req x%px not queued - skipping "
1996 				 "abort req xri x%x\n",
1997 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1998 		goto out_unlock;
1999 	}
2000 
2001 	atomic_inc(&lport->xmt_fcp_abort);
2002 	lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
2003 			 nvmereq_wqe->sli4_xritag,
2004 			 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
2005 
2006 	/* Outstanding abort is in progress */
2007 	if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
2008 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
2009 				 "6144 Outstanding NVME I/O Abort Request "
2010 				 "still pending on nvme_fcreq x%px, "
2011 				 "lpfc_ncmd %px xri x%x\n",
2012 				 pnvme_fcreq, lpfc_nbuf,
2013 				 nvmereq_wqe->sli4_xritag);
2014 		goto out_unlock;
2015 	}
2016 
2017 	abts_buf = __lpfc_sli_get_iocbq(phba);
2018 	if (!abts_buf) {
2019 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
2020 				 "6136 No available abort wqes. Skipping "
2021 				 "Abts req for nvme_fcreq x%px xri x%x\n",
2022 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
2023 		goto out_unlock;
2024 	}
2025 
2026 	/* Ready - mark outstanding as aborted by driver. */
2027 	nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
2028 
2029 	lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0);
2030 
2031 	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
2032 	abts_buf->iocb_flag |= LPFC_IO_NVME;
2033 	abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx;
2034 	abts_buf->vport = vport;
2035 	abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
2036 	ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf);
2037 	spin_unlock(&lpfc_nbuf->buf_lock);
2038 	spin_unlock_irqrestore(&phba->hbalock, flags);
2039 	if (ret_val) {
2040 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
2041 				 "6137 Failed abts issue_wqe with status x%x "
2042 				 "for nvme_fcreq x%px.\n",
2043 				 ret_val, pnvme_fcreq);
2044 		lpfc_sli_release_iocbq(phba, abts_buf);
2045 		return;
2046 	}
2047 
2048 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
2049 			 "6138 Transport Abort NVME Request Issued for "
2050 			 "ox_id x%x on reqtag x%x\n",
2051 			 nvmereq_wqe->sli4_xritag,
2052 			 abts_buf->iotag);
2053 	return;
2054 
2055 out_unlock:
2056 	spin_unlock(&lpfc_nbuf->buf_lock);
2057 	spin_unlock_irqrestore(&phba->hbalock, flags);
2058 	return;
2059 }
2060 
2061 /* Declare and initialization an instance of the FC NVME template. */
2062 static struct nvme_fc_port_template lpfc_nvme_template = {
2063 	/* initiator-based functions */
2064 	.localport_delete  = lpfc_nvme_localport_delete,
2065 	.remoteport_delete = lpfc_nvme_remoteport_delete,
2066 	.create_queue = lpfc_nvme_create_queue,
2067 	.delete_queue = lpfc_nvme_delete_queue,
2068 	.ls_req       = lpfc_nvme_ls_req,
2069 	.fcp_io       = lpfc_nvme_fcp_io_submit,
2070 	.ls_abort     = lpfc_nvme_ls_abort,
2071 	.fcp_abort    = lpfc_nvme_fcp_abort,
2072 	.xmt_ls_rsp   = lpfc_nvme_xmt_ls_rsp,
2073 
2074 	.max_hw_queues = 1,
2075 	.max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2076 	.max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2077 	.dma_boundary = 0xFFFFFFFF,
2078 
2079 	/* Sizes of additional private data for data structures.
2080 	 * No use for the last two sizes at this time.
2081 	 */
2082 	.local_priv_sz = sizeof(struct lpfc_nvme_lport),
2083 	.remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2084 	.lsrqst_priv_sz = 0,
2085 	.fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2086 };
2087 
2088 /**
2089  * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2090  * @phba: The HBA for which this call is being executed.
2091  *
2092  * This routine removes a nvme buffer from head of @hdwq io_buf_list
2093  * and returns to caller.
2094  *
2095  * Return codes:
2096  *   NULL - Error
2097  *   Pointer to lpfc_nvme_buf - Success
2098  **/
2099 static struct lpfc_io_buf *
2100 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2101 		  int idx, int expedite)
2102 {
2103 	struct lpfc_io_buf *lpfc_ncmd;
2104 	struct lpfc_sli4_hdw_queue *qp;
2105 	struct sli4_sge *sgl;
2106 	struct lpfc_iocbq *pwqeq;
2107 	union lpfc_wqe128 *wqe;
2108 
2109 	lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2110 
2111 	if (lpfc_ncmd) {
2112 		pwqeq = &(lpfc_ncmd->cur_iocbq);
2113 		wqe = &pwqeq->wqe;
2114 
2115 		/* Setup key fields in buffer that may have been changed
2116 		 * if other protocols used this buffer.
2117 		 */
2118 		pwqeq->iocb_flag = LPFC_IO_NVME;
2119 		pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
2120 		lpfc_ncmd->start_time = jiffies;
2121 		lpfc_ncmd->flags = 0;
2122 
2123 		/* Rsp SGE will be filled in when we rcv an IO
2124 		 * from the NVME Layer to be sent.
2125 		 * The cmd is going to be embedded so we need a SKIP SGE.
2126 		 */
2127 		sgl = lpfc_ncmd->dma_sgl;
2128 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2129 		bf_set(lpfc_sli4_sge_last, sgl, 0);
2130 		sgl->word2 = cpu_to_le32(sgl->word2);
2131 		/* Fill in word 3 / sgl_len during cmd submission */
2132 
2133 		/* Initialize 64 bytes only */
2134 		memset(wqe, 0, sizeof(union lpfc_wqe));
2135 
2136 		if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2137 			atomic_inc(&ndlp->cmd_pending);
2138 			lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2139 		}
2140 
2141 	} else {
2142 		qp = &phba->sli4_hba.hdwq[idx];
2143 		qp->empty_io_bufs++;
2144 	}
2145 
2146 	return  lpfc_ncmd;
2147 }
2148 
2149 /**
2150  * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2151  * @phba: The Hba for which this call is being executed.
2152  * @lpfc_ncmd: The nvme buffer which is being released.
2153  *
2154  * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2155  * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2156  * and cannot be reused for at least RA_TOV amount of time if it was
2157  * aborted.
2158  **/
2159 static void
2160 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2161 {
2162 	struct lpfc_sli4_hdw_queue *qp;
2163 	unsigned long iflag = 0;
2164 
2165 	if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2166 		atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2167 
2168 	lpfc_ncmd->ndlp = NULL;
2169 	lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2170 
2171 	qp = lpfc_ncmd->hdwq;
2172 	if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2173 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2174 				"6310 XB release deferred for "
2175 				"ox_id x%x on reqtag x%x\n",
2176 				lpfc_ncmd->cur_iocbq.sli4_xritag,
2177 				lpfc_ncmd->cur_iocbq.iotag);
2178 
2179 		spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2180 		list_add_tail(&lpfc_ncmd->list,
2181 			&qp->lpfc_abts_io_buf_list);
2182 		qp->abts_nvme_io_bufs++;
2183 		spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2184 	} else
2185 		lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2186 }
2187 
2188 /**
2189  * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2190  * @pvport - the lpfc_vport instance requesting a localport.
2191  *
2192  * This routine is invoked to create an nvme localport instance to bind
2193  * to the nvme_fc_transport.  It is called once during driver load
2194  * like lpfc_create_shost after all other services are initialized.
2195  * It requires a vport, vpi, and wwns at call time.  Other localport
2196  * parameters are modified as the driver's FCID and the Fabric WWN
2197  * are established.
2198  *
2199  * Return codes
2200  *      0 - successful
2201  *      -ENOMEM - no heap memory available
2202  *      other values - from nvme registration upcall
2203  **/
2204 int
2205 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2206 {
2207 	int ret = 0;
2208 	struct lpfc_hba  *phba = vport->phba;
2209 	struct nvme_fc_port_info nfcp_info;
2210 	struct nvme_fc_local_port *localport;
2211 	struct lpfc_nvme_lport *lport;
2212 
2213 	/* Initialize this localport instance.  The vport wwn usage ensures
2214 	 * that NPIV is accounted for.
2215 	 */
2216 	memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2217 	nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2218 	nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2219 	nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2220 
2221 	/* We need to tell the transport layer + 1 because it takes page
2222 	 * alignment into account. When space for the SGL is allocated we
2223 	 * allocate + 3, one for cmd, one for rsp and one for this alignment
2224 	 */
2225 	lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2226 
2227 	/* Advertise how many hw queues we support based on cfg_hdw_queue,
2228 	 * which will not exceed cpu count.
2229 	 */
2230 	lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2231 
2232 	if (!IS_ENABLED(CONFIG_NVME_FC))
2233 		return ret;
2234 
2235 	/* localport is allocated from the stack, but the registration
2236 	 * call allocates heap memory as well as the private area.
2237 	 */
2238 
2239 	ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2240 					 &vport->phba->pcidev->dev, &localport);
2241 	if (!ret) {
2242 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2243 				 "6005 Successfully registered local "
2244 				 "NVME port num %d, localP x%px, private "
2245 				 "x%px, sg_seg %d\n",
2246 				 localport->port_num, localport,
2247 				 localport->private,
2248 				 lpfc_nvme_template.max_sgl_segments);
2249 
2250 		/* Private is our lport size declared in the template. */
2251 		lport = (struct lpfc_nvme_lport *)localport->private;
2252 		vport->localport = localport;
2253 		lport->vport = vport;
2254 		vport->nvmei_support = 1;
2255 
2256 		atomic_set(&lport->xmt_fcp_noxri, 0);
2257 		atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2258 		atomic_set(&lport->xmt_fcp_qdepth, 0);
2259 		atomic_set(&lport->xmt_fcp_err, 0);
2260 		atomic_set(&lport->xmt_fcp_wqerr, 0);
2261 		atomic_set(&lport->xmt_fcp_abort, 0);
2262 		atomic_set(&lport->xmt_ls_abort, 0);
2263 		atomic_set(&lport->xmt_ls_err, 0);
2264 		atomic_set(&lport->cmpl_fcp_xb, 0);
2265 		atomic_set(&lport->cmpl_fcp_err, 0);
2266 		atomic_set(&lport->cmpl_ls_xb, 0);
2267 		atomic_set(&lport->cmpl_ls_err, 0);
2268 
2269 		atomic_set(&lport->fc4NvmeLsRequests, 0);
2270 		atomic_set(&lport->fc4NvmeLsCmpls, 0);
2271 	}
2272 
2273 	return ret;
2274 }
2275 
2276 #if (IS_ENABLED(CONFIG_NVME_FC))
2277 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2278  *
2279  * The driver has to wait for the host nvme transport to callback
2280  * indicating the localport has successfully unregistered all
2281  * resources.  Since this is an uninterruptible wait, loop every ten
2282  * seconds and print a message indicating no progress.
2283  *
2284  * An uninterruptible wait is used because of the risk of transport-to-
2285  * driver state mismatch.
2286  */
2287 static void
2288 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2289 			   struct lpfc_nvme_lport *lport,
2290 			   struct completion *lport_unreg_cmp)
2291 {
2292 	u32 wait_tmo;
2293 	int ret, i, pending = 0;
2294 	struct lpfc_sli_ring  *pring;
2295 	struct lpfc_hba  *phba = vport->phba;
2296 
2297 	/* Host transport has to clean up and confirm requiring an indefinite
2298 	 * wait. Print a message if a 10 second wait expires and renew the
2299 	 * wait. This is unexpected.
2300 	 */
2301 	wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2302 	while (true) {
2303 		ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2304 		if (unlikely(!ret)) {
2305 			pending = 0;
2306 			for (i = 0; i < phba->cfg_hdw_queue; i++) {
2307 				pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2308 				if (!pring)
2309 					continue;
2310 				if (pring->txcmplq_cnt)
2311 					pending += pring->txcmplq_cnt;
2312 			}
2313 			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
2314 					 "6176 Lport x%px Localport x%px wait "
2315 					 "timed out. Pending %d. Renewing.\n",
2316 					 lport, vport->localport, pending);
2317 			continue;
2318 		}
2319 		break;
2320 	}
2321 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2322 			 "6177 Lport x%px Localport x%px Complete Success\n",
2323 			 lport, vport->localport);
2324 }
2325 #endif
2326 
2327 /**
2328  * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2329  * @pnvme: pointer to lpfc nvme data structure.
2330  *
2331  * This routine is invoked to destroy all lports bound to the phba.
2332  * The lport memory was allocated by the nvme fc transport and is
2333  * released there.  This routine ensures all rports bound to the
2334  * lport have been disconnected.
2335  *
2336  **/
2337 void
2338 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2339 {
2340 #if (IS_ENABLED(CONFIG_NVME_FC))
2341 	struct nvme_fc_local_port *localport;
2342 	struct lpfc_nvme_lport *lport;
2343 	int ret;
2344 	DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2345 
2346 	if (vport->nvmei_support == 0)
2347 		return;
2348 
2349 	localport = vport->localport;
2350 	lport = (struct lpfc_nvme_lport *)localport->private;
2351 
2352 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2353 			 "6011 Destroying NVME localport x%px\n",
2354 			 localport);
2355 
2356 	/* lport's rport list is clear.  Unregister
2357 	 * lport and release resources.
2358 	 */
2359 	lport->lport_unreg_cmp = &lport_unreg_cmp;
2360 	ret = nvme_fc_unregister_localport(localport);
2361 
2362 	/* Wait for completion.  This either blocks
2363 	 * indefinitely or succeeds
2364 	 */
2365 	lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2366 	vport->localport = NULL;
2367 
2368 	/* Regardless of the unregister upcall response, clear
2369 	 * nvmei_support.  All rports are unregistered and the
2370 	 * driver will clean up.
2371 	 */
2372 	vport->nvmei_support = 0;
2373 	if (ret == 0) {
2374 		lpfc_printf_vlog(vport,
2375 				 KERN_INFO, LOG_NVME_DISC,
2376 				 "6009 Unregistered lport Success\n");
2377 	} else {
2378 		lpfc_printf_vlog(vport,
2379 				 KERN_INFO, LOG_NVME_DISC,
2380 				 "6010 Unregistered lport "
2381 				 "Failed, status x%x\n",
2382 				 ret);
2383 	}
2384 #endif
2385 }
2386 
2387 void
2388 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2389 {
2390 #if (IS_ENABLED(CONFIG_NVME_FC))
2391 	struct nvme_fc_local_port *localport;
2392 	struct lpfc_nvme_lport *lport;
2393 
2394 	localport = vport->localport;
2395 	if (!localport) {
2396 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2397 				 "6710 Update NVME fail. No localport\n");
2398 		return;
2399 	}
2400 	lport = (struct lpfc_nvme_lport *)localport->private;
2401 	if (!lport) {
2402 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2403 				 "6171 Update NVME fail. localP x%px, No lport\n",
2404 				 localport);
2405 		return;
2406 	}
2407 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2408 			 "6012 Update NVME lport x%px did x%x\n",
2409 			 localport, vport->fc_myDID);
2410 
2411 	localport->port_id = vport->fc_myDID;
2412 	if (localport->port_id == 0)
2413 		localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2414 	else
2415 		localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2416 
2417 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2418 			 "6030 bound lport x%px to DID x%06x\n",
2419 			 lport, localport->port_id);
2420 #endif
2421 }
2422 
2423 int
2424 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2425 {
2426 #if (IS_ENABLED(CONFIG_NVME_FC))
2427 	int ret = 0;
2428 	struct nvme_fc_local_port *localport;
2429 	struct lpfc_nvme_lport *lport;
2430 	struct lpfc_nvme_rport *rport;
2431 	struct lpfc_nvme_rport *oldrport;
2432 	struct nvme_fc_remote_port *remote_port;
2433 	struct nvme_fc_port_info rpinfo;
2434 	struct lpfc_nodelist *prev_ndlp = NULL;
2435 
2436 	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2437 			 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2438 			 ndlp->nlp_DID, ndlp->nlp_type);
2439 
2440 	localport = vport->localport;
2441 	if (!localport)
2442 		return 0;
2443 
2444 	lport = (struct lpfc_nvme_lport *)localport->private;
2445 
2446 	/* NVME rports are not preserved across devloss.
2447 	 * Just register this instance.  Note, rpinfo->dev_loss_tmo
2448 	 * is left 0 to indicate accept transport defaults.  The
2449 	 * driver communicates port role capabilities consistent
2450 	 * with the PRLI response data.
2451 	 */
2452 	memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2453 	rpinfo.port_id = ndlp->nlp_DID;
2454 	if (ndlp->nlp_type & NLP_NVME_TARGET)
2455 		rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2456 	if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2457 		rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2458 
2459 	if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2460 		rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2461 
2462 	rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2463 	rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2464 
2465 	spin_lock_irq(&vport->phba->hbalock);
2466 	oldrport = lpfc_ndlp_get_nrport(ndlp);
2467 	if (oldrport) {
2468 		prev_ndlp = oldrport->ndlp;
2469 		spin_unlock_irq(&vport->phba->hbalock);
2470 	} else {
2471 		spin_unlock_irq(&vport->phba->hbalock);
2472 		lpfc_nlp_get(ndlp);
2473 	}
2474 
2475 	ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2476 	if (!ret) {
2477 		/* If the ndlp already has an nrport, this is just
2478 		 * a resume of the existing rport.  Else this is a
2479 		 * new rport.
2480 		 */
2481 		/* Guard against an unregister/reregister
2482 		 * race that leaves the WAIT flag set.
2483 		 */
2484 		spin_lock_irq(&vport->phba->hbalock);
2485 		ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2486 		spin_unlock_irq(&vport->phba->hbalock);
2487 		rport = remote_port->private;
2488 		if (oldrport) {
2489 
2490 			/* Sever the ndlp<->rport association
2491 			 * before dropping the ndlp ref from
2492 			 * register.
2493 			 */
2494 			spin_lock_irq(&vport->phba->hbalock);
2495 			ndlp->nrport = NULL;
2496 			ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2497 			spin_unlock_irq(&vport->phba->hbalock);
2498 			rport->ndlp = NULL;
2499 			rport->remoteport = NULL;
2500 
2501 			/* Reference only removed if previous NDLP is no longer
2502 			 * active. It might be just a swap and removing the
2503 			 * reference would cause a premature cleanup.
2504 			 */
2505 			if (prev_ndlp && prev_ndlp != ndlp) {
2506 				if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
2507 				    (!prev_ndlp->nrport))
2508 					lpfc_nlp_put(prev_ndlp);
2509 			}
2510 		}
2511 
2512 		/* Clean bind the rport to the ndlp. */
2513 		rport->remoteport = remote_port;
2514 		rport->lport = lport;
2515 		rport->ndlp = ndlp;
2516 		spin_lock_irq(&vport->phba->hbalock);
2517 		ndlp->nrport = rport;
2518 		spin_unlock_irq(&vport->phba->hbalock);
2519 		lpfc_printf_vlog(vport, KERN_INFO,
2520 				 LOG_NVME_DISC | LOG_NODE,
2521 				 "6022 Bind lport x%px to remoteport x%px "
2522 				 "rport x%px WWNN 0x%llx, "
2523 				 "Rport WWPN 0x%llx DID "
2524 				 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2525 				 lport, remote_port, rport,
2526 				 rpinfo.node_name, rpinfo.port_name,
2527 				 rpinfo.port_id, rpinfo.port_role,
2528 				 ndlp, prev_ndlp);
2529 	} else {
2530 		lpfc_printf_vlog(vport, KERN_ERR,
2531 				 LOG_NVME_DISC | LOG_NODE,
2532 				 "6031 RemotePort Registration failed "
2533 				 "err: %d, DID x%06x\n",
2534 				 ret, ndlp->nlp_DID);
2535 	}
2536 
2537 	return ret;
2538 #else
2539 	return 0;
2540 #endif
2541 }
2542 
2543 /**
2544  * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2545  *
2546  * If the ndlp represents an NVME Target, that we are logged into,
2547  * ping the NVME FC Transport layer to initiate a device rescan
2548  * on this remote NPort.
2549  */
2550 void
2551 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2552 {
2553 #if (IS_ENABLED(CONFIG_NVME_FC))
2554 	struct lpfc_nvme_rport *nrport;
2555 	struct nvme_fc_remote_port *remoteport = NULL;
2556 
2557 	spin_lock_irq(&vport->phba->hbalock);
2558 	nrport = lpfc_ndlp_get_nrport(ndlp);
2559 	if (nrport)
2560 		remoteport = nrport->remoteport;
2561 	spin_unlock_irq(&vport->phba->hbalock);
2562 
2563 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2564 			 "6170 Rescan NPort DID x%06x type x%x "
2565 			 "state x%x nrport x%px remoteport x%px\n",
2566 			 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2567 			 nrport, remoteport);
2568 
2569 	if (!nrport || !remoteport)
2570 		goto rescan_exit;
2571 
2572 	/* Only rescan if we are an NVME target in the MAPPED state */
2573 	if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2574 	    ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2575 		nvme_fc_rescan_remoteport(remoteport);
2576 
2577 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
2578 				 "6172 NVME rescanned DID x%06x "
2579 				 "port_state x%x\n",
2580 				 ndlp->nlp_DID, remoteport->port_state);
2581 	}
2582 	return;
2583  rescan_exit:
2584 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2585 			 "6169 Skip NVME Rport Rescan, NVME remoteport "
2586 			 "unregistered\n");
2587 #endif
2588 }
2589 
2590 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2591  *
2592  * There is no notion of Devloss or rport recovery from the current
2593  * nvme_transport perspective.  Loss of an rport just means IO cannot
2594  * be sent and recovery is completely up to the initator.
2595  * For now, the driver just unbinds the DID and port_role so that
2596  * no further IO can be issued.  Changes are planned for later.
2597  *
2598  * Notes - the ndlp reference count is not decremented here since
2599  * since there is no nvme_transport api for devloss.  Node ref count
2600  * is only adjusted in driver unload.
2601  */
2602 void
2603 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2604 {
2605 #if (IS_ENABLED(CONFIG_NVME_FC))
2606 	int ret;
2607 	struct nvme_fc_local_port *localport;
2608 	struct lpfc_nvme_lport *lport;
2609 	struct lpfc_nvme_rport *rport;
2610 	struct nvme_fc_remote_port *remoteport = NULL;
2611 
2612 	localport = vport->localport;
2613 
2614 	/* This is fundamental error.  The localport is always
2615 	 * available until driver unload.  Just exit.
2616 	 */
2617 	if (!localport)
2618 		return;
2619 
2620 	lport = (struct lpfc_nvme_lport *)localport->private;
2621 	if (!lport)
2622 		goto input_err;
2623 
2624 	spin_lock_irq(&vport->phba->hbalock);
2625 	rport = lpfc_ndlp_get_nrport(ndlp);
2626 	if (rport)
2627 		remoteport = rport->remoteport;
2628 	spin_unlock_irq(&vport->phba->hbalock);
2629 	if (!remoteport)
2630 		goto input_err;
2631 
2632 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2633 			 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2634 			 "port_id x%06x, portstate x%x port type x%x\n",
2635 			 remoteport, remoteport->port_name,
2636 			 remoteport->port_id, remoteport->port_state,
2637 			 ndlp->nlp_type);
2638 
2639 	/* Sanity check ndlp type.  Only call for NVME ports. Don't
2640 	 * clear any rport state until the transport calls back.
2641 	 */
2642 
2643 	if (ndlp->nlp_type & NLP_NVME_TARGET) {
2644 		/* No concern about the role change on the nvme remoteport.
2645 		 * The transport will update it.
2646 		 */
2647 		ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
2648 
2649 		/* Don't let the host nvme transport keep sending keep-alives
2650 		 * on this remoteport. Vport is unloading, no recovery. The
2651 		 * return values is ignored.  The upcall is a courtesy to the
2652 		 * transport.
2653 		 */
2654 		if (vport->load_flag & FC_UNLOADING)
2655 			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2656 
2657 		ret = nvme_fc_unregister_remoteport(remoteport);
2658 		if (ret != 0) {
2659 			lpfc_nlp_put(ndlp);
2660 			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
2661 					 "6167 NVME unregister failed %d "
2662 					 "port_state x%x\n",
2663 					 ret, remoteport->port_state);
2664 		}
2665 	}
2666 	return;
2667 
2668  input_err:
2669 #endif
2670 	lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
2671 			 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2672 			 vport->localport, ndlp->rport, ndlp->nlp_DID);
2673 }
2674 
2675 /**
2676  * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2677  * @phba: pointer to lpfc hba data structure.
2678  * @axri: pointer to the fcp xri abort wcqe structure.
2679  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2680  *
2681  * This routine is invoked by the worker thread to process a SLI4 fast-path
2682  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2683  * here.
2684  **/
2685 void
2686 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2687 			   struct sli4_wcqe_xri_aborted *axri,
2688 			   struct lpfc_io_buf *lpfc_ncmd)
2689 {
2690 	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2691 	struct nvmefc_fcp_req *nvme_cmd = NULL;
2692 	struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2693 
2694 
2695 	if (ndlp)
2696 		lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2697 
2698 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2699 			"6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2700 			"xri released\n",
2701 			lpfc_ncmd->nvmeCmd, xri,
2702 			lpfc_ncmd->cur_iocbq.iotag);
2703 
2704 	/* Aborted NVME commands are required to not complete
2705 	 * before the abort exchange command fully completes.
2706 	 * Once completed, it is available via the put list.
2707 	 */
2708 	if (lpfc_ncmd->nvmeCmd) {
2709 		nvme_cmd = lpfc_ncmd->nvmeCmd;
2710 		nvme_cmd->done(nvme_cmd);
2711 		lpfc_ncmd->nvmeCmd = NULL;
2712 	}
2713 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2714 }
2715 
2716 /**
2717  * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2718  * @phba: Pointer to HBA context object.
2719  *
2720  * This function flushes all wqes in the nvme rings and frees all resources
2721  * in the txcmplq. This function does not issue abort wqes for the IO
2722  * commands in txcmplq, they will just be returned with
2723  * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2724  * slot has been permanently disabled.
2725  **/
2726 void
2727 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2728 {
2729 	struct lpfc_sli_ring  *pring;
2730 	u32 i, wait_cnt = 0;
2731 
2732 	if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2733 		return;
2734 
2735 	/* Cycle through all IO rings and make sure all outstanding
2736 	 * WQEs have been removed from the txcmplqs.
2737 	 */
2738 	for (i = 0; i < phba->cfg_hdw_queue; i++) {
2739 		if (!phba->sli4_hba.hdwq[i].io_wq)
2740 			continue;
2741 		pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2742 
2743 		if (!pring)
2744 			continue;
2745 
2746 		/* Retrieve everything on the txcmplq */
2747 		while (!list_empty(&pring->txcmplq)) {
2748 			msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2749 			wait_cnt++;
2750 
2751 			/* The sleep is 10mS.  Every ten seconds,
2752 			 * dump a message.  Something is wrong.
2753 			 */
2754 			if ((wait_cnt % 1000) == 0) {
2755 				lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2756 						"6178 NVME IO not empty, "
2757 						"cnt %d\n", wait_cnt);
2758 			}
2759 		}
2760 	}
2761 }
2762 
2763 void
2764 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn)
2765 {
2766 #if (IS_ENABLED(CONFIG_NVME_FC))
2767 	struct lpfc_io_buf *lpfc_ncmd;
2768 	struct nvmefc_fcp_req *nCmd;
2769 	struct lpfc_nvme_fcpreq_priv *freqpriv;
2770 
2771 	if (!pwqeIn->context1) {
2772 		lpfc_sli_release_iocbq(phba, pwqeIn);
2773 		return;
2774 	}
2775 	/* For abort iocb just return, IO iocb will do a done call */
2776 	if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2777 	    CMD_ABORT_XRI_CX) {
2778 		lpfc_sli_release_iocbq(phba, pwqeIn);
2779 		return;
2780 	}
2781 	lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1;
2782 
2783 	spin_lock(&lpfc_ncmd->buf_lock);
2784 	if (!lpfc_ncmd->nvmeCmd) {
2785 		spin_unlock(&lpfc_ncmd->buf_lock);
2786 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
2787 		return;
2788 	}
2789 
2790 	nCmd = lpfc_ncmd->nvmeCmd;
2791 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2792 			"6194 NVME Cancel xri %x\n",
2793 			lpfc_ncmd->cur_iocbq.sli4_xritag);
2794 
2795 	nCmd->transferred_length = 0;
2796 	nCmd->rcv_rsplen = 0;
2797 	nCmd->status = NVME_SC_INTERNAL;
2798 	freqpriv = nCmd->private;
2799 	freqpriv->nvme_buf = NULL;
2800 	lpfc_ncmd->nvmeCmd = NULL;
2801 
2802 	spin_unlock(&lpfc_ncmd->buf_lock);
2803 	nCmd->done(nCmd);
2804 
2805 	/* Call release with XB=1 to queue the IO into the abort list. */
2806 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2807 #endif
2808 }
2809