xref: /openbmc/linux/drivers/scsi/lpfc/lpfc_nvme.c (revision 8795a739)
1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channel Host Bus Adapters.                                *
4  * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term *
5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30 
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38 
39 #include <linux/nvme.h>
40 #include <linux/nvme-fc-driver.h>
41 #include <linux/nvme-fc.h>
42 #include "lpfc_version.h"
43 #include "lpfc_hw4.h"
44 #include "lpfc_hw.h"
45 #include "lpfc_sli.h"
46 #include "lpfc_sli4.h"
47 #include "lpfc_nl.h"
48 #include "lpfc_disc.h"
49 #include "lpfc.h"
50 #include "lpfc_nvme.h"
51 #include "lpfc_scsi.h"
52 #include "lpfc_logmsg.h"
53 #include "lpfc_crtn.h"
54 #include "lpfc_vport.h"
55 #include "lpfc_debugfs.h"
56 
57 /* NVME initiator-based functions */
58 
59 static struct lpfc_io_buf *
60 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
61 		  int idx, int expedite);
62 
63 static void
64 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
65 
66 static struct nvme_fc_port_template lpfc_nvme_template;
67 
68 static union lpfc_wqe128 lpfc_iread_cmd_template;
69 static union lpfc_wqe128 lpfc_iwrite_cmd_template;
70 static union lpfc_wqe128 lpfc_icmnd_cmd_template;
71 
72 /* Setup WQE templates for NVME IOs */
73 void
74 lpfc_nvme_cmd_template(void)
75 {
76 	union lpfc_wqe128 *wqe;
77 
78 	/* IREAD template */
79 	wqe = &lpfc_iread_cmd_template;
80 	memset(wqe, 0, sizeof(union lpfc_wqe128));
81 
82 	/* Word 0, 1, 2 - BDE is variable */
83 
84 	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
85 
86 	/* Word 4 - total_xfer_len is variable */
87 
88 	/* Word 5 - is zero */
89 
90 	/* Word 6 - ctxt_tag, xri_tag is variable */
91 
92 	/* Word 7 */
93 	bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE);
94 	bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK);
95 	bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3);
96 	bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI);
97 
98 	/* Word 8 - abort_tag is variable */
99 
100 	/* Word 9  - reqtag is variable */
101 
102 	/* Word 10 - dbde, wqes is variable */
103 	bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0);
104 	bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1);
105 	bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ);
106 	bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4);
107 	bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0);
108 	bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1);
109 
110 	/* Word 11 - pbde is variable */
111 	bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD);
112 	bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
113 	bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1);
114 
115 	/* Word 12 - is zero */
116 
117 	/* Word 13, 14, 15 - PBDE is variable */
118 
119 	/* IWRITE template */
120 	wqe = &lpfc_iwrite_cmd_template;
121 	memset(wqe, 0, sizeof(union lpfc_wqe128));
122 
123 	/* Word 0, 1, 2 - BDE is variable */
124 
125 	/* Word 3 - cmd_buff_len, payload_offset_len is zero */
126 
127 	/* Word 4 - total_xfer_len is variable */
128 
129 	/* Word 5 - initial_xfer_len is variable */
130 
131 	/* Word 6 - ctxt_tag, xri_tag is variable */
132 
133 	/* Word 7 */
134 	bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE);
135 	bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK);
136 	bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3);
137 	bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI);
138 
139 	/* Word 8 - abort_tag is variable */
140 
141 	/* Word 9  - reqtag is variable */
142 
143 	/* Word 10 - dbde, wqes is variable */
144 	bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0);
145 	bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1);
146 	bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE);
147 	bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4);
148 	bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0);
149 	bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
150 
151 	/* Word 11 - pbde is variable */
152 	bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD);
153 	bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
154 	bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1);
155 
156 	/* Word 12 - is zero */
157 
158 	/* Word 13, 14, 15 - PBDE is variable */
159 
160 	/* ICMND template */
161 	wqe = &lpfc_icmnd_cmd_template;
162 	memset(wqe, 0, sizeof(union lpfc_wqe128));
163 
164 	/* Word 0, 1, 2 - BDE is variable */
165 
166 	/* Word 3 - payload_offset_len is variable */
167 
168 	/* Word 4, 5 - is zero */
169 
170 	/* Word 6 - ctxt_tag, xri_tag is variable */
171 
172 	/* Word 7 */
173 	bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE);
174 	bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0);
175 	bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3);
176 	bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI);
177 
178 	/* Word 8 - abort_tag is variable */
179 
180 	/* Word 9  - reqtag is variable */
181 
182 	/* Word 10 - dbde, wqes is variable */
183 	bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1);
184 	bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1);
185 	bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE);
186 	bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE);
187 	bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0);
188 	bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1);
189 
190 	/* Word 11 */
191 	bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND);
192 	bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
193 	bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0);
194 
195 	/* Word 12, 13, 14, 15 - is zero */
196 }
197 
198 /**
199  * lpfc_nvme_create_queue -
200  * @lpfc_pnvme: Pointer to the driver's nvme instance data
201  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
202  * @handle: An opaque driver handle used in follow-up calls.
203  *
204  * Driver registers this routine to preallocate and initialize any
205  * internal data structures to bind the @qidx to its internal IO queues.
206  * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
207  *
208  * Return value :
209  *   0 - Success
210  *   -EINVAL - Unsupported input value.
211  *   -ENOMEM - Could not alloc necessary memory
212  **/
213 static int
214 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
215 		       unsigned int qidx, u16 qsize,
216 		       void **handle)
217 {
218 	struct lpfc_nvme_lport *lport;
219 	struct lpfc_vport *vport;
220 	struct lpfc_nvme_qhandle *qhandle;
221 	char *str;
222 
223 	if (!pnvme_lport->private)
224 		return -ENOMEM;
225 
226 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
227 	vport = lport->vport;
228 	qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
229 	if (qhandle == NULL)
230 		return -ENOMEM;
231 
232 	qhandle->cpu_id = raw_smp_processor_id();
233 	qhandle->qidx = qidx;
234 	/*
235 	 * NVME qidx == 0 is the admin queue, so both admin queue
236 	 * and first IO queue will use MSI-X vector and associated
237 	 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
238 	 */
239 	if (qidx) {
240 		str = "IO ";  /* IO queue */
241 		qhandle->index = ((qidx - 1) %
242 			lpfc_nvme_template.max_hw_queues);
243 	} else {
244 		str = "ADM";  /* Admin queue */
245 		qhandle->index = qidx;
246 	}
247 
248 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
249 			 "6073 Binding %s HdwQueue %d  (cpu %d) to "
250 			 "hdw_queue %d qhandle x%px\n", str,
251 			 qidx, qhandle->cpu_id, qhandle->index, qhandle);
252 	*handle = (void *)qhandle;
253 	return 0;
254 }
255 
256 /**
257  * lpfc_nvme_delete_queue -
258  * @lpfc_pnvme: Pointer to the driver's nvme instance data
259  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
260  * @handle: An opaque driver handle from lpfc_nvme_create_queue
261  *
262  * Driver registers this routine to free
263  * any internal data structures to bind the @qidx to its internal
264  * IO queues.
265  *
266  * Return value :
267  *   0 - Success
268  *   TODO:  What are the failure codes.
269  **/
270 static void
271 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
272 		       unsigned int qidx,
273 		       void *handle)
274 {
275 	struct lpfc_nvme_lport *lport;
276 	struct lpfc_vport *vport;
277 
278 	if (!pnvme_lport->private)
279 		return;
280 
281 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
282 	vport = lport->vport;
283 
284 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
285 			"6001 ENTER.  lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
286 			lport, qidx, handle);
287 	kfree(handle);
288 }
289 
290 static void
291 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
292 {
293 	struct lpfc_nvme_lport *lport = localport->private;
294 
295 	lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
296 			 "6173 localport x%px delete complete\n",
297 			 lport);
298 
299 	/* release any threads waiting for the unreg to complete */
300 	if (lport->vport->localport)
301 		complete(lport->lport_unreg_cmp);
302 }
303 
304 /* lpfc_nvme_remoteport_delete
305  *
306  * @remoteport: Pointer to an nvme transport remoteport instance.
307  *
308  * This is a template downcall.  NVME transport calls this function
309  * when it has completed the unregistration of a previously
310  * registered remoteport.
311  *
312  * Return value :
313  * None
314  */
315 static void
316 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
317 {
318 	struct lpfc_nvme_rport *rport = remoteport->private;
319 	struct lpfc_vport *vport;
320 	struct lpfc_nodelist *ndlp;
321 
322 	ndlp = rport->ndlp;
323 	if (!ndlp)
324 		goto rport_err;
325 
326 	vport = ndlp->vport;
327 	if (!vport)
328 		goto rport_err;
329 
330 	/* Remove this rport from the lport's list - memory is owned by the
331 	 * transport. Remove the ndlp reference for the NVME transport before
332 	 * calling state machine to remove the node.
333 	 */
334 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
335 			"6146 remoteport delete of remoteport x%px\n",
336 			remoteport);
337 	spin_lock_irq(&vport->phba->hbalock);
338 
339 	/* The register rebind might have occurred before the delete
340 	 * downcall.  Guard against this race.
341 	 */
342 	if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) {
343 		ndlp->nrport = NULL;
344 		ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
345 	}
346 	spin_unlock_irq(&vport->phba->hbalock);
347 
348 	/* Remove original register reference. The host transport
349 	 * won't reference this rport/remoteport any further.
350 	 */
351 	lpfc_nlp_put(ndlp);
352 
353  rport_err:
354 	return;
355 }
356 
357 static void
358 lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
359 		       struct lpfc_wcqe_complete *wcqe)
360 {
361 	struct lpfc_vport *vport = cmdwqe->vport;
362 	struct lpfc_nvme_lport *lport;
363 	uint32_t status;
364 	struct nvmefc_ls_req *pnvme_lsreq;
365 	struct lpfc_dmabuf *buf_ptr;
366 	struct lpfc_nodelist *ndlp;
367 
368 	pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2;
369 	status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
370 
371 	if (vport->localport) {
372 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
373 		if (lport) {
374 			atomic_inc(&lport->fc4NvmeLsCmpls);
375 			if (status) {
376 				if (bf_get(lpfc_wcqe_c_xb, wcqe))
377 					atomic_inc(&lport->cmpl_ls_xb);
378 				atomic_inc(&lport->cmpl_ls_err);
379 			}
380 		}
381 	}
382 
383 	ndlp = (struct lpfc_nodelist *)cmdwqe->context1;
384 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
385 			 "6047 nvme cmpl Enter "
386 			 "Data %px DID %x Xri: %x status %x reason x%x "
387 			 "cmd:x%px lsreg:x%px bmp:x%px ndlp:x%px\n",
388 			 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
389 			 cmdwqe->sli4_xritag, status,
390 			 (wcqe->parameter & 0xffff),
391 			 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp);
392 
393 	lpfc_nvmeio_data(phba, "NVME LS  CMPL: xri x%x stat x%x parm x%x\n",
394 			 cmdwqe->sli4_xritag, status, wcqe->parameter);
395 
396 	if (cmdwqe->context3) {
397 		buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3;
398 		lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
399 		kfree(buf_ptr);
400 		cmdwqe->context3 = NULL;
401 	}
402 	if (pnvme_lsreq->done)
403 		pnvme_lsreq->done(pnvme_lsreq, status);
404 	else
405 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
406 				 "6046 nvme cmpl without done call back? "
407 				 "Data %px DID %x Xri: %x status %x\n",
408 				pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
409 				cmdwqe->sli4_xritag, status);
410 	if (ndlp) {
411 		lpfc_nlp_put(ndlp);
412 		cmdwqe->context1 = NULL;
413 	}
414 	lpfc_sli_release_iocbq(phba, cmdwqe);
415 }
416 
417 static int
418 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
419 		  struct lpfc_dmabuf *inp,
420 		  struct nvmefc_ls_req *pnvme_lsreq,
421 		  void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
422 			       struct lpfc_wcqe_complete *),
423 		  struct lpfc_nodelist *ndlp, uint32_t num_entry,
424 		  uint32_t tmo, uint8_t retry)
425 {
426 	struct lpfc_hba *phba = vport->phba;
427 	union lpfc_wqe128 *wqe;
428 	struct lpfc_iocbq *genwqe;
429 	struct ulp_bde64 *bpl;
430 	struct ulp_bde64 bde;
431 	int i, rc, xmit_len, first_len;
432 
433 	/* Allocate buffer for  command WQE */
434 	genwqe = lpfc_sli_get_iocbq(phba);
435 	if (genwqe == NULL)
436 		return 1;
437 
438 	wqe = &genwqe->wqe;
439 	/* Initialize only 64 bytes */
440 	memset(wqe, 0, sizeof(union lpfc_wqe));
441 
442 	genwqe->context3 = (uint8_t *)bmp;
443 	genwqe->iocb_flag |= LPFC_IO_NVME_LS;
444 
445 	/* Save for completion so we can release these resources */
446 	genwqe->context1 = lpfc_nlp_get(ndlp);
447 	genwqe->context2 = (uint8_t *)pnvme_lsreq;
448 	/* Fill in payload, bp points to frame payload */
449 
450 	if (!tmo)
451 		/* FC spec states we need 3 * ratov for CT requests */
452 		tmo = (3 * phba->fc_ratov);
453 
454 	/* For this command calculate the xmit length of the request bde. */
455 	xmit_len = 0;
456 	first_len = 0;
457 	bpl = (struct ulp_bde64 *)bmp->virt;
458 	for (i = 0; i < num_entry; i++) {
459 		bde.tus.w = bpl[i].tus.w;
460 		if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
461 			break;
462 		xmit_len += bde.tus.f.bdeSize;
463 		if (i == 0)
464 			first_len = xmit_len;
465 	}
466 
467 	genwqe->rsvd2 = num_entry;
468 	genwqe->hba_wqidx = 0;
469 
470 	/* Words 0 - 2 */
471 	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
472 	wqe->generic.bde.tus.f.bdeSize = first_len;
473 	wqe->generic.bde.addrLow = bpl[0].addrLow;
474 	wqe->generic.bde.addrHigh = bpl[0].addrHigh;
475 
476 	/* Word 3 */
477 	wqe->gen_req.request_payload_len = first_len;
478 
479 	/* Word 4 */
480 
481 	/* Word 5 */
482 	bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
483 	bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
484 	bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
485 	bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
486 	bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
487 
488 	/* Word 6 */
489 	bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
490 	       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
491 	bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
492 
493 	/* Word 7 */
494 	bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1));
495 	bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
496 	bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
497 	bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
498 
499 	/* Word 8 */
500 	wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
501 
502 	/* Word 9 */
503 	bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
504 
505 	/* Word 10 */
506 	bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
507 	bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
508 	bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
509 	bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
510 	bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
511 
512 	/* Word 11 */
513 	bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
514 	bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
515 
516 
517 	/* Issue GEN REQ WQE for NPORT <did> */
518 	lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS,
519 			 "6050 Issue GEN REQ WQE to NPORT x%x "
520 			 "Data: x%x x%x wq:x%px lsreq:x%px bmp:x%px "
521 			 "xmit:%d 1st:%d\n",
522 			 ndlp->nlp_DID, genwqe->iotag,
523 			 vport->port_state,
524 			genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
525 	genwqe->wqe_cmpl = cmpl;
526 	genwqe->iocb_cmpl = NULL;
527 	genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
528 	genwqe->vport = vport;
529 	genwqe->retry = retry;
530 
531 	lpfc_nvmeio_data(phba, "NVME LS  XMIT: xri x%x iotag x%x to x%06x\n",
532 			 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
533 
534 	rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
535 	if (rc) {
536 		lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS,
537 				 "6045 Issue GEN REQ WQE to NPORT x%x "
538 				 "Data: x%x x%x\n",
539 				 ndlp->nlp_DID, genwqe->iotag,
540 				 vport->port_state);
541 		lpfc_sli_release_iocbq(phba, genwqe);
542 		return 1;
543 	}
544 	return 0;
545 }
546 
547 /**
548  * lpfc_nvme_ls_req - Issue an Link Service request
549  * @lpfc_pnvme: Pointer to the driver's nvme instance data
550  * @lpfc_nvme_lport: Pointer to the driver's local port data
551  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
552  *
553  * Driver registers this routine to handle any link service request
554  * from the nvme_fc transport to a remote nvme-aware port.
555  *
556  * Return value :
557  *   0 - Success
558  *   TODO: What are the failure codes.
559  **/
560 static int
561 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
562 		 struct nvme_fc_remote_port *pnvme_rport,
563 		 struct nvmefc_ls_req *pnvme_lsreq)
564 {
565 	int ret = 0;
566 	struct lpfc_nvme_lport *lport;
567 	struct lpfc_nvme_rport *rport;
568 	struct lpfc_vport *vport;
569 	struct lpfc_nodelist *ndlp;
570 	struct ulp_bde64 *bpl;
571 	struct lpfc_dmabuf *bmp;
572 	uint16_t ntype, nstate;
573 
574 	/* there are two dma buf in the request, actually there is one and
575 	 * the second one is just the start address + cmd size.
576 	 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
577 	 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
578 	 * because the nvem layer owns the data bufs.
579 	 * We do not have to break these packets open, we don't care what is in
580 	 * them. And we do not have to look at the resonse data, we only care
581 	 * that we got a response. All of the caring is going to happen in the
582 	 * nvme-fc layer.
583 	 */
584 
585 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
586 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
587 	if (unlikely(!lport) || unlikely(!rport))
588 		return -EINVAL;
589 
590 	vport = lport->vport;
591 
592 	if (vport->load_flag & FC_UNLOADING)
593 		return -ENODEV;
594 
595 	/* Need the ndlp.  It is stored in the driver's rport. */
596 	ndlp = rport->ndlp;
597 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
598 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
599 				 "6051 Remoteport x%px, rport has invalid ndlp. "
600 				 "Failing LS Req\n", pnvme_rport);
601 		return -ENODEV;
602 	}
603 
604 	/* The remote node has to be a mapped nvme target or an
605 	 * unmapped nvme initiator or it's an error.
606 	 */
607 	ntype = ndlp->nlp_type;
608 	nstate = ndlp->nlp_state;
609 	if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
610 	    (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
611 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
612 				 "6088 DID x%06x not ready for "
613 				 "IO. State x%x, Type x%x\n",
614 				 pnvme_rport->port_id,
615 				 ndlp->nlp_state, ndlp->nlp_type);
616 		return -ENODEV;
617 	}
618 	bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL);
619 	if (!bmp) {
620 
621 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
622 				 "6044 Could not find node for DID %x\n",
623 				 pnvme_rport->port_id);
624 		return 2;
625 	}
626 	INIT_LIST_HEAD(&bmp->list);
627 	bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
628 	if (!bmp->virt) {
629 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
630 				 "6042 Could not find node for DID %x\n",
631 				 pnvme_rport->port_id);
632 		kfree(bmp);
633 		return 3;
634 	}
635 	bpl = (struct ulp_bde64 *)bmp->virt;
636 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
637 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
638 	bpl->tus.f.bdeFlags = 0;
639 	bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
640 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
641 	bpl++;
642 
643 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
644 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
645 	bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
646 	bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
647 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
648 
649 	/* Expand print to include key fields. */
650 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
651 			 "6149 Issue LS Req to DID 0x%06x lport x%px, "
652 			 "rport x%px lsreq x%px rqstlen:%d rsplen:%d "
653 			 "%pad %pad\n",
654 			 ndlp->nlp_DID, pnvme_lport, pnvme_rport,
655 			 pnvme_lsreq, pnvme_lsreq->rqstlen,
656 			 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
657 			 &pnvme_lsreq->rspdma);
658 
659 	atomic_inc(&lport->fc4NvmeLsRequests);
660 
661 	/* Hardcode the wait to 30 seconds.  Connections are failing otherwise.
662 	 * This code allows it all to work.
663 	 */
664 	ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
665 				pnvme_lsreq, lpfc_nvme_cmpl_gen_req,
666 				ndlp, 2, 30, 0);
667 	if (ret != WQE_SUCCESS) {
668 		atomic_inc(&lport->xmt_ls_err);
669 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
670 				 "6052 EXIT. issue ls wqe failed lport x%px, "
671 				 "rport x%px lsreq x%px Status %x DID %x\n",
672 				 pnvme_lport, pnvme_rport, pnvme_lsreq,
673 				 ret, ndlp->nlp_DID);
674 		lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
675 		kfree(bmp);
676 		return ret;
677 	}
678 
679 	/* Stub in routine and return 0 for now. */
680 	return ret;
681 }
682 
683 /**
684  * lpfc_nvme_ls_abort - Issue an Link Service request
685  * @lpfc_pnvme: Pointer to the driver's nvme instance data
686  * @lpfc_nvme_lport: Pointer to the driver's local port data
687  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
688  *
689  * Driver registers this routine to handle any link service request
690  * from the nvme_fc transport to a remote nvme-aware port.
691  *
692  * Return value :
693  *   0 - Success
694  *   TODO: What are the failure codes.
695  **/
696 static void
697 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
698 		   struct nvme_fc_remote_port *pnvme_rport,
699 		   struct nvmefc_ls_req *pnvme_lsreq)
700 {
701 	struct lpfc_nvme_lport *lport;
702 	struct lpfc_vport *vport;
703 	struct lpfc_hba *phba;
704 	struct lpfc_nodelist *ndlp;
705 	LIST_HEAD(abort_list);
706 	struct lpfc_sli_ring *pring;
707 	struct lpfc_iocbq *wqe, *next_wqe;
708 
709 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
710 	if (unlikely(!lport))
711 		return;
712 	vport = lport->vport;
713 	phba = vport->phba;
714 
715 	if (vport->load_flag & FC_UNLOADING)
716 		return;
717 
718 	ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
719 	if (!ndlp) {
720 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
721 				 "6049 Could not find node for DID %x\n",
722 				 pnvme_rport->port_id);
723 		return;
724 	}
725 
726 	/* Expand print to include key fields. */
727 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
728 			 "6040 ENTER.  lport x%px, rport x%px lsreq x%px rqstlen:%d "
729 			 "rsplen:%d %pad %pad\n",
730 			 pnvme_lport, pnvme_rport,
731 			 pnvme_lsreq, pnvme_lsreq->rqstlen,
732 			 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
733 			 &pnvme_lsreq->rspdma);
734 
735 	/*
736 	 * Lock the ELS ring txcmplq and build a local list of all ELS IOs
737 	 * that need an ABTS.  The IOs need to stay on the txcmplq so that
738 	 * the abort operation completes them successfully.
739 	 */
740 	pring = phba->sli4_hba.nvmels_wq->pring;
741 	spin_lock_irq(&phba->hbalock);
742 	spin_lock(&pring->ring_lock);
743 	list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
744 		/* Add to abort_list on on NDLP match. */
745 		if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) {
746 			wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
747 			list_add_tail(&wqe->dlist, &abort_list);
748 		}
749 	}
750 	spin_unlock(&pring->ring_lock);
751 	spin_unlock_irq(&phba->hbalock);
752 
753 	/* Abort the targeted IOs and remove them from the abort list. */
754 	list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) {
755 		atomic_inc(&lport->xmt_ls_abort);
756 		spin_lock_irq(&phba->hbalock);
757 		list_del_init(&wqe->dlist);
758 		lpfc_sli_issue_abort_iotag(phba, pring, wqe);
759 		spin_unlock_irq(&phba->hbalock);
760 	}
761 }
762 
763 /* Fix up the existing sgls for NVME IO. */
764 static inline void
765 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
766 		       struct lpfc_io_buf *lpfc_ncmd,
767 		       struct nvmefc_fcp_req *nCmd)
768 {
769 	struct lpfc_hba  *phba = vport->phba;
770 	struct sli4_sge *sgl;
771 	union lpfc_wqe128 *wqe;
772 	uint32_t *wptr, *dptr;
773 
774 	/*
775 	 * Get a local pointer to the built-in wqe and correct
776 	 * the cmd size to match NVME's 96 bytes and fix
777 	 * the dma address.
778 	 */
779 
780 	wqe = &lpfc_ncmd->cur_iocbq.wqe;
781 
782 	/*
783 	 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
784 	 * match NVME.  NVME sends 96 bytes. Also, use the
785 	 * nvme commands command and response dma addresses
786 	 * rather than the virtual memory to ease the restore
787 	 * operation.
788 	 */
789 	sgl = lpfc_ncmd->dma_sgl;
790 	sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
791 	if (phba->cfg_nvme_embed_cmd) {
792 		sgl->addr_hi = 0;
793 		sgl->addr_lo = 0;
794 
795 		/* Word 0-2 - NVME CMND IU (embedded payload) */
796 		wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
797 		wqe->generic.bde.tus.f.bdeSize = 56;
798 		wqe->generic.bde.addrHigh = 0;
799 		wqe->generic.bde.addrLow =  64;  /* Word 16 */
800 
801 		/* Word 10  - dbde is 0, wqes is 1 in template */
802 
803 		/*
804 		 * Embed the payload in the last half of the WQE
805 		 * WQE words 16-30 get the NVME CMD IU payload
806 		 *
807 		 * WQE words 16-19 get payload Words 1-4
808 		 * WQE words 20-21 get payload Words 6-7
809 		 * WQE words 22-29 get payload Words 16-23
810 		 */
811 		wptr = &wqe->words[16];  /* WQE ptr */
812 		dptr = (uint32_t *)nCmd->cmdaddr;  /* payload ptr */
813 		dptr++;			/* Skip Word 0 in payload */
814 
815 		*wptr++ = *dptr++;	/* Word 1 */
816 		*wptr++ = *dptr++;	/* Word 2 */
817 		*wptr++ = *dptr++;	/* Word 3 */
818 		*wptr++ = *dptr++;	/* Word 4 */
819 		dptr++;			/* Skip Word 5 in payload */
820 		*wptr++ = *dptr++;	/* Word 6 */
821 		*wptr++ = *dptr++;	/* Word 7 */
822 		dptr += 8;		/* Skip Words 8-15 in payload */
823 		*wptr++ = *dptr++;	/* Word 16 */
824 		*wptr++ = *dptr++;	/* Word 17 */
825 		*wptr++ = *dptr++;	/* Word 18 */
826 		*wptr++ = *dptr++;	/* Word 19 */
827 		*wptr++ = *dptr++;	/* Word 20 */
828 		*wptr++ = *dptr++;	/* Word 21 */
829 		*wptr++ = *dptr++;	/* Word 22 */
830 		*wptr   = *dptr;	/* Word 23 */
831 	} else {
832 		sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
833 		sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
834 
835 		/* Word 0-2 - NVME CMND IU Inline BDE */
836 		wqe->generic.bde.tus.f.bdeFlags =  BUFF_TYPE_BDE_64;
837 		wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
838 		wqe->generic.bde.addrHigh = sgl->addr_hi;
839 		wqe->generic.bde.addrLow =  sgl->addr_lo;
840 
841 		/* Word 10 */
842 		bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
843 		bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
844 	}
845 
846 	sgl++;
847 
848 	/* Setup the physical region for the FCP RSP */
849 	sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
850 	sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
851 	sgl->word2 = le32_to_cpu(sgl->word2);
852 	if (nCmd->sg_cnt)
853 		bf_set(lpfc_sli4_sge_last, sgl, 0);
854 	else
855 		bf_set(lpfc_sli4_sge_last, sgl, 1);
856 	sgl->word2 = cpu_to_le32(sgl->word2);
857 	sgl->sge_len = cpu_to_le32(nCmd->rsplen);
858 }
859 
860 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
861 static void
862 lpfc_nvme_ktime(struct lpfc_hba *phba,
863 		struct lpfc_io_buf *lpfc_ncmd)
864 {
865 	uint64_t seg1, seg2, seg3, seg4;
866 	uint64_t segsum;
867 
868 	if (!lpfc_ncmd->ts_last_cmd ||
869 	    !lpfc_ncmd->ts_cmd_start ||
870 	    !lpfc_ncmd->ts_cmd_wqput ||
871 	    !lpfc_ncmd->ts_isr_cmpl ||
872 	    !lpfc_ncmd->ts_data_nvme)
873 		return;
874 
875 	if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_cmd_start)
876 		return;
877 	if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd)
878 		return;
879 	if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start)
880 		return;
881 	if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput)
882 		return;
883 	if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl)
884 		return;
885 	/*
886 	 * Segment 1 - Time from Last FCP command cmpl is handed
887 	 * off to NVME Layer to start of next command.
888 	 * Segment 2 - Time from Driver receives a IO cmd start
889 	 * from NVME Layer to WQ put is done on IO cmd.
890 	 * Segment 3 - Time from Driver WQ put is done on IO cmd
891 	 * to MSI-X ISR for IO cmpl.
892 	 * Segment 4 - Time from MSI-X ISR for IO cmpl to when
893 	 * cmpl is handled off to the NVME Layer.
894 	 */
895 	seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd;
896 	if (seg1 > 5000000)  /* 5 ms - for sequential IOs only */
897 		seg1 = 0;
898 
899 	/* Calculate times relative to start of IO */
900 	seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start);
901 	segsum = seg2;
902 	seg3 = lpfc_ncmd->ts_isr_cmpl - lpfc_ncmd->ts_cmd_start;
903 	if (segsum > seg3)
904 		return;
905 	seg3 -= segsum;
906 	segsum += seg3;
907 
908 	seg4 = lpfc_ncmd->ts_data_nvme - lpfc_ncmd->ts_cmd_start;
909 	if (segsum > seg4)
910 		return;
911 	seg4 -= segsum;
912 
913 	phba->ktime_data_samples++;
914 	phba->ktime_seg1_total += seg1;
915 	if (seg1 < phba->ktime_seg1_min)
916 		phba->ktime_seg1_min = seg1;
917 	else if (seg1 > phba->ktime_seg1_max)
918 		phba->ktime_seg1_max = seg1;
919 	phba->ktime_seg2_total += seg2;
920 	if (seg2 < phba->ktime_seg2_min)
921 		phba->ktime_seg2_min = seg2;
922 	else if (seg2 > phba->ktime_seg2_max)
923 		phba->ktime_seg2_max = seg2;
924 	phba->ktime_seg3_total += seg3;
925 	if (seg3 < phba->ktime_seg3_min)
926 		phba->ktime_seg3_min = seg3;
927 	else if (seg3 > phba->ktime_seg3_max)
928 		phba->ktime_seg3_max = seg3;
929 	phba->ktime_seg4_total += seg4;
930 	if (seg4 < phba->ktime_seg4_min)
931 		phba->ktime_seg4_min = seg4;
932 	else if (seg4 > phba->ktime_seg4_max)
933 		phba->ktime_seg4_max = seg4;
934 
935 	lpfc_ncmd->ts_last_cmd = 0;
936 	lpfc_ncmd->ts_cmd_start = 0;
937 	lpfc_ncmd->ts_cmd_wqput  = 0;
938 	lpfc_ncmd->ts_isr_cmpl = 0;
939 	lpfc_ncmd->ts_data_nvme = 0;
940 }
941 #endif
942 
943 /**
944  * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
945  * @lpfc_pnvme: Pointer to the driver's nvme instance data
946  * @lpfc_nvme_lport: Pointer to the driver's local port data
947  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
948  *
949  * Driver registers this routine as it io request handler.  This
950  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
951  * data structure to the rport indicated in @lpfc_nvme_rport.
952  *
953  * Return value :
954  *   0 - Success
955  *   TODO: What are the failure codes.
956  **/
957 static void
958 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
959 			  struct lpfc_wcqe_complete *wcqe)
960 {
961 	struct lpfc_io_buf *lpfc_ncmd =
962 		(struct lpfc_io_buf *)pwqeIn->context1;
963 	struct lpfc_vport *vport = pwqeIn->vport;
964 	struct nvmefc_fcp_req *nCmd;
965 	struct nvme_fc_ersp_iu *ep;
966 	struct nvme_fc_cmd_iu *cp;
967 	struct lpfc_nodelist *ndlp;
968 	struct lpfc_nvme_fcpreq_priv *freqpriv;
969 	struct lpfc_nvme_lport *lport;
970 	uint32_t code, status, idx;
971 	uint16_t cid, sqhd, data;
972 	uint32_t *ptr;
973 
974 	/* Sanity check on return of outstanding command */
975 	if (!lpfc_ncmd) {
976 		lpfc_printf_vlog(vport, KERN_ERR,
977 				 LOG_NODE | LOG_NVME_IOERR,
978 				 "6071 Null lpfc_ncmd pointer. No "
979 				 "release, skip completion\n");
980 		return;
981 	}
982 
983 	/* Guard against abort handler being called at same time */
984 	spin_lock(&lpfc_ncmd->buf_lock);
985 
986 	if (!lpfc_ncmd->nvmeCmd) {
987 		spin_unlock(&lpfc_ncmd->buf_lock);
988 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR,
989 				 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
990 				 "nvmeCmd x%px\n",
991 				 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
992 
993 		/* Release the lpfc_ncmd regardless of the missing elements. */
994 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
995 		return;
996 	}
997 	nCmd = lpfc_ncmd->nvmeCmd;
998 	status = bf_get(lpfc_wcqe_c_status, wcqe);
999 
1000 	idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
1001 	phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
1002 
1003 	if (unlikely(status && vport->localport)) {
1004 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
1005 		if (lport) {
1006 			if (bf_get(lpfc_wcqe_c_xb, wcqe))
1007 				atomic_inc(&lport->cmpl_fcp_xb);
1008 			atomic_inc(&lport->cmpl_fcp_err);
1009 		}
1010 	}
1011 
1012 	lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
1013 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
1014 			 status, wcqe->parameter);
1015 	/*
1016 	 * Catch race where our node has transitioned, but the
1017 	 * transport is still transitioning.
1018 	 */
1019 	ndlp = lpfc_ncmd->ndlp;
1020 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1021 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
1022 				 "6062 Ignoring NVME cmpl.  No ndlp\n");
1023 		goto out_err;
1024 	}
1025 
1026 	code = bf_get(lpfc_wcqe_c_code, wcqe);
1027 	if (code == CQE_CODE_NVME_ERSP) {
1028 		/* For this type of CQE, we need to rebuild the rsp */
1029 		ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1030 
1031 		/*
1032 		 * Get Command Id from cmd to plug into response. This
1033 		 * code is not needed in the next NVME Transport drop.
1034 		 */
1035 		cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1036 		cid = cp->sqe.common.command_id;
1037 
1038 		/*
1039 		 * RSN is in CQE word 2
1040 		 * SQHD is in CQE Word 3 bits 15:0
1041 		 * Cmd Specific info is in CQE Word 1
1042 		 * and in CQE Word 0 bits 15:0
1043 		 */
1044 		sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1045 
1046 		/* Now lets build the NVME ERSP IU */
1047 		ep->iu_len = cpu_to_be16(8);
1048 		ep->rsn = wcqe->parameter;
1049 		ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1050 		ep->rsvd12 = 0;
1051 		ptr = (uint32_t *)&ep->cqe.result.u64;
1052 		*ptr++ = wcqe->total_data_placed;
1053 		data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1054 		*ptr = (uint32_t)data;
1055 		ep->cqe.sq_head = sqhd;
1056 		ep->cqe.sq_id =  nCmd->sqid;
1057 		ep->cqe.command_id = cid;
1058 		ep->cqe.status = 0;
1059 
1060 		lpfc_ncmd->status = IOSTAT_SUCCESS;
1061 		lpfc_ncmd->result = 0;
1062 		nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1063 		nCmd->transferred_length = nCmd->payload_length;
1064 	} else {
1065 		lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);
1066 		lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1067 
1068 		/* For NVME, the only failure path that results in an
1069 		 * IO error is when the adapter rejects it.  All other
1070 		 * conditions are a success case and resolved by the
1071 		 * transport.
1072 		 * IOSTAT_FCP_RSP_ERROR means:
1073 		 * 1. Length of data received doesn't match total
1074 		 *    transfer length in WQE
1075 		 * 2. If the RSP payload does NOT match these cases:
1076 		 *    a. RSP length 12/24 bytes and all zeros
1077 		 *    b. NVME ERSP
1078 		 */
1079 		switch (lpfc_ncmd->status) {
1080 		case IOSTAT_SUCCESS:
1081 			nCmd->transferred_length = wcqe->total_data_placed;
1082 			nCmd->rcv_rsplen = 0;
1083 			nCmd->status = 0;
1084 			break;
1085 		case IOSTAT_FCP_RSP_ERROR:
1086 			nCmd->transferred_length = wcqe->total_data_placed;
1087 			nCmd->rcv_rsplen = wcqe->parameter;
1088 			nCmd->status = 0;
1089 			/* Sanity check */
1090 			if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN)
1091 				break;
1092 			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
1093 					 "6081 NVME Completion Protocol Error: "
1094 					 "xri %x status x%x result x%x "
1095 					 "placed x%x\n",
1096 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1097 					 lpfc_ncmd->status, lpfc_ncmd->result,
1098 					 wcqe->total_data_placed);
1099 			break;
1100 		case IOSTAT_LOCAL_REJECT:
1101 			/* Let fall through to set command final state. */
1102 			if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1103 				lpfc_printf_vlog(vport, KERN_INFO,
1104 					 LOG_NVME_IOERR,
1105 					 "6032 Delay Aborted cmd x%px "
1106 					 "nvme cmd x%px, xri x%x, "
1107 					 "xb %d\n",
1108 					 lpfc_ncmd, nCmd,
1109 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1110 					 bf_get(lpfc_wcqe_c_xb, wcqe));
1111 			/* fall through */
1112 		default:
1113 out_err:
1114 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1115 					 "6072 NVME Completion Error: xri %x "
1116 					 "status x%x result x%x [x%x] "
1117 					 "placed x%x\n",
1118 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1119 					 lpfc_ncmd->status, lpfc_ncmd->result,
1120 					 wcqe->parameter,
1121 					 wcqe->total_data_placed);
1122 			nCmd->transferred_length = 0;
1123 			nCmd->rcv_rsplen = 0;
1124 			nCmd->status = NVME_SC_INTERNAL;
1125 		}
1126 	}
1127 
1128 	/* pick up SLI4 exhange busy condition */
1129 	if (bf_get(lpfc_wcqe_c_xb, wcqe))
1130 		lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1131 	else
1132 		lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1133 
1134 	/* Update stats and complete the IO.  There is
1135 	 * no need for dma unprep because the nvme_transport
1136 	 * owns the dma address.
1137 	 */
1138 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1139 	if (lpfc_ncmd->ts_cmd_start) {
1140 		lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1141 		lpfc_ncmd->ts_data_nvme = ktime_get_ns();
1142 		phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme;
1143 		lpfc_nvme_ktime(phba, lpfc_ncmd);
1144 	}
1145 	if (unlikely(phba->cpucheck_on & LPFC_CHECK_NVME_IO)) {
1146 		uint32_t cpu;
1147 		idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
1148 		cpu = raw_smp_processor_id();
1149 		if (cpu < LPFC_CHECK_CPU_CNT) {
1150 			if (lpfc_ncmd->cpu != cpu)
1151 				lpfc_printf_vlog(vport,
1152 						 KERN_INFO, LOG_NVME_IOERR,
1153 						 "6701 CPU Check cmpl: "
1154 						 "cpu %d expect %d\n",
1155 						 cpu, lpfc_ncmd->cpu);
1156 			phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++;
1157 		}
1158 	}
1159 #endif
1160 
1161 	/* NVME targets need completion held off until the abort exchange
1162 	 * completes unless the NVME Rport is getting unregistered.
1163 	 */
1164 
1165 	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1166 		freqpriv = nCmd->private;
1167 		freqpriv->nvme_buf = NULL;
1168 		lpfc_ncmd->nvmeCmd = NULL;
1169 		spin_unlock(&lpfc_ncmd->buf_lock);
1170 		nCmd->done(nCmd);
1171 	} else
1172 		spin_unlock(&lpfc_ncmd->buf_lock);
1173 
1174 	/* Call release with XB=1 to queue the IO into the abort list. */
1175 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1176 }
1177 
1178 
1179 /**
1180  * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1181  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1182  * @lpfc_nvme_lport: Pointer to the driver's local port data
1183  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1184  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1185  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1186  *
1187  * Driver registers this routine as it io request handler.  This
1188  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1189  * data structure to the rport indicated in @lpfc_nvme_rport.
1190  *
1191  * Return value :
1192  *   0 - Success
1193  *   TODO: What are the failure codes.
1194  **/
1195 static int
1196 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1197 		      struct lpfc_io_buf *lpfc_ncmd,
1198 		      struct lpfc_nodelist *pnode,
1199 		      struct lpfc_fc4_ctrl_stat *cstat)
1200 {
1201 	struct lpfc_hba *phba = vport->phba;
1202 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1203 	struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);
1204 	union lpfc_wqe128 *wqe = &pwqeq->wqe;
1205 	uint32_t req_len;
1206 
1207 	if (!NLP_CHK_NODE_ACT(pnode))
1208 		return -EINVAL;
1209 
1210 	/*
1211 	 * There are three possibilities here - use scatter-gather segment, use
1212 	 * the single mapping, or neither.
1213 	 */
1214 	if (nCmd->sg_cnt) {
1215 		if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1216 			/* From the iwrite template, initialize words 7 - 11 */
1217 			memcpy(&wqe->words[7],
1218 			       &lpfc_iwrite_cmd_template.words[7],
1219 			       sizeof(uint32_t) * 5);
1220 
1221 			/* Word 4 */
1222 			wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1223 
1224 			/* Word 5 */
1225 			if ((phba->cfg_nvme_enable_fb) &&
1226 			    (pnode->nlp_flag & NLP_FIRSTBURST)) {
1227 				req_len = lpfc_ncmd->nvmeCmd->payload_length;
1228 				if (req_len < pnode->nvme_fb_size)
1229 					wqe->fcp_iwrite.initial_xfer_len =
1230 						req_len;
1231 				else
1232 					wqe->fcp_iwrite.initial_xfer_len =
1233 						pnode->nvme_fb_size;
1234 			} else {
1235 				wqe->fcp_iwrite.initial_xfer_len = 0;
1236 			}
1237 			cstat->output_requests++;
1238 		} else {
1239 			/* From the iread template, initialize words 7 - 11 */
1240 			memcpy(&wqe->words[7],
1241 			       &lpfc_iread_cmd_template.words[7],
1242 			       sizeof(uint32_t) * 5);
1243 
1244 			/* Word 4 */
1245 			wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1246 
1247 			/* Word 5 */
1248 			wqe->fcp_iread.rsrvd5 = 0;
1249 
1250 			cstat->input_requests++;
1251 		}
1252 	} else {
1253 		/* From the icmnd template, initialize words 4 - 11 */
1254 		memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1255 		       sizeof(uint32_t) * 8);
1256 		cstat->control_requests++;
1257 	}
1258 
1259 	if (pnode->nlp_nvme_info & NLP_NVME_NSLER)
1260 		bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1261 	/*
1262 	 * Finish initializing those WQE fields that are independent
1263 	 * of the nvme_cmnd request_buffer
1264 	 */
1265 
1266 	/* Word 3 */
1267 	bf_set(payload_offset_len, &wqe->fcp_icmd,
1268 	       (nCmd->rsplen + nCmd->cmdlen));
1269 
1270 	/* Word 6 */
1271 	bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1272 	       phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1273 	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1274 
1275 	/* Word 8 */
1276 	wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1277 
1278 	/* Word 9 */
1279 	bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1280 
1281 	/* Words 13 14 15 are for PBDE support */
1282 
1283 	pwqeq->vport = vport;
1284 	return 0;
1285 }
1286 
1287 
1288 /**
1289  * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1290  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1291  * @lpfc_nvme_lport: Pointer to the driver's local port data
1292  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1293  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1294  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1295  *
1296  * Driver registers this routine as it io request handler.  This
1297  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1298  * data structure to the rport indicated in @lpfc_nvme_rport.
1299  *
1300  * Return value :
1301  *   0 - Success
1302  *   TODO: What are the failure codes.
1303  **/
1304 static int
1305 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1306 		      struct lpfc_io_buf *lpfc_ncmd)
1307 {
1308 	struct lpfc_hba *phba = vport->phba;
1309 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1310 	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1311 	struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1312 	struct sli4_hybrid_sgl *sgl_xtra = NULL;
1313 	struct scatterlist *data_sg;
1314 	struct sli4_sge *first_data_sgl;
1315 	struct ulp_bde64 *bde;
1316 	dma_addr_t physaddr = 0;
1317 	uint32_t num_bde = 0;
1318 	uint32_t dma_len = 0;
1319 	uint32_t dma_offset = 0;
1320 	int nseg, i, j;
1321 	bool lsp_just_set = false;
1322 
1323 	/* Fix up the command and response DMA stuff. */
1324 	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1325 
1326 	/*
1327 	 * There are three possibilities here - use scatter-gather segment, use
1328 	 * the single mapping, or neither.
1329 	 */
1330 	if (nCmd->sg_cnt) {
1331 		/*
1332 		 * Jump over the cmd and rsp SGEs.  The fix routine
1333 		 * has already adjusted for this.
1334 		 */
1335 		sgl += 2;
1336 
1337 		first_data_sgl = sgl;
1338 		lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1339 		if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1340 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1341 					"6058 Too many sg segments from "
1342 					"NVME Transport.  Max %d, "
1343 					"nvmeIO sg_cnt %d\n",
1344 					phba->cfg_nvme_seg_cnt + 1,
1345 					lpfc_ncmd->seg_cnt);
1346 			lpfc_ncmd->seg_cnt = 0;
1347 			return 1;
1348 		}
1349 
1350 		/*
1351 		 * The driver established a maximum scatter-gather segment count
1352 		 * during probe that limits the number of sg elements in any
1353 		 * single nvme command.  Just run through the seg_cnt and format
1354 		 * the sge's.
1355 		 */
1356 		nseg = nCmd->sg_cnt;
1357 		data_sg = nCmd->first_sgl;
1358 
1359 		/* for tracking the segment boundaries */
1360 		j = 2;
1361 		for (i = 0; i < nseg; i++) {
1362 			if (data_sg == NULL) {
1363 				lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1364 						"6059 dptr err %d, nseg %d\n",
1365 						i, nseg);
1366 				lpfc_ncmd->seg_cnt = 0;
1367 				return 1;
1368 			}
1369 
1370 			sgl->word2 = 0;
1371 			if ((num_bde + 1) == nseg) {
1372 				bf_set(lpfc_sli4_sge_last, sgl, 1);
1373 				bf_set(lpfc_sli4_sge_type, sgl,
1374 				       LPFC_SGE_TYPE_DATA);
1375 			} else {
1376 				bf_set(lpfc_sli4_sge_last, sgl, 0);
1377 
1378 				/* expand the segment */
1379 				if (!lsp_just_set &&
1380 				    !((j + 1) % phba->border_sge_num) &&
1381 				    ((nseg - 1) != i)) {
1382 					/* set LSP type */
1383 					bf_set(lpfc_sli4_sge_type, sgl,
1384 					       LPFC_SGE_TYPE_LSP);
1385 
1386 					sgl_xtra = lpfc_get_sgl_per_hdwq(
1387 							phba, lpfc_ncmd);
1388 
1389 					if (unlikely(!sgl_xtra)) {
1390 						lpfc_ncmd->seg_cnt = 0;
1391 						return 1;
1392 					}
1393 					sgl->addr_lo = cpu_to_le32(putPaddrLow(
1394 						       sgl_xtra->dma_phys_sgl));
1395 					sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1396 						       sgl_xtra->dma_phys_sgl));
1397 
1398 				} else {
1399 					bf_set(lpfc_sli4_sge_type, sgl,
1400 					       LPFC_SGE_TYPE_DATA);
1401 				}
1402 			}
1403 
1404 			if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1405 				     LPFC_SGE_TYPE_LSP)) {
1406 				if ((nseg - 1) == i)
1407 					bf_set(lpfc_sli4_sge_last, sgl, 1);
1408 
1409 				physaddr = data_sg->dma_address;
1410 				dma_len = data_sg->length;
1411 				sgl->addr_lo = cpu_to_le32(
1412 							 putPaddrLow(physaddr));
1413 				sgl->addr_hi = cpu_to_le32(
1414 							putPaddrHigh(physaddr));
1415 
1416 				bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1417 				sgl->word2 = cpu_to_le32(sgl->word2);
1418 				sgl->sge_len = cpu_to_le32(dma_len);
1419 
1420 				dma_offset += dma_len;
1421 				data_sg = sg_next(data_sg);
1422 
1423 				sgl++;
1424 
1425 				lsp_just_set = false;
1426 			} else {
1427 				sgl->word2 = cpu_to_le32(sgl->word2);
1428 
1429 				sgl->sge_len = cpu_to_le32(
1430 						     phba->cfg_sg_dma_buf_size);
1431 
1432 				sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1433 				i = i - 1;
1434 
1435 				lsp_just_set = true;
1436 			}
1437 
1438 			j++;
1439 		}
1440 		if (phba->cfg_enable_pbde) {
1441 			/* Use PBDE support for first SGL only, offset == 0 */
1442 			/* Words 13-15 */
1443 			bde = (struct ulp_bde64 *)
1444 				&wqe->words[13];
1445 			bde->addrLow = first_data_sgl->addr_lo;
1446 			bde->addrHigh = first_data_sgl->addr_hi;
1447 			bde->tus.f.bdeSize =
1448 				le32_to_cpu(first_data_sgl->sge_len);
1449 			bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1450 			bde->tus.w = cpu_to_le32(bde->tus.w);
1451 			/* wqe_pbde is 1 in template */
1452 		} else {
1453 			memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1454 			bf_set(wqe_pbde, &wqe->generic.wqe_com, 0);
1455 		}
1456 
1457 	} else {
1458 		lpfc_ncmd->seg_cnt = 0;
1459 
1460 		/* For this clause to be valid, the payload_length
1461 		 * and sg_cnt must zero.
1462 		 */
1463 		if (nCmd->payload_length != 0) {
1464 			lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
1465 					"6063 NVME DMA Prep Err: sg_cnt %d "
1466 					"payload_length x%x\n",
1467 					nCmd->sg_cnt, nCmd->payload_length);
1468 			return 1;
1469 		}
1470 	}
1471 	return 0;
1472 }
1473 
1474 /**
1475  * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1476  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1477  * @lpfc_nvme_lport: Pointer to the driver's local port data
1478  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1479  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1480  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1481  *
1482  * Driver registers this routine as it io request handler.  This
1483  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1484  * data structure to the rport
1485  indicated in @lpfc_nvme_rport.
1486  *
1487  * Return value :
1488  *   0 - Success
1489  *   TODO: What are the failure codes.
1490  **/
1491 static int
1492 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1493 			struct nvme_fc_remote_port *pnvme_rport,
1494 			void *hw_queue_handle,
1495 			struct nvmefc_fcp_req *pnvme_fcreq)
1496 {
1497 	int ret = 0;
1498 	int expedite = 0;
1499 	int idx, cpu;
1500 	struct lpfc_nvme_lport *lport;
1501 	struct lpfc_fc4_ctrl_stat *cstat;
1502 	struct lpfc_vport *vport;
1503 	struct lpfc_hba *phba;
1504 	struct lpfc_nodelist *ndlp;
1505 	struct lpfc_io_buf *lpfc_ncmd;
1506 	struct lpfc_nvme_rport *rport;
1507 	struct lpfc_nvme_qhandle *lpfc_queue_info;
1508 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1509 	struct nvme_common_command *sqe;
1510 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1511 	uint64_t start = 0;
1512 #endif
1513 
1514 	/* Validate pointers. LLDD fault handling with transport does
1515 	 * have timing races.
1516 	 */
1517 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1518 	if (unlikely(!lport)) {
1519 		ret = -EINVAL;
1520 		goto out_fail;
1521 	}
1522 
1523 	vport = lport->vport;
1524 
1525 	if (unlikely(!hw_queue_handle)) {
1526 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1527 				 "6117 Fail IO, NULL hw_queue_handle\n");
1528 		atomic_inc(&lport->xmt_fcp_err);
1529 		ret = -EBUSY;
1530 		goto out_fail;
1531 	}
1532 
1533 	phba = vport->phba;
1534 
1535 	if (vport->load_flag & FC_UNLOADING) {
1536 		ret = -ENODEV;
1537 		goto out_fail;
1538 	}
1539 
1540 	if (unlikely(vport->load_flag & FC_UNLOADING)) {
1541 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1542 				 "6124 Fail IO, Driver unload\n");
1543 		atomic_inc(&lport->xmt_fcp_err);
1544 		ret = -ENODEV;
1545 		goto out_fail;
1546 	}
1547 
1548 	freqpriv = pnvme_fcreq->private;
1549 	if (unlikely(!freqpriv)) {
1550 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1551 				 "6158 Fail IO, NULL request data\n");
1552 		atomic_inc(&lport->xmt_fcp_err);
1553 		ret = -EINVAL;
1554 		goto out_fail;
1555 	}
1556 
1557 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1558 	if (phba->ktime_on)
1559 		start = ktime_get_ns();
1560 #endif
1561 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1562 	lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1563 
1564 	/*
1565 	 * Catch race where our node has transitioned, but the
1566 	 * transport is still transitioning.
1567 	 */
1568 	ndlp = rport->ndlp;
1569 	if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) {
1570 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1571 				 "6053 Busy IO, ndlp not ready: rport x%px "
1572 				  "ndlp x%px, DID x%06x\n",
1573 				 rport, ndlp, pnvme_rport->port_id);
1574 		atomic_inc(&lport->xmt_fcp_err);
1575 		ret = -EBUSY;
1576 		goto out_fail;
1577 	}
1578 
1579 	/* The remote node has to be a mapped target or it's an error. */
1580 	if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1581 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1582 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1583 				 "6036 Fail IO, DID x%06x not ready for "
1584 				 "IO. State x%x, Type x%x Flg x%x\n",
1585 				 pnvme_rport->port_id,
1586 				 ndlp->nlp_state, ndlp->nlp_type,
1587 				 ndlp->upcall_flags);
1588 		atomic_inc(&lport->xmt_fcp_bad_ndlp);
1589 		ret = -EBUSY;
1590 		goto out_fail;
1591 
1592 	}
1593 
1594 	/* Currently only NVME Keep alive commands should be expedited
1595 	 * if the driver runs out of a resource. These should only be
1596 	 * issued on the admin queue, qidx 0
1597 	 */
1598 	if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1599 		sqe = &((struct nvme_fc_cmd_iu *)
1600 			pnvme_fcreq->cmdaddr)->sqe.common;
1601 		if (sqe->opcode == nvme_admin_keep_alive)
1602 			expedite = 1;
1603 	}
1604 
1605 	/* The node is shared with FCP IO, make sure the IO pending count does
1606 	 * not exceed the programmed depth.
1607 	 */
1608 	if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1609 		if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1610 		    !expedite) {
1611 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1612 					 "6174 Fail IO, ndlp qdepth exceeded: "
1613 					 "idx %d DID %x pend %d qdepth %d\n",
1614 					 lpfc_queue_info->index, ndlp->nlp_DID,
1615 					 atomic_read(&ndlp->cmd_pending),
1616 					 ndlp->cmd_qdepth);
1617 			atomic_inc(&lport->xmt_fcp_qdepth);
1618 			ret = -EBUSY;
1619 			goto out_fail;
1620 		}
1621 	}
1622 
1623 	/* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1624 	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1625 		idx = lpfc_queue_info->index;
1626 	} else {
1627 		cpu = raw_smp_processor_id();
1628 		idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1629 	}
1630 
1631 	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1632 	if (lpfc_ncmd == NULL) {
1633 		atomic_inc(&lport->xmt_fcp_noxri);
1634 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1635 				 "6065 Fail IO, driver buffer pool is empty: "
1636 				 "idx %d DID %x\n",
1637 				 lpfc_queue_info->index, ndlp->nlp_DID);
1638 		ret = -EBUSY;
1639 		goto out_fail;
1640 	}
1641 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1642 	if (start) {
1643 		lpfc_ncmd->ts_cmd_start = start;
1644 		lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1645 	} else {
1646 		lpfc_ncmd->ts_cmd_start = 0;
1647 	}
1648 #endif
1649 
1650 	/*
1651 	 * Store the data needed by the driver to issue, abort, and complete
1652 	 * an IO.
1653 	 * Do not let the IO hang out forever.  There is no midlayer issuing
1654 	 * an abort so inform the FW of the maximum IO pending time.
1655 	 */
1656 	freqpriv->nvme_buf = lpfc_ncmd;
1657 	lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1658 	lpfc_ncmd->ndlp = ndlp;
1659 	lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1660 
1661 	/*
1662 	 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1663 	 * This identfier was create in our hardware queue create callback
1664 	 * routine. The driver now is dependent on the IO queue steering from
1665 	 * the transport.  We are trusting the upper NVME layers know which
1666 	 * index to use and that they have affinitized a CPU to this hardware
1667 	 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1668 	 */
1669 	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1670 	cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1671 
1672 	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1673 	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1674 	if (ret) {
1675 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1676 				 "6175 Fail IO, Prep DMA: "
1677 				 "idx %d DID %x\n",
1678 				 lpfc_queue_info->index, ndlp->nlp_DID);
1679 		atomic_inc(&lport->xmt_fcp_err);
1680 		ret = -ENOMEM;
1681 		goto out_free_nvme_buf;
1682 	}
1683 
1684 	lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1685 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
1686 			 lpfc_queue_info->index, ndlp->nlp_DID);
1687 
1688 	ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1689 	if (ret) {
1690 		atomic_inc(&lport->xmt_fcp_wqerr);
1691 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1692 				 "6113 Fail IO, Could not issue WQE err %x "
1693 				 "sid: x%x did: x%x oxid: x%x\n",
1694 				 ret, vport->fc_myDID, ndlp->nlp_DID,
1695 				 lpfc_ncmd->cur_iocbq.sli4_xritag);
1696 		goto out_free_nvme_buf;
1697 	}
1698 
1699 	if (phba->cfg_xri_rebalancing)
1700 		lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1701 
1702 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1703 	if (lpfc_ncmd->ts_cmd_start)
1704 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1705 
1706 	if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) {
1707 		cpu = raw_smp_processor_id();
1708 		if (cpu < LPFC_CHECK_CPU_CNT) {
1709 			lpfc_ncmd->cpu = cpu;
1710 			if (idx != cpu)
1711 				lpfc_printf_vlog(vport,
1712 						 KERN_INFO, LOG_NVME_IOERR,
1713 						"6702 CPU Check cmd: "
1714 						"cpu %d wq %d\n",
1715 						lpfc_ncmd->cpu,
1716 						lpfc_queue_info->index);
1717 			phba->sli4_hba.hdwq[idx].cpucheck_xmt_io[cpu]++;
1718 		}
1719 	}
1720 #endif
1721 	return 0;
1722 
1723  out_free_nvme_buf:
1724 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1725 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1726 			cstat->output_requests--;
1727 		else
1728 			cstat->input_requests--;
1729 	} else
1730 		cstat->control_requests--;
1731 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1732  out_fail:
1733 	return ret;
1734 }
1735 
1736 /**
1737  * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1738  * @phba: Pointer to HBA context object
1739  * @cmdiocb: Pointer to command iocb object.
1740  * @rspiocb: Pointer to response iocb object.
1741  *
1742  * This is the callback function for any NVME FCP IO that was aborted.
1743  *
1744  * Return value:
1745  *   None
1746  **/
1747 void
1748 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1749 			   struct lpfc_wcqe_complete *abts_cmpl)
1750 {
1751 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1752 			"6145 ABORT_XRI_CN completing on rpi x%x "
1753 			"original iotag x%x, abort cmd iotag x%x "
1754 			"req_tag x%x, status x%x, hwstatus x%x\n",
1755 			cmdiocb->iocb.un.acxri.abortContextTag,
1756 			cmdiocb->iocb.un.acxri.abortIoTag,
1757 			cmdiocb->iotag,
1758 			bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1759 			bf_get(lpfc_wcqe_c_status, abts_cmpl),
1760 			bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1761 	lpfc_sli_release_iocbq(phba, cmdiocb);
1762 }
1763 
1764 /**
1765  * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1766  * @lpfc_pnvme: Pointer to the driver's nvme instance data
1767  * @lpfc_nvme_lport: Pointer to the driver's local port data
1768  * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1769  * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1770  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1771  *
1772  * Driver registers this routine as its nvme request io abort handler.  This
1773  * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1774  * data structure to the rport indicated in @lpfc_nvme_rport.  This routine
1775  * is executed asynchronously - one the target is validated as "MAPPED" and
1776  * ready for IO, the driver issues the abort request and returns.
1777  *
1778  * Return value:
1779  *   None
1780  **/
1781 static void
1782 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1783 		    struct nvme_fc_remote_port *pnvme_rport,
1784 		    void *hw_queue_handle,
1785 		    struct nvmefc_fcp_req *pnvme_fcreq)
1786 {
1787 	struct lpfc_nvme_lport *lport;
1788 	struct lpfc_vport *vport;
1789 	struct lpfc_hba *phba;
1790 	struct lpfc_io_buf *lpfc_nbuf;
1791 	struct lpfc_iocbq *abts_buf;
1792 	struct lpfc_iocbq *nvmereq_wqe;
1793 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1794 	union lpfc_wqe128 *abts_wqe;
1795 	unsigned long flags;
1796 	int ret_val;
1797 
1798 	/* Validate pointers. LLDD fault handling with transport does
1799 	 * have timing races.
1800 	 */
1801 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1802 	if (unlikely(!lport))
1803 		return;
1804 
1805 	vport = lport->vport;
1806 
1807 	if (unlikely(!hw_queue_handle)) {
1808 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1809 				 "6129 Fail Abort, HW Queue Handle NULL.\n");
1810 		return;
1811 	}
1812 
1813 	phba = vport->phba;
1814 	freqpriv = pnvme_fcreq->private;
1815 
1816 	if (unlikely(!freqpriv))
1817 		return;
1818 	if (vport->load_flag & FC_UNLOADING)
1819 		return;
1820 
1821 	/* Announce entry to new IO submit field. */
1822 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1823 			 "6002 Abort Request to rport DID x%06x "
1824 			 "for nvme_fc_req x%px\n",
1825 			 pnvme_rport->port_id,
1826 			 pnvme_fcreq);
1827 
1828 	/* If the hba is getting reset, this flag is set.  It is
1829 	 * cleared when the reset is complete and rings reestablished.
1830 	 */
1831 	spin_lock_irqsave(&phba->hbalock, flags);
1832 	/* driver queued commands are in process of being flushed */
1833 	if (phba->hba_flag & HBA_IOQ_FLUSH) {
1834 		spin_unlock_irqrestore(&phba->hbalock, flags);
1835 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1836 				 "6139 Driver in reset cleanup - flushing "
1837 				 "NVME Req now.  hba_flag x%x\n",
1838 				 phba->hba_flag);
1839 		return;
1840 	}
1841 
1842 	lpfc_nbuf = freqpriv->nvme_buf;
1843 	if (!lpfc_nbuf) {
1844 		spin_unlock_irqrestore(&phba->hbalock, flags);
1845 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1846 				 "6140 NVME IO req has no matching lpfc nvme "
1847 				 "io buffer.  Skipping abort req.\n");
1848 		return;
1849 	} else if (!lpfc_nbuf->nvmeCmd) {
1850 		spin_unlock_irqrestore(&phba->hbalock, flags);
1851 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1852 				 "6141 lpfc NVME IO req has no nvme_fcreq "
1853 				 "io buffer.  Skipping abort req.\n");
1854 		return;
1855 	}
1856 	nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1857 
1858 	/* Guard against IO completion being called at same time */
1859 	spin_lock(&lpfc_nbuf->buf_lock);
1860 
1861 	/*
1862 	 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1863 	 * state must match the nvme_fcreq passed by the nvme
1864 	 * transport.  If they don't match, it is likely the driver
1865 	 * has already completed the NVME IO and the nvme transport
1866 	 * has not seen it yet.
1867 	 */
1868 	if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1869 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1870 				 "6143 NVME req mismatch: "
1871 				 "lpfc_nbuf x%px nvmeCmd x%px, "
1872 				 "pnvme_fcreq x%px.  Skipping Abort xri x%x\n",
1873 				 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1874 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1875 		goto out_unlock;
1876 	}
1877 
1878 	/* Don't abort IOs no longer on the pending queue. */
1879 	if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) {
1880 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1881 				 "6142 NVME IO req x%px not queued - skipping "
1882 				 "abort req xri x%x\n",
1883 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1884 		goto out_unlock;
1885 	}
1886 
1887 	atomic_inc(&lport->xmt_fcp_abort);
1888 	lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1889 			 nvmereq_wqe->sli4_xritag,
1890 			 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1891 
1892 	/* Outstanding abort is in progress */
1893 	if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) {
1894 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1895 				 "6144 Outstanding NVME I/O Abort Request "
1896 				 "still pending on nvme_fcreq x%px, "
1897 				 "lpfc_ncmd %px xri x%x\n",
1898 				 pnvme_fcreq, lpfc_nbuf,
1899 				 nvmereq_wqe->sli4_xritag);
1900 		goto out_unlock;
1901 	}
1902 
1903 	abts_buf = __lpfc_sli_get_iocbq(phba);
1904 	if (!abts_buf) {
1905 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1906 				 "6136 No available abort wqes. Skipping "
1907 				 "Abts req for nvme_fcreq x%px xri x%x\n",
1908 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1909 		goto out_unlock;
1910 	}
1911 
1912 	/* Ready - mark outstanding as aborted by driver. */
1913 	nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED;
1914 
1915 	/* Complete prepping the abort wqe and issue to the FW. */
1916 	abts_wqe = &abts_buf->wqe;
1917 
1918 	/* WQEs are reused.  Clear stale data and set key fields to
1919 	 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
1920 	 */
1921 	memset(abts_wqe, 0, sizeof(*abts_wqe));
1922 	bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG);
1923 
1924 	/* word 7 */
1925 	bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX);
1926 	bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com,
1927 	       nvmereq_wqe->iocb.ulpClass);
1928 
1929 	/* word 8 - tell the FW to abort the IO associated with this
1930 	 * outstanding exchange ID.
1931 	 */
1932 	abts_wqe->abort_cmd.wqe_com.abort_tag = nvmereq_wqe->sli4_xritag;
1933 
1934 	/* word 9 - this is the iotag for the abts_wqe completion. */
1935 	bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com,
1936 	       abts_buf->iotag);
1937 
1938 	/* word 10 */
1939 	bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1);
1940 	bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE);
1941 
1942 	/* word 11 */
1943 	bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND);
1944 	bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1);
1945 	bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
1946 
1947 	/* ABTS WQE must go to the same WQ as the WQE to be aborted */
1948 	abts_buf->iocb_flag |= LPFC_IO_NVME;
1949 	abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx;
1950 	abts_buf->vport = vport;
1951 	abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl;
1952 	ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf);
1953 	spin_unlock(&lpfc_nbuf->buf_lock);
1954 	spin_unlock_irqrestore(&phba->hbalock, flags);
1955 	if (ret_val) {
1956 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS,
1957 				 "6137 Failed abts issue_wqe with status x%x "
1958 				 "for nvme_fcreq x%px.\n",
1959 				 ret_val, pnvme_fcreq);
1960 		lpfc_sli_release_iocbq(phba, abts_buf);
1961 		return;
1962 	}
1963 
1964 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1965 			 "6138 Transport Abort NVME Request Issued for "
1966 			 "ox_id x%x on reqtag x%x\n",
1967 			 nvmereq_wqe->sli4_xritag,
1968 			 abts_buf->iotag);
1969 	return;
1970 
1971 out_unlock:
1972 	spin_unlock(&lpfc_nbuf->buf_lock);
1973 	spin_unlock_irqrestore(&phba->hbalock, flags);
1974 	return;
1975 }
1976 
1977 /* Declare and initialization an instance of the FC NVME template. */
1978 static struct nvme_fc_port_template lpfc_nvme_template = {
1979 	/* initiator-based functions */
1980 	.localport_delete  = lpfc_nvme_localport_delete,
1981 	.remoteport_delete = lpfc_nvme_remoteport_delete,
1982 	.create_queue = lpfc_nvme_create_queue,
1983 	.delete_queue = lpfc_nvme_delete_queue,
1984 	.ls_req       = lpfc_nvme_ls_req,
1985 	.fcp_io       = lpfc_nvme_fcp_io_submit,
1986 	.ls_abort     = lpfc_nvme_ls_abort,
1987 	.fcp_abort    = lpfc_nvme_fcp_abort,
1988 
1989 	.max_hw_queues = 1,
1990 	.max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
1991 	.max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
1992 	.dma_boundary = 0xFFFFFFFF,
1993 
1994 	/* Sizes of additional private data for data structures.
1995 	 * No use for the last two sizes at this time.
1996 	 */
1997 	.local_priv_sz = sizeof(struct lpfc_nvme_lport),
1998 	.remote_priv_sz = sizeof(struct lpfc_nvme_rport),
1999 	.lsrqst_priv_sz = 0,
2000 	.fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2001 };
2002 
2003 /**
2004  * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2005  * @phba: The HBA for which this call is being executed.
2006  *
2007  * This routine removes a nvme buffer from head of @hdwq io_buf_list
2008  * and returns to caller.
2009  *
2010  * Return codes:
2011  *   NULL - Error
2012  *   Pointer to lpfc_nvme_buf - Success
2013  **/
2014 static struct lpfc_io_buf *
2015 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2016 		  int idx, int expedite)
2017 {
2018 	struct lpfc_io_buf *lpfc_ncmd;
2019 	struct lpfc_sli4_hdw_queue *qp;
2020 	struct sli4_sge *sgl;
2021 	struct lpfc_iocbq *pwqeq;
2022 	union lpfc_wqe128 *wqe;
2023 
2024 	lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2025 
2026 	if (lpfc_ncmd) {
2027 		pwqeq = &(lpfc_ncmd->cur_iocbq);
2028 		wqe = &pwqeq->wqe;
2029 
2030 		/* Setup key fields in buffer that may have been changed
2031 		 * if other protocols used this buffer.
2032 		 */
2033 		pwqeq->iocb_flag = LPFC_IO_NVME;
2034 		pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl;
2035 		lpfc_ncmd->start_time = jiffies;
2036 		lpfc_ncmd->flags = 0;
2037 
2038 		/* Rsp SGE will be filled in when we rcv an IO
2039 		 * from the NVME Layer to be sent.
2040 		 * The cmd is going to be embedded so we need a SKIP SGE.
2041 		 */
2042 		sgl = lpfc_ncmd->dma_sgl;
2043 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2044 		bf_set(lpfc_sli4_sge_last, sgl, 0);
2045 		sgl->word2 = cpu_to_le32(sgl->word2);
2046 		/* Fill in word 3 / sgl_len during cmd submission */
2047 
2048 		/* Initialize 64 bytes only */
2049 		memset(wqe, 0, sizeof(union lpfc_wqe));
2050 
2051 		if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2052 			atomic_inc(&ndlp->cmd_pending);
2053 			lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2054 		}
2055 
2056 	} else {
2057 		qp = &phba->sli4_hba.hdwq[idx];
2058 		qp->empty_io_bufs++;
2059 	}
2060 
2061 	return  lpfc_ncmd;
2062 }
2063 
2064 /**
2065  * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2066  * @phba: The Hba for which this call is being executed.
2067  * @lpfc_ncmd: The nvme buffer which is being released.
2068  *
2069  * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2070  * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2071  * and cannot be reused for at least RA_TOV amount of time if it was
2072  * aborted.
2073  **/
2074 static void
2075 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2076 {
2077 	struct lpfc_sli4_hdw_queue *qp;
2078 	unsigned long iflag = 0;
2079 
2080 	if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2081 		atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2082 
2083 	lpfc_ncmd->ndlp = NULL;
2084 	lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2085 
2086 	qp = lpfc_ncmd->hdwq;
2087 	if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) {
2088 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2089 				"6310 XB release deferred for "
2090 				"ox_id x%x on reqtag x%x\n",
2091 				lpfc_ncmd->cur_iocbq.sli4_xritag,
2092 				lpfc_ncmd->cur_iocbq.iotag);
2093 
2094 		spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2095 		list_add_tail(&lpfc_ncmd->list,
2096 			&qp->lpfc_abts_io_buf_list);
2097 		qp->abts_nvme_io_bufs++;
2098 		spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2099 	} else
2100 		lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2101 }
2102 
2103 /**
2104  * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2105  * @pvport - the lpfc_vport instance requesting a localport.
2106  *
2107  * This routine is invoked to create an nvme localport instance to bind
2108  * to the nvme_fc_transport.  It is called once during driver load
2109  * like lpfc_create_shost after all other services are initialized.
2110  * It requires a vport, vpi, and wwns at call time.  Other localport
2111  * parameters are modified as the driver's FCID and the Fabric WWN
2112  * are established.
2113  *
2114  * Return codes
2115  *      0 - successful
2116  *      -ENOMEM - no heap memory available
2117  *      other values - from nvme registration upcall
2118  **/
2119 int
2120 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2121 {
2122 	int ret = 0;
2123 	struct lpfc_hba  *phba = vport->phba;
2124 	struct nvme_fc_port_info nfcp_info;
2125 	struct nvme_fc_local_port *localport;
2126 	struct lpfc_nvme_lport *lport;
2127 
2128 	/* Initialize this localport instance.  The vport wwn usage ensures
2129 	 * that NPIV is accounted for.
2130 	 */
2131 	memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2132 	nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2133 	nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2134 	nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2135 
2136 	/* We need to tell the transport layer + 1 because it takes page
2137 	 * alignment into account. When space for the SGL is allocated we
2138 	 * allocate + 3, one for cmd, one for rsp and one for this alignment
2139 	 */
2140 	lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2141 
2142 	/* Advertise how many hw queues we support based on fcp_io_sched */
2143 	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ)
2144 		lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2145 	else
2146 		lpfc_nvme_template.max_hw_queues =
2147 			phba->sli4_hba.num_present_cpu;
2148 
2149 	if (!IS_ENABLED(CONFIG_NVME_FC))
2150 		return ret;
2151 
2152 	/* localport is allocated from the stack, but the registration
2153 	 * call allocates heap memory as well as the private area.
2154 	 */
2155 
2156 	ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2157 					 &vport->phba->pcidev->dev, &localport);
2158 	if (!ret) {
2159 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2160 				 "6005 Successfully registered local "
2161 				 "NVME port num %d, localP x%px, private "
2162 				 "x%px, sg_seg %d\n",
2163 				 localport->port_num, localport,
2164 				 localport->private,
2165 				 lpfc_nvme_template.max_sgl_segments);
2166 
2167 		/* Private is our lport size declared in the template. */
2168 		lport = (struct lpfc_nvme_lport *)localport->private;
2169 		vport->localport = localport;
2170 		lport->vport = vport;
2171 		vport->nvmei_support = 1;
2172 
2173 		atomic_set(&lport->xmt_fcp_noxri, 0);
2174 		atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2175 		atomic_set(&lport->xmt_fcp_qdepth, 0);
2176 		atomic_set(&lport->xmt_fcp_err, 0);
2177 		atomic_set(&lport->xmt_fcp_wqerr, 0);
2178 		atomic_set(&lport->xmt_fcp_abort, 0);
2179 		atomic_set(&lport->xmt_ls_abort, 0);
2180 		atomic_set(&lport->xmt_ls_err, 0);
2181 		atomic_set(&lport->cmpl_fcp_xb, 0);
2182 		atomic_set(&lport->cmpl_fcp_err, 0);
2183 		atomic_set(&lport->cmpl_ls_xb, 0);
2184 		atomic_set(&lport->cmpl_ls_err, 0);
2185 		atomic_set(&lport->fc4NvmeLsRequests, 0);
2186 		atomic_set(&lport->fc4NvmeLsCmpls, 0);
2187 	}
2188 
2189 	return ret;
2190 }
2191 
2192 #if (IS_ENABLED(CONFIG_NVME_FC))
2193 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2194  *
2195  * The driver has to wait for the host nvme transport to callback
2196  * indicating the localport has successfully unregistered all
2197  * resources.  Since this is an uninterruptible wait, loop every ten
2198  * seconds and print a message indicating no progress.
2199  *
2200  * An uninterruptible wait is used because of the risk of transport-to-
2201  * driver state mismatch.
2202  */
2203 static void
2204 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2205 			   struct lpfc_nvme_lport *lport,
2206 			   struct completion *lport_unreg_cmp)
2207 {
2208 	u32 wait_tmo;
2209 	int ret, i, pending = 0;
2210 	struct lpfc_sli_ring  *pring;
2211 	struct lpfc_hba  *phba = vport->phba;
2212 
2213 	/* Host transport has to clean up and confirm requiring an indefinite
2214 	 * wait. Print a message if a 10 second wait expires and renew the
2215 	 * wait. This is unexpected.
2216 	 */
2217 	wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2218 	while (true) {
2219 		ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2220 		if (unlikely(!ret)) {
2221 			pending = 0;
2222 			for (i = 0; i < phba->cfg_hdw_queue; i++) {
2223 				pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2224 				if (!pring)
2225 					continue;
2226 				if (pring->txcmplq_cnt)
2227 					pending += pring->txcmplq_cnt;
2228 			}
2229 			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR,
2230 					 "6176 Lport x%px Localport x%px wait "
2231 					 "timed out. Pending %d. Renewing.\n",
2232 					 lport, vport->localport, pending);
2233 			continue;
2234 		}
2235 		break;
2236 	}
2237 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2238 			 "6177 Lport x%px Localport x%px Complete Success\n",
2239 			 lport, vport->localport);
2240 }
2241 #endif
2242 
2243 /**
2244  * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2245  * @pnvme: pointer to lpfc nvme data structure.
2246  *
2247  * This routine is invoked to destroy all lports bound to the phba.
2248  * The lport memory was allocated by the nvme fc transport and is
2249  * released there.  This routine ensures all rports bound to the
2250  * lport have been disconnected.
2251  *
2252  **/
2253 void
2254 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2255 {
2256 #if (IS_ENABLED(CONFIG_NVME_FC))
2257 	struct nvme_fc_local_port *localport;
2258 	struct lpfc_nvme_lport *lport;
2259 	int ret;
2260 	DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2261 
2262 	if (vport->nvmei_support == 0)
2263 		return;
2264 
2265 	localport = vport->localport;
2266 	lport = (struct lpfc_nvme_lport *)localport->private;
2267 
2268 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2269 			 "6011 Destroying NVME localport x%px\n",
2270 			 localport);
2271 
2272 	/* lport's rport list is clear.  Unregister
2273 	 * lport and release resources.
2274 	 */
2275 	lport->lport_unreg_cmp = &lport_unreg_cmp;
2276 	ret = nvme_fc_unregister_localport(localport);
2277 
2278 	/* Wait for completion.  This either blocks
2279 	 * indefinitely or succeeds
2280 	 */
2281 	lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2282 	vport->localport = NULL;
2283 
2284 	/* Regardless of the unregister upcall response, clear
2285 	 * nvmei_support.  All rports are unregistered and the
2286 	 * driver will clean up.
2287 	 */
2288 	vport->nvmei_support = 0;
2289 	if (ret == 0) {
2290 		lpfc_printf_vlog(vport,
2291 				 KERN_INFO, LOG_NVME_DISC,
2292 				 "6009 Unregistered lport Success\n");
2293 	} else {
2294 		lpfc_printf_vlog(vport,
2295 				 KERN_INFO, LOG_NVME_DISC,
2296 				 "6010 Unregistered lport "
2297 				 "Failed, status x%x\n",
2298 				 ret);
2299 	}
2300 #endif
2301 }
2302 
2303 void
2304 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2305 {
2306 #if (IS_ENABLED(CONFIG_NVME_FC))
2307 	struct nvme_fc_local_port *localport;
2308 	struct lpfc_nvme_lport *lport;
2309 
2310 	localport = vport->localport;
2311 	if (!localport) {
2312 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2313 				 "6710 Update NVME fail. No localport\n");
2314 		return;
2315 	}
2316 	lport = (struct lpfc_nvme_lport *)localport->private;
2317 	if (!lport) {
2318 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2319 				 "6171 Update NVME fail. localP x%px, No lport\n",
2320 				 localport);
2321 		return;
2322 	}
2323 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2324 			 "6012 Update NVME lport x%px did x%x\n",
2325 			 localport, vport->fc_myDID);
2326 
2327 	localport->port_id = vport->fc_myDID;
2328 	if (localport->port_id == 0)
2329 		localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2330 	else
2331 		localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2332 
2333 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2334 			 "6030 bound lport x%px to DID x%06x\n",
2335 			 lport, localport->port_id);
2336 #endif
2337 }
2338 
2339 int
2340 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2341 {
2342 #if (IS_ENABLED(CONFIG_NVME_FC))
2343 	int ret = 0;
2344 	struct nvme_fc_local_port *localport;
2345 	struct lpfc_nvme_lport *lport;
2346 	struct lpfc_nvme_rport *rport;
2347 	struct lpfc_nvme_rport *oldrport;
2348 	struct nvme_fc_remote_port *remote_port;
2349 	struct nvme_fc_port_info rpinfo;
2350 	struct lpfc_nodelist *prev_ndlp = NULL;
2351 
2352 	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2353 			 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2354 			 ndlp->nlp_DID, ndlp->nlp_type);
2355 
2356 	localport = vport->localport;
2357 	if (!localport)
2358 		return 0;
2359 
2360 	lport = (struct lpfc_nvme_lport *)localport->private;
2361 
2362 	/* NVME rports are not preserved across devloss.
2363 	 * Just register this instance.  Note, rpinfo->dev_loss_tmo
2364 	 * is left 0 to indicate accept transport defaults.  The
2365 	 * driver communicates port role capabilities consistent
2366 	 * with the PRLI response data.
2367 	 */
2368 	memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2369 	rpinfo.port_id = ndlp->nlp_DID;
2370 	if (ndlp->nlp_type & NLP_NVME_TARGET)
2371 		rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2372 	if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2373 		rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2374 
2375 	if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2376 		rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2377 
2378 	rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2379 	rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2380 
2381 	spin_lock_irq(&vport->phba->hbalock);
2382 	oldrport = lpfc_ndlp_get_nrport(ndlp);
2383 	if (oldrport) {
2384 		prev_ndlp = oldrport->ndlp;
2385 		spin_unlock_irq(&vport->phba->hbalock);
2386 	} else {
2387 		spin_unlock_irq(&vport->phba->hbalock);
2388 		lpfc_nlp_get(ndlp);
2389 	}
2390 
2391 	ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2392 	if (!ret) {
2393 		/* If the ndlp already has an nrport, this is just
2394 		 * a resume of the existing rport.  Else this is a
2395 		 * new rport.
2396 		 */
2397 		/* Guard against an unregister/reregister
2398 		 * race that leaves the WAIT flag set.
2399 		 */
2400 		spin_lock_irq(&vport->phba->hbalock);
2401 		ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2402 		spin_unlock_irq(&vport->phba->hbalock);
2403 		rport = remote_port->private;
2404 		if (oldrport) {
2405 			/* New remoteport record does not guarantee valid
2406 			 * host private memory area.
2407 			 */
2408 			if (oldrport == remote_port->private) {
2409 				/* Same remoteport - ndlp should match.
2410 				 * Just reuse.
2411 				 */
2412 				lpfc_printf_vlog(ndlp->vport, KERN_INFO,
2413 						 LOG_NVME_DISC,
2414 						 "6014 Rebind lport to current "
2415 						 "remoteport x%px wwpn 0x%llx, "
2416 						 "Data: x%x x%x x%px x%px x%x "
2417 						 " x%06x\n",
2418 						 remote_port,
2419 						 remote_port->port_name,
2420 						 remote_port->port_id,
2421 						 remote_port->port_role,
2422 						 oldrport->ndlp,
2423 						 ndlp,
2424 						 ndlp->nlp_type,
2425 						 ndlp->nlp_DID);
2426 
2427 				/* It's a complete rebind only if the driver
2428 				 * is registering with the same ndlp. Otherwise
2429 				 * the driver likely executed a node swap
2430 				 * prior to this registration and the ndlp to
2431 				 * remoteport binding needs to be redone.
2432 				 */
2433 				if (prev_ndlp == ndlp)
2434 					return 0;
2435 
2436 			}
2437 
2438 			/* Sever the ndlp<->rport association
2439 			 * before dropping the ndlp ref from
2440 			 * register.
2441 			 */
2442 			spin_lock_irq(&vport->phba->hbalock);
2443 			ndlp->nrport = NULL;
2444 			ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG;
2445 			spin_unlock_irq(&vport->phba->hbalock);
2446 			rport->ndlp = NULL;
2447 			rport->remoteport = NULL;
2448 
2449 			/* Reference only removed if previous NDLP is no longer
2450 			 * active. It might be just a swap and removing the
2451 			 * reference would cause a premature cleanup.
2452 			 */
2453 			if (prev_ndlp && prev_ndlp != ndlp) {
2454 				if ((!NLP_CHK_NODE_ACT(prev_ndlp)) ||
2455 				    (!prev_ndlp->nrport))
2456 					lpfc_nlp_put(prev_ndlp);
2457 			}
2458 		}
2459 
2460 		/* Clean bind the rport to the ndlp. */
2461 		rport->remoteport = remote_port;
2462 		rport->lport = lport;
2463 		rport->ndlp = ndlp;
2464 		spin_lock_irq(&vport->phba->hbalock);
2465 		ndlp->nrport = rport;
2466 		spin_unlock_irq(&vport->phba->hbalock);
2467 		lpfc_printf_vlog(vport, KERN_INFO,
2468 				 LOG_NVME_DISC | LOG_NODE,
2469 				 "6022 Bind lport x%px to remoteport x%px "
2470 				 "rport x%px WWNN 0x%llx, "
2471 				 "Rport WWPN 0x%llx DID "
2472 				 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2473 				 lport, remote_port, rport,
2474 				 rpinfo.node_name, rpinfo.port_name,
2475 				 rpinfo.port_id, rpinfo.port_role,
2476 				 ndlp, prev_ndlp);
2477 	} else {
2478 		lpfc_printf_vlog(vport, KERN_ERR,
2479 				 LOG_NVME_DISC | LOG_NODE,
2480 				 "6031 RemotePort Registration failed "
2481 				 "err: %d, DID x%06x\n",
2482 				 ret, ndlp->nlp_DID);
2483 	}
2484 
2485 	return ret;
2486 #else
2487 	return 0;
2488 #endif
2489 }
2490 
2491 /**
2492  * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2493  *
2494  * If the ndlp represents an NVME Target, that we are logged into,
2495  * ping the NVME FC Transport layer to initiate a device rescan
2496  * on this remote NPort.
2497  */
2498 void
2499 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2500 {
2501 #if (IS_ENABLED(CONFIG_NVME_FC))
2502 	struct lpfc_nvme_rport *nrport;
2503 	struct nvme_fc_remote_port *remoteport = NULL;
2504 
2505 	spin_lock_irq(&vport->phba->hbalock);
2506 	nrport = lpfc_ndlp_get_nrport(ndlp);
2507 	if (nrport)
2508 		remoteport = nrport->remoteport;
2509 	spin_unlock_irq(&vport->phba->hbalock);
2510 
2511 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2512 			 "6170 Rescan NPort DID x%06x type x%x "
2513 			 "state x%x nrport x%px remoteport x%px\n",
2514 			 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2515 			 nrport, remoteport);
2516 
2517 	if (!nrport || !remoteport)
2518 		goto rescan_exit;
2519 
2520 	/* Only rescan if we are an NVME target in the MAPPED state */
2521 	if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2522 	    ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2523 		nvme_fc_rescan_remoteport(remoteport);
2524 
2525 		lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
2526 				 "6172 NVME rescanned DID x%06x "
2527 				 "port_state x%x\n",
2528 				 ndlp->nlp_DID, remoteport->port_state);
2529 	}
2530 	return;
2531  rescan_exit:
2532 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2533 			 "6169 Skip NVME Rport Rescan, NVME remoteport "
2534 			 "unregistered\n");
2535 #endif
2536 }
2537 
2538 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2539  *
2540  * There is no notion of Devloss or rport recovery from the current
2541  * nvme_transport perspective.  Loss of an rport just means IO cannot
2542  * be sent and recovery is completely up to the initator.
2543  * For now, the driver just unbinds the DID and port_role so that
2544  * no further IO can be issued.  Changes are planned for later.
2545  *
2546  * Notes - the ndlp reference count is not decremented here since
2547  * since there is no nvme_transport api for devloss.  Node ref count
2548  * is only adjusted in driver unload.
2549  */
2550 void
2551 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2552 {
2553 #if (IS_ENABLED(CONFIG_NVME_FC))
2554 	int ret;
2555 	struct nvme_fc_local_port *localport;
2556 	struct lpfc_nvme_lport *lport;
2557 	struct lpfc_nvme_rport *rport;
2558 	struct nvme_fc_remote_port *remoteport = NULL;
2559 
2560 	localport = vport->localport;
2561 
2562 	/* This is fundamental error.  The localport is always
2563 	 * available until driver unload.  Just exit.
2564 	 */
2565 	if (!localport)
2566 		return;
2567 
2568 	lport = (struct lpfc_nvme_lport *)localport->private;
2569 	if (!lport)
2570 		goto input_err;
2571 
2572 	spin_lock_irq(&vport->phba->hbalock);
2573 	rport = lpfc_ndlp_get_nrport(ndlp);
2574 	if (rport)
2575 		remoteport = rport->remoteport;
2576 	spin_unlock_irq(&vport->phba->hbalock);
2577 	if (!remoteport)
2578 		goto input_err;
2579 
2580 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2581 			 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2582 			 "port_id x%06x, portstate x%x port type x%x\n",
2583 			 remoteport, remoteport->port_name,
2584 			 remoteport->port_id, remoteport->port_state,
2585 			 ndlp->nlp_type);
2586 
2587 	/* Sanity check ndlp type.  Only call for NVME ports. Don't
2588 	 * clear any rport state until the transport calls back.
2589 	 */
2590 
2591 	if (ndlp->nlp_type & NLP_NVME_TARGET) {
2592 		/* No concern about the role change on the nvme remoteport.
2593 		 * The transport will update it.
2594 		 */
2595 		ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG;
2596 
2597 		/* Don't let the host nvme transport keep sending keep-alives
2598 		 * on this remoteport. Vport is unloading, no recovery. The
2599 		 * return values is ignored.  The upcall is a courtesy to the
2600 		 * transport.
2601 		 */
2602 		if (vport->load_flag & FC_UNLOADING)
2603 			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2604 
2605 		ret = nvme_fc_unregister_remoteport(remoteport);
2606 		if (ret != 0) {
2607 			lpfc_nlp_put(ndlp);
2608 			lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
2609 					 "6167 NVME unregister failed %d "
2610 					 "port_state x%x\n",
2611 					 ret, remoteport->port_state);
2612 		}
2613 	}
2614 	return;
2615 
2616  input_err:
2617 #endif
2618 	lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC,
2619 			 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2620 			 vport->localport, ndlp->rport, ndlp->nlp_DID);
2621 }
2622 
2623 /**
2624  * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2625  * @phba: pointer to lpfc hba data structure.
2626  * @axri: pointer to the fcp xri abort wcqe structure.
2627  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2628  *
2629  * This routine is invoked by the worker thread to process a SLI4 fast-path
2630  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2631  * here.
2632  **/
2633 void
2634 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2635 			   struct sli4_wcqe_xri_aborted *axri,
2636 			   struct lpfc_io_buf *lpfc_ncmd)
2637 {
2638 	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2639 	struct nvmefc_fcp_req *nvme_cmd = NULL;
2640 	struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2641 
2642 
2643 	if (ndlp)
2644 		lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2645 
2646 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2647 			"6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2648 			"xri released\n",
2649 			lpfc_ncmd->nvmeCmd, xri,
2650 			lpfc_ncmd->cur_iocbq.iotag);
2651 
2652 	/* Aborted NVME commands are required to not complete
2653 	 * before the abort exchange command fully completes.
2654 	 * Once completed, it is available via the put list.
2655 	 */
2656 	if (lpfc_ncmd->nvmeCmd) {
2657 		nvme_cmd = lpfc_ncmd->nvmeCmd;
2658 		nvme_cmd->done(nvme_cmd);
2659 		lpfc_ncmd->nvmeCmd = NULL;
2660 	}
2661 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2662 }
2663 
2664 /**
2665  * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2666  * @phba: Pointer to HBA context object.
2667  *
2668  * This function flushes all wqes in the nvme rings and frees all resources
2669  * in the txcmplq. This function does not issue abort wqes for the IO
2670  * commands in txcmplq, they will just be returned with
2671  * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2672  * slot has been permanently disabled.
2673  **/
2674 void
2675 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2676 {
2677 	struct lpfc_sli_ring  *pring;
2678 	u32 i, wait_cnt = 0;
2679 
2680 	if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2681 		return;
2682 
2683 	/* Cycle through all IO rings and make sure all outstanding
2684 	 * WQEs have been removed from the txcmplqs.
2685 	 */
2686 	for (i = 0; i < phba->cfg_hdw_queue; i++) {
2687 		if (!phba->sli4_hba.hdwq[i].io_wq)
2688 			continue;
2689 		pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2690 
2691 		if (!pring)
2692 			continue;
2693 
2694 		/* Retrieve everything on the txcmplq */
2695 		while (!list_empty(&pring->txcmplq)) {
2696 			msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2697 			wait_cnt++;
2698 
2699 			/* The sleep is 10mS.  Every ten seconds,
2700 			 * dump a message.  Something is wrong.
2701 			 */
2702 			if ((wait_cnt % 1000) == 0) {
2703 				lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR,
2704 						"6178 NVME IO not empty, "
2705 						"cnt %d\n", wait_cnt);
2706 			}
2707 		}
2708 	}
2709 }
2710 
2711 void
2712 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn)
2713 {
2714 #if (IS_ENABLED(CONFIG_NVME_FC))
2715 	struct lpfc_io_buf *lpfc_ncmd;
2716 	struct nvmefc_fcp_req *nCmd;
2717 	struct lpfc_nvme_fcpreq_priv *freqpriv;
2718 
2719 	if (!pwqeIn->context1) {
2720 		lpfc_sli_release_iocbq(phba, pwqeIn);
2721 		return;
2722 	}
2723 	/* For abort iocb just return, IO iocb will do a done call */
2724 	if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2725 	    CMD_ABORT_XRI_CX) {
2726 		lpfc_sli_release_iocbq(phba, pwqeIn);
2727 		return;
2728 	}
2729 	lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1;
2730 
2731 	spin_lock(&lpfc_ncmd->buf_lock);
2732 	if (!lpfc_ncmd->nvmeCmd) {
2733 		spin_unlock(&lpfc_ncmd->buf_lock);
2734 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
2735 		return;
2736 	}
2737 
2738 	nCmd = lpfc_ncmd->nvmeCmd;
2739 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2740 			"6194 NVME Cancel xri %x\n",
2741 			lpfc_ncmd->cur_iocbq.sli4_xritag);
2742 
2743 	nCmd->transferred_length = 0;
2744 	nCmd->rcv_rsplen = 0;
2745 	nCmd->status = NVME_SC_INTERNAL;
2746 	freqpriv = nCmd->private;
2747 	freqpriv->nvme_buf = NULL;
2748 	lpfc_ncmd->nvmeCmd = NULL;
2749 
2750 	spin_unlock(&lpfc_ncmd->buf_lock);
2751 	nCmd->done(nCmd);
2752 
2753 	/* Call release with XB=1 to queue the IO into the abort list. */
2754 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2755 #endif
2756 }
2757