xref: /openbmc/linux/drivers/scsi/lpfc/lpfc_nvme.c (revision 8a1d8dea)
1 /*******************************************************************
2  * This file is part of the Emulex Linux Device Driver for         *
3  * Fibre Channel Host Bus Adapters.                                *
4  * Copyright (C) 2017-2023 Broadcom. All Rights Reserved. The term *
5  * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.  *
6  * Copyright (C) 2004-2016 Emulex.  All rights reserved.           *
7  * EMULEX and SLI are trademarks of Emulex.                        *
8  * www.broadcom.com                                                *
9  * Portions Copyright (C) 2004-2005 Christoph Hellwig              *
10  *                                                                 *
11  * This program is free software; you can redistribute it and/or   *
12  * modify it under the terms of version 2 of the GNU General       *
13  * Public License as published by the Free Software Foundation.    *
14  * This program is distributed in the hope that it will be useful. *
15  * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND          *
16  * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY,  *
17  * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE      *
18  * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19  * TO BE LEGALLY INVALID.  See the GNU General Public License for  *
20  * more details, a copy of which can be found in the file COPYING  *
21  * included with this package.                                     *
22  ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
30 
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
38 
39 #include "lpfc_version.h"
40 #include "lpfc_hw4.h"
41 #include "lpfc_hw.h"
42 #include "lpfc_sli.h"
43 #include "lpfc_sli4.h"
44 #include "lpfc_nl.h"
45 #include "lpfc_disc.h"
46 #include "lpfc.h"
47 #include "lpfc_nvme.h"
48 #include "lpfc_scsi.h"
49 #include "lpfc_logmsg.h"
50 #include "lpfc_crtn.h"
51 #include "lpfc_vport.h"
52 #include "lpfc_debugfs.h"
53 
54 /* NVME initiator-based functions */
55 
56 static struct lpfc_io_buf *
57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
58 		  int idx, int expedite);
59 
60 static void
61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);
62 
63 static struct nvme_fc_port_template lpfc_nvme_template;
64 
65 /**
66  * lpfc_nvme_create_queue -
67  * @pnvme_lport: Transport localport that LS is to be issued from
68  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
69  * @qsize: Size of the queue in bytes
70  * @handle: An opaque driver handle used in follow-up calls.
71  *
72  * Driver registers this routine to preallocate and initialize any
73  * internal data structures to bind the @qidx to its internal IO queues.
74  * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
75  *
76  * Return value :
77  *   0 - Success
78  *   -EINVAL - Unsupported input value.
79  *   -ENOMEM - Could not alloc necessary memory
80  **/
81 static int
lpfc_nvme_create_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,u16 qsize,void ** handle)82 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,
83 		       unsigned int qidx, u16 qsize,
84 		       void **handle)
85 {
86 	struct lpfc_nvme_lport *lport;
87 	struct lpfc_vport *vport;
88 	struct lpfc_nvme_qhandle *qhandle;
89 	char *str;
90 
91 	if (!pnvme_lport->private)
92 		return -ENOMEM;
93 
94 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
95 	vport = lport->vport;
96 
97 	if (!vport || vport->load_flag & FC_UNLOADING ||
98 	    vport->phba->hba_flag & HBA_IOQ_FLUSH)
99 		return -ENODEV;
100 
101 	qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);
102 	if (qhandle == NULL)
103 		return -ENOMEM;
104 
105 	qhandle->cpu_id = raw_smp_processor_id();
106 	qhandle->qidx = qidx;
107 	/*
108 	 * NVME qidx == 0 is the admin queue, so both admin queue
109 	 * and first IO queue will use MSI-X vector and associated
110 	 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
111 	 */
112 	if (qidx) {
113 		str = "IO ";  /* IO queue */
114 		qhandle->index = ((qidx - 1) %
115 			lpfc_nvme_template.max_hw_queues);
116 	} else {
117 		str = "ADM";  /* Admin queue */
118 		qhandle->index = qidx;
119 	}
120 
121 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
122 			 "6073 Binding %s HdwQueue %d  (cpu %d) to "
123 			 "hdw_queue %d qhandle x%px\n", str,
124 			 qidx, qhandle->cpu_id, qhandle->index, qhandle);
125 	*handle = (void *)qhandle;
126 	return 0;
127 }
128 
129 /**
130  * lpfc_nvme_delete_queue -
131  * @pnvme_lport: Transport localport that LS is to be issued from
132  * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
133  * @handle: An opaque driver handle from lpfc_nvme_create_queue
134  *
135  * Driver registers this routine to free
136  * any internal data structures to bind the @qidx to its internal
137  * IO queues.
138  *
139  * Return value :
140  *   0 - Success
141  *   TODO:  What are the failure codes.
142  **/
143 static void
lpfc_nvme_delete_queue(struct nvme_fc_local_port * pnvme_lport,unsigned int qidx,void * handle)144 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,
145 		       unsigned int qidx,
146 		       void *handle)
147 {
148 	struct lpfc_nvme_lport *lport;
149 	struct lpfc_vport *vport;
150 
151 	if (!pnvme_lport->private)
152 		return;
153 
154 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
155 	vport = lport->vport;
156 
157 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
158 			"6001 ENTER.  lpfc_pnvme x%px, qidx x%x qhandle x%px\n",
159 			lport, qidx, handle);
160 	kfree(handle);
161 }
162 
163 static void
lpfc_nvme_localport_delete(struct nvme_fc_local_port * localport)164 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)
165 {
166 	struct lpfc_nvme_lport *lport = localport->private;
167 
168 	lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,
169 			 "6173 localport x%px delete complete\n",
170 			 lport);
171 
172 	/* release any threads waiting for the unreg to complete */
173 	if (lport->vport->localport)
174 		complete(lport->lport_unreg_cmp);
175 }
176 
177 /* lpfc_nvme_remoteport_delete
178  *
179  * @remoteport: Pointer to an nvme transport remoteport instance.
180  *
181  * This is a template downcall.  NVME transport calls this function
182  * when it has completed the unregistration of a previously
183  * registered remoteport.
184  *
185  * Return value :
186  * None
187  */
188 static void
lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port * remoteport)189 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)
190 {
191 	struct lpfc_nvme_rport *rport = remoteport->private;
192 	struct lpfc_vport *vport;
193 	struct lpfc_nodelist *ndlp;
194 	u32 fc4_xpt_flags;
195 
196 	ndlp = rport->ndlp;
197 	if (!ndlp) {
198 		pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",
199 		       __func__, rport, remoteport);
200 		goto rport_err;
201 	}
202 
203 	vport = ndlp->vport;
204 	if (!vport) {
205 		pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",
206 		       __func__, ndlp, ndlp->nlp_state, rport);
207 		goto rport_err;
208 	}
209 
210 	fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;
211 
212 	/* Remove this rport from the lport's list - memory is owned by the
213 	 * transport. Remove the ndlp reference for the NVME transport before
214 	 * calling state machine to remove the node.
215 	 */
216 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
217 			 "6146 remoteport delete of remoteport x%px, ndlp x%px "
218 			 "DID x%x xflags x%x\n",
219 			 remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);
220 	spin_lock_irq(&ndlp->lock);
221 
222 	/* The register rebind might have occurred before the delete
223 	 * downcall.  Guard against this race.
224 	 */
225 	if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)
226 		ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);
227 
228 	spin_unlock_irq(&ndlp->lock);
229 
230 	/* On a devloss timeout event, one more put is executed provided the
231 	 * NVME and SCSI rport unregister requests are complete.
232 	 */
233 	if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))
234 		lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);
235 
236  rport_err:
237 	return;
238 }
239 
240 /**
241  * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request
242  * @phba: pointer to lpfc hba data structure.
243  * @axchg: pointer to exchange context for the NVME LS request
244  *
245  * This routine is used for processing an asychronously received NVME LS
246  * request. Any remaining validation is done and the LS is then forwarded
247  * to the nvme-fc transport via nvme_fc_rcv_ls_req().
248  *
249  * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)
250  * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.
251  * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
252  *
253  * Returns 0 if LS was handled and delivered to the transport
254  * Returns 1 if LS failed to be handled and should be dropped
255  */
256 int
lpfc_nvme_handle_lsreq(struct lpfc_hba * phba,struct lpfc_async_xchg_ctx * axchg)257 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,
258 			struct lpfc_async_xchg_ctx *axchg)
259 {
260 #if (IS_ENABLED(CONFIG_NVME_FC))
261 	struct lpfc_vport *vport;
262 	struct lpfc_nvme_rport *lpfc_rport;
263 	struct nvme_fc_remote_port *remoteport;
264 	struct lpfc_nvme_lport *lport;
265 	uint32_t *payload = axchg->payload;
266 	int rc;
267 
268 	vport = axchg->ndlp->vport;
269 	lpfc_rport = axchg->ndlp->nrport;
270 	if (!lpfc_rport)
271 		return -EINVAL;
272 
273 	remoteport = lpfc_rport->remoteport;
274 	if (!vport->localport ||
275 	    vport->phba->hba_flag & HBA_IOQ_FLUSH)
276 		return -EINVAL;
277 
278 	lport = vport->localport->private;
279 	if (!lport)
280 		return -EINVAL;
281 
282 	rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,
283 				axchg->size);
284 
285 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
286 			"6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "
287 			"%08x %08x %08x\n",
288 			axchg->size, rc,
289 			*payload, *(payload+1), *(payload+2),
290 			*(payload+3), *(payload+4), *(payload+5));
291 
292 	if (!rc)
293 		return 0;
294 #endif
295 	return 1;
296 }
297 
298 /**
299  * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME
300  *        LS request.
301  * @phba: Pointer to HBA context object
302  * @vport: The local port that issued the LS
303  * @cmdwqe: Pointer to driver command WQE object.
304  * @wcqe: Pointer to driver response CQE object.
305  *
306  * This function is the generic completion handler for NVME LS requests.
307  * The function updates any states and statistics, calls the transport
308  * ls_req done() routine, then tears down the command and buffers used
309  * for the LS request.
310  **/
311 void
__lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_vport * vport,struct lpfc_iocbq * cmdwqe,struct lpfc_wcqe_complete * wcqe)312 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,
313 			struct lpfc_iocbq *cmdwqe,
314 			struct lpfc_wcqe_complete *wcqe)
315 {
316 	struct nvmefc_ls_req *pnvme_lsreq;
317 	struct lpfc_dmabuf *buf_ptr;
318 	struct lpfc_nodelist *ndlp;
319 	int status;
320 
321 	pnvme_lsreq = cmdwqe->context_un.nvme_lsreq;
322 	ndlp = cmdwqe->ndlp;
323 	buf_ptr = cmdwqe->bpl_dmabuf;
324 
325 	status = bf_get(lpfc_wcqe_c_status, wcqe);
326 
327 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
328 			 "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "
329 			 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "
330 			 "ndlp:x%px\n",
331 			 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
332 			 cmdwqe->sli4_xritag, status,
333 			 (wcqe->parameter & 0xffff),
334 			 cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf,
335 			 ndlp);
336 
337 	lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",
338 			 cmdwqe->sli4_xritag, status, wcqe->parameter);
339 
340 	if (buf_ptr) {
341 		lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);
342 		kfree(buf_ptr);
343 		cmdwqe->bpl_dmabuf = NULL;
344 	}
345 	if (pnvme_lsreq->done) {
346 		if (status != CQE_STATUS_SUCCESS)
347 			status = -ENXIO;
348 		pnvme_lsreq->done(pnvme_lsreq, status);
349 	} else {
350 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
351 				 "6046 NVMEx cmpl without done call back? "
352 				 "Data x%px DID %x Xri: %x status %x\n",
353 				pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,
354 				cmdwqe->sli4_xritag, status);
355 	}
356 	if (ndlp) {
357 		lpfc_nlp_put(ndlp);
358 		cmdwqe->ndlp = NULL;
359 	}
360 	lpfc_sli_release_iocbq(phba, cmdwqe);
361 }
362 
363 static void
lpfc_nvme_ls_req_cmp(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe)364 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
365 		     struct lpfc_iocbq *rspwqe)
366 {
367 	struct lpfc_vport *vport = cmdwqe->vport;
368 	struct lpfc_nvme_lport *lport;
369 	uint32_t status;
370 	struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
371 
372 	status = bf_get(lpfc_wcqe_c_status, wcqe);
373 
374 	if (vport->localport) {
375 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
376 		if (lport) {
377 			atomic_inc(&lport->fc4NvmeLsCmpls);
378 			if (status) {
379 				if (bf_get(lpfc_wcqe_c_xb, wcqe))
380 					atomic_inc(&lport->cmpl_ls_xb);
381 				atomic_inc(&lport->cmpl_ls_err);
382 			}
383 		}
384 	}
385 
386 	__lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);
387 }
388 
389 static int
lpfc_nvme_gen_req(struct lpfc_vport * vport,struct lpfc_dmabuf * bmp,struct lpfc_dmabuf * inp,struct nvmefc_ls_req * pnvme_lsreq,void (* cmpl)(struct lpfc_hba *,struct lpfc_iocbq *,struct lpfc_iocbq *),struct lpfc_nodelist * ndlp,uint32_t num_entry,uint32_t tmo,uint8_t retry)390 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,
391 		  struct lpfc_dmabuf *inp,
392 		  struct nvmefc_ls_req *pnvme_lsreq,
393 		  void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,
394 			       struct lpfc_iocbq *),
395 		  struct lpfc_nodelist *ndlp, uint32_t num_entry,
396 		  uint32_t tmo, uint8_t retry)
397 {
398 	struct lpfc_hba *phba = vport->phba;
399 	union lpfc_wqe128 *wqe;
400 	struct lpfc_iocbq *genwqe;
401 	struct ulp_bde64 *bpl;
402 	struct ulp_bde64 bde;
403 	int i, rc, xmit_len, first_len;
404 
405 	/* Allocate buffer for  command WQE */
406 	genwqe = lpfc_sli_get_iocbq(phba);
407 	if (genwqe == NULL)
408 		return 1;
409 
410 	wqe = &genwqe->wqe;
411 	/* Initialize only 64 bytes */
412 	memset(wqe, 0, sizeof(union lpfc_wqe));
413 
414 	genwqe->bpl_dmabuf = bmp;
415 	genwqe->cmd_flag |= LPFC_IO_NVME_LS;
416 
417 	/* Save for completion so we can release these resources */
418 	genwqe->ndlp = lpfc_nlp_get(ndlp);
419 	if (!genwqe->ndlp) {
420 		dev_warn(&phba->pcidev->dev,
421 			 "Warning: Failed node ref, not sending LS_REQ\n");
422 		lpfc_sli_release_iocbq(phba, genwqe);
423 		return 1;
424 	}
425 
426 	genwqe->context_un.nvme_lsreq = pnvme_lsreq;
427 	/* Fill in payload, bp points to frame payload */
428 
429 	if (!tmo)
430 		/* FC spec states we need 3 * ratov for CT requests */
431 		tmo = (3 * phba->fc_ratov);
432 
433 	/* For this command calculate the xmit length of the request bde. */
434 	xmit_len = 0;
435 	first_len = 0;
436 	bpl = (struct ulp_bde64 *)bmp->virt;
437 	for (i = 0; i < num_entry; i++) {
438 		bde.tus.w = bpl[i].tus.w;
439 		if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)
440 			break;
441 		xmit_len += bde.tus.f.bdeSize;
442 		if (i == 0)
443 			first_len = xmit_len;
444 	}
445 
446 	genwqe->num_bdes = num_entry;
447 	genwqe->hba_wqidx = 0;
448 
449 	/* Words 0 - 2 */
450 	wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
451 	wqe->generic.bde.tus.f.bdeSize = first_len;
452 	wqe->generic.bde.addrLow = bpl[0].addrLow;
453 	wqe->generic.bde.addrHigh = bpl[0].addrHigh;
454 
455 	/* Word 3 */
456 	wqe->gen_req.request_payload_len = first_len;
457 
458 	/* Word 4 */
459 
460 	/* Word 5 */
461 	bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);
462 	bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);
463 	bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);
464 	bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);
465 	bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);
466 
467 	/* Word 6 */
468 	bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,
469 	       phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
470 	bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);
471 
472 	/* Word 7 */
473 	bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);
474 	bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);
475 	bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);
476 	bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);
477 
478 	/* Word 8 */
479 	wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;
480 
481 	/* Word 9 */
482 	bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);
483 
484 	/* Word 10 */
485 	bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);
486 	bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);
487 	bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);
488 	bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);
489 	bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);
490 
491 	/* Word 11 */
492 	bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
493 	bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);
494 
495 
496 	/* Issue GEN REQ WQE for NPORT <did> */
497 	genwqe->cmd_cmpl = cmpl;
498 	genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;
499 	genwqe->vport = vport;
500 	genwqe->retry = retry;
501 
502 	lpfc_nvmeio_data(phba, "NVME LS  XMIT: xri x%x iotag x%x to x%06x\n",
503 			 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);
504 
505 	rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);
506 	if (rc) {
507 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
508 				 "6045 Issue GEN REQ WQE to NPORT x%x "
509 				 "Data: x%x x%x  rc x%x\n",
510 				 ndlp->nlp_DID, genwqe->iotag,
511 				 vport->port_state, rc);
512 		lpfc_nlp_put(ndlp);
513 		lpfc_sli_release_iocbq(phba, genwqe);
514 		return 1;
515 	}
516 
517 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,
518 			 "6050 Issue GEN REQ WQE to NPORT x%x "
519 			 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "
520 			 "bmp:x%px xmit:%d 1st:%d\n",
521 			 ndlp->nlp_DID, genwqe->sli4_xritag,
522 			 vport->port_state,
523 			 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);
524 	return 0;
525 }
526 
527 
528 /**
529  * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request
530  * @vport: The local port issuing the LS
531  * @ndlp: The remote port to send the LS to
532  * @pnvme_lsreq: Pointer to LS request structure from the transport
533  * @gen_req_cmp: Completion call-back
534  *
535  * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST
536  * WQE to perform the LS operation.
537  *
538  * Return value :
539  *   0 - Success
540  *   non-zero: various error codes, in form of -Exxx
541  **/
542 int
__lpfc_nvme_ls_req(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq,void (* gen_req_cmp)(struct lpfc_hba * phba,struct lpfc_iocbq * cmdwqe,struct lpfc_iocbq * rspwqe))543 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
544 		      struct nvmefc_ls_req *pnvme_lsreq,
545 		      void (*gen_req_cmp)(struct lpfc_hba *phba,
546 				struct lpfc_iocbq *cmdwqe,
547 				struct lpfc_iocbq *rspwqe))
548 {
549 	struct lpfc_dmabuf *bmp;
550 	struct ulp_bde64 *bpl;
551 	int ret;
552 	uint16_t ntype, nstate;
553 
554 	if (!ndlp) {
555 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
556 				 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "
557 				 "LS Req\n",
558 				 ndlp);
559 		return -ENODEV;
560 	}
561 
562 	ntype = ndlp->nlp_type;
563 	nstate = ndlp->nlp_state;
564 	if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||
565 	    (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {
566 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
567 				 "6088 NVMEx LS REQ: Fail DID x%06x not "
568 				 "ready for IO. Type x%x, State x%x\n",
569 				 ndlp->nlp_DID, ntype, nstate);
570 		return -ENODEV;
571 	}
572 	if (vport->phba->hba_flag & HBA_IOQ_FLUSH)
573 		return -ENODEV;
574 
575 	if (!vport->phba->sli4_hba.nvmels_wq)
576 		return -ENOMEM;
577 
578 	/*
579 	 * there are two dma buf in the request, actually there is one and
580 	 * the second one is just the start address + cmd size.
581 	 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
582 	 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
583 	 * because the nvem layer owns the data bufs.
584 	 * We do not have to break these packets open, we don't care what is
585 	 * in them. And we do not have to look at the resonse data, we only
586 	 * care that we got a response. All of the caring is going to happen
587 	 * in the nvme-fc layer.
588 	 */
589 
590 	bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);
591 	if (!bmp) {
592 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
593 				 "6044 NVMEx LS REQ: Could not alloc LS buf "
594 				 "for DID %x\n",
595 				 ndlp->nlp_DID);
596 		return -ENOMEM;
597 	}
598 
599 	bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));
600 	if (!bmp->virt) {
601 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
602 				 "6042 NVMEx LS REQ: Could not alloc mbuf "
603 				 "for DID %x\n",
604 				 ndlp->nlp_DID);
605 		kfree(bmp);
606 		return -ENOMEM;
607 	}
608 
609 	INIT_LIST_HEAD(&bmp->list);
610 
611 	bpl = (struct ulp_bde64 *)bmp->virt;
612 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));
613 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));
614 	bpl->tus.f.bdeFlags = 0;
615 	bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;
616 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
617 	bpl++;
618 
619 	bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));
620 	bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));
621 	bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;
622 	bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;
623 	bpl->tus.w = le32_to_cpu(bpl->tus.w);
624 
625 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
626 			"6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "
627 			"rqstlen:%d rsplen:%d %pad %pad\n",
628 			ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,
629 			pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
630 			&pnvme_lsreq->rspdma);
631 
632 	ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,
633 				pnvme_lsreq, gen_req_cmp, ndlp, 2,
634 				pnvme_lsreq->timeout, 0);
635 	if (ret != WQE_SUCCESS) {
636 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
637 				 "6052 NVMEx REQ: EXIT. issue ls wqe failed "
638 				 "lsreq x%px Status %x DID %x\n",
639 				 pnvme_lsreq, ret, ndlp->nlp_DID);
640 		lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);
641 		kfree(bmp);
642 		return -EIO;
643 	}
644 
645 	return 0;
646 }
647 
648 /**
649  * lpfc_nvme_ls_req - Issue an NVME Link Service request
650  * @pnvme_lport: Transport localport that LS is to be issued from.
651  * @pnvme_rport: Transport remoteport that LS is to be sent to.
652  * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
653  *
654  * Driver registers this routine to handle any link service request
655  * from the nvme_fc transport to a remote nvme-aware port.
656  *
657  * Return value :
658  *   0 - Success
659  *   non-zero: various error codes, in form of -Exxx
660  **/
661 static int
lpfc_nvme_ls_req(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)662 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,
663 		 struct nvme_fc_remote_port *pnvme_rport,
664 		 struct nvmefc_ls_req *pnvme_lsreq)
665 {
666 	struct lpfc_nvme_lport *lport;
667 	struct lpfc_nvme_rport *rport;
668 	struct lpfc_vport *vport;
669 	int ret;
670 
671 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
672 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
673 	if (unlikely(!lport) || unlikely(!rport))
674 		return -EINVAL;
675 
676 	vport = lport->vport;
677 	if (vport->load_flag & FC_UNLOADING ||
678 	    vport->phba->hba_flag & HBA_IOQ_FLUSH)
679 		return -ENODEV;
680 
681 	atomic_inc(&lport->fc4NvmeLsRequests);
682 
683 	ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,
684 				 lpfc_nvme_ls_req_cmp);
685 	if (ret)
686 		atomic_inc(&lport->xmt_ls_err);
687 
688 	return ret;
689 }
690 
691 /**
692  * __lpfc_nvme_ls_abort - Generic service routine to abort a prior
693  *         NVME LS request
694  * @vport: The local port that issued the LS
695  * @ndlp: The remote port the LS was sent to
696  * @pnvme_lsreq: Pointer to LS request structure from the transport
697  *
698  * The driver validates the ndlp, looks for the LS, and aborts the
699  * LS if found.
700  *
701  * Returns:
702  * 0 : if LS found and aborted
703  * non-zero: various error conditions in form -Exxx
704  **/
705 int
__lpfc_nvme_ls_abort(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp,struct nvmefc_ls_req * pnvme_lsreq)706 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
707 			struct nvmefc_ls_req *pnvme_lsreq)
708 {
709 	struct lpfc_hba *phba = vport->phba;
710 	struct lpfc_sli_ring *pring;
711 	struct lpfc_iocbq *wqe, *next_wqe;
712 	bool foundit = false;
713 
714 	if (!ndlp) {
715 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
716 				"6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "
717 				"x%06x, Failing LS Req\n",
718 				ndlp, ndlp ? ndlp->nlp_DID : 0);
719 		return -EINVAL;
720 	}
721 
722 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
723 			 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "
724 			 "x%px rqstlen:%d rsplen:%d %pad %pad\n",
725 			 pnvme_lsreq, pnvme_lsreq->rqstlen,
726 			 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,
727 			 &pnvme_lsreq->rspdma);
728 
729 	/*
730 	 * Lock the ELS ring txcmplq and look for the wqe that matches
731 	 * this ELS. If found, issue an abort on the wqe.
732 	 */
733 	pring = phba->sli4_hba.nvmels_wq->pring;
734 	spin_lock_irq(&phba->hbalock);
735 	spin_lock(&pring->ring_lock);
736 	list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {
737 		if (wqe->context_un.nvme_lsreq == pnvme_lsreq) {
738 			wqe->cmd_flag |= LPFC_DRIVER_ABORTED;
739 			foundit = true;
740 			break;
741 		}
742 	}
743 	spin_unlock(&pring->ring_lock);
744 
745 	if (foundit)
746 		lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);
747 	spin_unlock_irq(&phba->hbalock);
748 
749 	if (foundit)
750 		return 0;
751 
752 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,
753 			 "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",
754 			 pnvme_lsreq);
755 	return -EINVAL;
756 }
757 
758 static int
lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port * localport,struct nvme_fc_remote_port * remoteport,struct nvmefc_ls_rsp * ls_rsp)759 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,
760 		     struct nvme_fc_remote_port *remoteport,
761 		     struct nvmefc_ls_rsp *ls_rsp)
762 {
763 	struct lpfc_async_xchg_ctx *axchg =
764 		container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
765 	struct lpfc_nvme_lport *lport;
766 	int rc;
767 
768 	if (axchg->phba->pport->load_flag & FC_UNLOADING)
769 		return -ENODEV;
770 
771 	lport = (struct lpfc_nvme_lport *)localport->private;
772 
773 	rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);
774 
775 	if (rc) {
776 		/*
777 		 * unless the failure is due to having already sent
778 		 * the response, an abort will be generated for the
779 		 * exchange if the rsp can't be sent.
780 		 */
781 		if (rc != -EALREADY)
782 			atomic_inc(&lport->xmt_ls_abort);
783 		return rc;
784 	}
785 
786 	return 0;
787 }
788 
789 /**
790  * lpfc_nvme_ls_abort - Abort a prior NVME LS request
791  * @pnvme_lport: Transport localport that LS is to be issued from.
792  * @pnvme_rport: Transport remoteport that LS is to be sent to.
793  * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
794  *
795  * Driver registers this routine to abort a NVME LS request that is
796  * in progress (from the transports perspective).
797  **/
798 static void
lpfc_nvme_ls_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,struct nvmefc_ls_req * pnvme_lsreq)799 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,
800 		   struct nvme_fc_remote_port *pnvme_rport,
801 		   struct nvmefc_ls_req *pnvme_lsreq)
802 {
803 	struct lpfc_nvme_lport *lport;
804 	struct lpfc_vport *vport;
805 	struct lpfc_nodelist *ndlp;
806 	int ret;
807 
808 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
809 	if (unlikely(!lport))
810 		return;
811 	vport = lport->vport;
812 
813 	if (vport->load_flag & FC_UNLOADING)
814 		return;
815 
816 	ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);
817 
818 	ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);
819 	if (!ret)
820 		atomic_inc(&lport->xmt_ls_abort);
821 }
822 
823 /* Fix up the existing sgls for NVME IO. */
824 static inline void
lpfc_nvme_adj_fcp_sgls(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct nvmefc_fcp_req * nCmd)825 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,
826 		       struct lpfc_io_buf *lpfc_ncmd,
827 		       struct nvmefc_fcp_req *nCmd)
828 {
829 	struct lpfc_hba  *phba = vport->phba;
830 	struct sli4_sge *sgl;
831 	union lpfc_wqe128 *wqe;
832 	uint32_t *wptr, *dptr;
833 
834 	/*
835 	 * Get a local pointer to the built-in wqe and correct
836 	 * the cmd size to match NVME's 96 bytes and fix
837 	 * the dma address.
838 	 */
839 
840 	wqe = &lpfc_ncmd->cur_iocbq.wqe;
841 
842 	/*
843 	 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
844 	 * match NVME.  NVME sends 96 bytes. Also, use the
845 	 * nvme commands command and response dma addresses
846 	 * rather than the virtual memory to ease the restore
847 	 * operation.
848 	 */
849 	sgl = lpfc_ncmd->dma_sgl;
850 	sgl->sge_len = cpu_to_le32(nCmd->cmdlen);
851 	if (phba->cfg_nvme_embed_cmd) {
852 		sgl->addr_hi = 0;
853 		sgl->addr_lo = 0;
854 
855 		/* Word 0-2 - NVME CMND IU (embedded payload) */
856 		wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;
857 		wqe->generic.bde.tus.f.bdeSize = 56;
858 		wqe->generic.bde.addrHigh = 0;
859 		wqe->generic.bde.addrLow =  64;  /* Word 16 */
860 
861 		/* Word 10  - dbde is 0, wqes is 1 in template */
862 
863 		/*
864 		 * Embed the payload in the last half of the WQE
865 		 * WQE words 16-30 get the NVME CMD IU payload
866 		 *
867 		 * WQE words 16-19 get payload Words 1-4
868 		 * WQE words 20-21 get payload Words 6-7
869 		 * WQE words 22-29 get payload Words 16-23
870 		 */
871 		wptr = &wqe->words[16];  /* WQE ptr */
872 		dptr = (uint32_t *)nCmd->cmdaddr;  /* payload ptr */
873 		dptr++;			/* Skip Word 0 in payload */
874 
875 		*wptr++ = *dptr++;	/* Word 1 */
876 		*wptr++ = *dptr++;	/* Word 2 */
877 		*wptr++ = *dptr++;	/* Word 3 */
878 		*wptr++ = *dptr++;	/* Word 4 */
879 		dptr++;			/* Skip Word 5 in payload */
880 		*wptr++ = *dptr++;	/* Word 6 */
881 		*wptr++ = *dptr++;	/* Word 7 */
882 		dptr += 8;		/* Skip Words 8-15 in payload */
883 		*wptr++ = *dptr++;	/* Word 16 */
884 		*wptr++ = *dptr++;	/* Word 17 */
885 		*wptr++ = *dptr++;	/* Word 18 */
886 		*wptr++ = *dptr++;	/* Word 19 */
887 		*wptr++ = *dptr++;	/* Word 20 */
888 		*wptr++ = *dptr++;	/* Word 21 */
889 		*wptr++ = *dptr++;	/* Word 22 */
890 		*wptr   = *dptr;	/* Word 23 */
891 	} else {
892 		sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));
893 		sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));
894 
895 		/* Word 0-2 - NVME CMND IU Inline BDE */
896 		wqe->generic.bde.tus.f.bdeFlags =  BUFF_TYPE_BDE_64;
897 		wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;
898 		wqe->generic.bde.addrHigh = sgl->addr_hi;
899 		wqe->generic.bde.addrLow =  sgl->addr_lo;
900 
901 		/* Word 10 */
902 		bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);
903 		bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);
904 	}
905 
906 	sgl++;
907 
908 	/* Setup the physical region for the FCP RSP */
909 	sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));
910 	sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));
911 	sgl->word2 = le32_to_cpu(sgl->word2);
912 	if (nCmd->sg_cnt)
913 		bf_set(lpfc_sli4_sge_last, sgl, 0);
914 	else
915 		bf_set(lpfc_sli4_sge_last, sgl, 1);
916 	sgl->word2 = cpu_to_le32(sgl->word2);
917 	sgl->sge_len = cpu_to_le32(nCmd->rsplen);
918 }
919 
920 
921 /*
922  * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO
923  *
924  * Driver registers this routine as it io request handler.  This
925  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
926  * data structure to the rport indicated in @lpfc_nvme_rport.
927  *
928  * Return value :
929  *   0 - Success
930  *   TODO: What are the failure codes.
931  **/
932 static void
lpfc_nvme_io_cmd_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,struct lpfc_iocbq * pwqeOut)933 lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
934 		      struct lpfc_iocbq *pwqeOut)
935 {
936 	struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;
937 	struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;
938 	struct lpfc_vport *vport = pwqeIn->vport;
939 	struct nvmefc_fcp_req *nCmd;
940 	struct nvme_fc_ersp_iu *ep;
941 	struct nvme_fc_cmd_iu *cp;
942 	struct lpfc_nodelist *ndlp;
943 	struct lpfc_nvme_fcpreq_priv *freqpriv;
944 	struct lpfc_nvme_lport *lport;
945 	uint32_t code, status, idx;
946 	uint16_t cid, sqhd, data;
947 	uint32_t *ptr;
948 	uint32_t lat;
949 	bool call_done = false;
950 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
951 	int cpu;
952 #endif
953 	int offline = 0;
954 
955 	/* Sanity check on return of outstanding command */
956 	if (!lpfc_ncmd) {
957 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
958 				 "6071 Null lpfc_ncmd pointer. No "
959 				 "release, skip completion\n");
960 		return;
961 	}
962 
963 	/* Guard against abort handler being called at same time */
964 	spin_lock(&lpfc_ncmd->buf_lock);
965 
966 	if (!lpfc_ncmd->nvmeCmd) {
967 		spin_unlock(&lpfc_ncmd->buf_lock);
968 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
969 				 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "
970 				 "nvmeCmd x%px\n",
971 				 lpfc_ncmd, lpfc_ncmd->nvmeCmd);
972 
973 		/* Release the lpfc_ncmd regardless of the missing elements. */
974 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
975 		return;
976 	}
977 	nCmd = lpfc_ncmd->nvmeCmd;
978 	status = bf_get(lpfc_wcqe_c_status, wcqe);
979 
980 	idx = lpfc_ncmd->cur_iocbq.hba_wqidx;
981 	phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;
982 
983 	if (unlikely(status && vport->localport)) {
984 		lport = (struct lpfc_nvme_lport *)vport->localport->private;
985 		if (lport) {
986 			if (bf_get(lpfc_wcqe_c_xb, wcqe))
987 				atomic_inc(&lport->cmpl_fcp_xb);
988 			atomic_inc(&lport->cmpl_fcp_err);
989 		}
990 	}
991 
992 	lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
993 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
994 			 status, wcqe->parameter);
995 	/*
996 	 * Catch race where our node has transitioned, but the
997 	 * transport is still transitioning.
998 	 */
999 	ndlp = lpfc_ncmd->ndlp;
1000 	if (!ndlp) {
1001 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1002 				 "6062 Ignoring NVME cmpl.  No ndlp\n");
1003 		goto out_err;
1004 	}
1005 
1006 	code = bf_get(lpfc_wcqe_c_code, wcqe);
1007 	if (code == CQE_CODE_NVME_ERSP) {
1008 		/* For this type of CQE, we need to rebuild the rsp */
1009 		ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1010 
1011 		/*
1012 		 * Get Command Id from cmd to plug into response. This
1013 		 * code is not needed in the next NVME Transport drop.
1014 		 */
1015 		cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1016 		cid = cp->sqe.common.command_id;
1017 
1018 		/*
1019 		 * RSN is in CQE word 2
1020 		 * SQHD is in CQE Word 3 bits 15:0
1021 		 * Cmd Specific info is in CQE Word 1
1022 		 * and in CQE Word 0 bits 15:0
1023 		 */
1024 		sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);
1025 
1026 		/* Now lets build the NVME ERSP IU */
1027 		ep->iu_len = cpu_to_be16(8);
1028 		ep->rsn = wcqe->parameter;
1029 		ep->xfrd_len = cpu_to_be32(nCmd->payload_length);
1030 		ep->rsvd12 = 0;
1031 		ptr = (uint32_t *)&ep->cqe.result.u64;
1032 		*ptr++ = wcqe->total_data_placed;
1033 		data = bf_get(lpfc_wcqe_c_ersp0, wcqe);
1034 		*ptr = (uint32_t)data;
1035 		ep->cqe.sq_head = sqhd;
1036 		ep->cqe.sq_id =  nCmd->sqid;
1037 		ep->cqe.command_id = cid;
1038 		ep->cqe.status = 0;
1039 
1040 		lpfc_ncmd->status = IOSTAT_SUCCESS;
1041 		lpfc_ncmd->result = 0;
1042 		nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;
1043 		nCmd->transferred_length = nCmd->payload_length;
1044 	} else {
1045 		lpfc_ncmd->status = status;
1046 		lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);
1047 
1048 		/* For NVME, the only failure path that results in an
1049 		 * IO error is when the adapter rejects it.  All other
1050 		 * conditions are a success case and resolved by the
1051 		 * transport.
1052 		 * IOSTAT_FCP_RSP_ERROR means:
1053 		 * 1. Length of data received doesn't match total
1054 		 *    transfer length in WQE
1055 		 * 2. If the RSP payload does NOT match these cases:
1056 		 *    a. RSP length 12/24 bytes and all zeros
1057 		 *    b. NVME ERSP
1058 		 */
1059 		switch (lpfc_ncmd->status) {
1060 		case IOSTAT_SUCCESS:
1061 			nCmd->transferred_length = wcqe->total_data_placed;
1062 			nCmd->rcv_rsplen = 0;
1063 			nCmd->status = 0;
1064 			break;
1065 		case IOSTAT_FCP_RSP_ERROR:
1066 			nCmd->transferred_length = wcqe->total_data_placed;
1067 			nCmd->rcv_rsplen = wcqe->parameter;
1068 			nCmd->status = 0;
1069 
1070 			/* Get the NVME cmd details for this unique error. */
1071 			cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;
1072 			ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;
1073 
1074 			/* Check if this is really an ERSP */
1075 			if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {
1076 				lpfc_ncmd->status = IOSTAT_SUCCESS;
1077 				lpfc_ncmd->result = 0;
1078 
1079 				lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
1080 					"6084 NVME FCP_ERR ERSP: "
1081 					"xri %x placed x%x opcode x%x cmd_id "
1082 					"x%x cqe_status x%x\n",
1083 					lpfc_ncmd->cur_iocbq.sli4_xritag,
1084 					wcqe->total_data_placed,
1085 					cp->sqe.common.opcode,
1086 					cp->sqe.common.command_id,
1087 					ep->cqe.status);
1088 				break;
1089 			}
1090 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1091 					 "6081 NVME Completion Protocol Error: "
1092 					 "xri %x status x%x result x%x "
1093 					 "placed x%x opcode x%x cmd_id x%x, "
1094 					 "cqe_status x%x\n",
1095 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1096 					 lpfc_ncmd->status, lpfc_ncmd->result,
1097 					 wcqe->total_data_placed,
1098 					 cp->sqe.common.opcode,
1099 					 cp->sqe.common.command_id,
1100 					 ep->cqe.status);
1101 			break;
1102 		case IOSTAT_LOCAL_REJECT:
1103 			/* Let fall through to set command final state. */
1104 			if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)
1105 				lpfc_printf_vlog(vport, KERN_INFO,
1106 					 LOG_NVME_IOERR,
1107 					 "6032 Delay Aborted cmd x%px "
1108 					 "nvme cmd x%px, xri x%x, "
1109 					 "xb %d\n",
1110 					 lpfc_ncmd, nCmd,
1111 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1112 					 bf_get(lpfc_wcqe_c_xb, wcqe));
1113 			fallthrough;
1114 		default:
1115 out_err:
1116 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1117 					 "6072 NVME Completion Error: xri %x "
1118 					 "status x%x result x%x [x%x] "
1119 					 "placed x%x\n",
1120 					 lpfc_ncmd->cur_iocbq.sli4_xritag,
1121 					 lpfc_ncmd->status, lpfc_ncmd->result,
1122 					 wcqe->parameter,
1123 					 wcqe->total_data_placed);
1124 			nCmd->transferred_length = 0;
1125 			nCmd->rcv_rsplen = 0;
1126 			nCmd->status = NVME_SC_INTERNAL;
1127 			offline = pci_channel_offline(vport->phba->pcidev);
1128 		}
1129 	}
1130 
1131 	/* pick up SLI4 exhange busy condition */
1132 	if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline)
1133 		lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;
1134 	else
1135 		lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;
1136 
1137 	/* Update stats and complete the IO.  There is
1138 	 * no need for dma unprep because the nvme_transport
1139 	 * owns the dma address.
1140 	 */
1141 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1142 	if (lpfc_ncmd->ts_cmd_start) {
1143 		lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;
1144 		lpfc_ncmd->ts_data_io = ktime_get_ns();
1145 		phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;
1146 		lpfc_io_ktime(phba, lpfc_ncmd);
1147 	}
1148 	if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {
1149 		cpu = raw_smp_processor_id();
1150 		this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
1151 		if (lpfc_ncmd->cpu != cpu)
1152 			lpfc_printf_vlog(vport,
1153 					 KERN_INFO, LOG_NVME_IOERR,
1154 					 "6701 CPU Check cmpl: "
1155 					 "cpu %d expect %d\n",
1156 					 cpu, lpfc_ncmd->cpu);
1157 	}
1158 #endif
1159 
1160 	/* NVME targets need completion held off until the abort exchange
1161 	 * completes unless the NVME Rport is getting unregistered.
1162 	 */
1163 
1164 	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
1165 		freqpriv = nCmd->private;
1166 		freqpriv->nvme_buf = NULL;
1167 		lpfc_ncmd->nvmeCmd = NULL;
1168 		call_done = true;
1169 	}
1170 	spin_unlock(&lpfc_ncmd->buf_lock);
1171 
1172 	/* Check if IO qualified for CMF */
1173 	if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1174 	    nCmd->io_dir == NVMEFC_FCP_READ &&
1175 	    nCmd->payload_length) {
1176 		/* Used when calculating average latency */
1177 		lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;
1178 		lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL);
1179 	}
1180 
1181 	if (call_done)
1182 		nCmd->done(nCmd);
1183 
1184 	/* Call release with XB=1 to queue the IO into the abort list. */
1185 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1186 }
1187 
1188 
1189 /**
1190  * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1191  * @vport: pointer to a host virtual N_Port data structure
1192  * @lpfc_ncmd: Pointer to lpfc scsi command
1193  * @pnode: pointer to a node-list data structure
1194  * @cstat: pointer to the control status structure
1195  *
1196  * Driver registers this routine as it io request handler.  This
1197  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1198  * data structure to the rport indicated in @lpfc_nvme_rport.
1199  *
1200  * Return value :
1201  *   0 - Success
1202  *   TODO: What are the failure codes.
1203  **/
1204 static int
lpfc_nvme_prep_io_cmd(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd,struct lpfc_nodelist * pnode,struct lpfc_fc4_ctrl_stat * cstat)1205 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,
1206 		      struct lpfc_io_buf *lpfc_ncmd,
1207 		      struct lpfc_nodelist *pnode,
1208 		      struct lpfc_fc4_ctrl_stat *cstat)
1209 {
1210 	struct lpfc_hba *phba = vport->phba;
1211 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1212 	struct nvme_common_command *sqe;
1213 	struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;
1214 	union lpfc_wqe128 *wqe = &pwqeq->wqe;
1215 	uint32_t req_len;
1216 
1217 	/*
1218 	 * There are three possibilities here - use scatter-gather segment, use
1219 	 * the single mapping, or neither.
1220 	 */
1221 	if (nCmd->sg_cnt) {
1222 		if (nCmd->io_dir == NVMEFC_FCP_WRITE) {
1223 			/* From the iwrite template, initialize words 7 - 11 */
1224 			memcpy(&wqe->words[7],
1225 			       &lpfc_iwrite_cmd_template.words[7],
1226 			       sizeof(uint32_t) * 5);
1227 
1228 			/* Word 4 */
1229 			wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;
1230 
1231 			/* Word 5 */
1232 			if ((phba->cfg_nvme_enable_fb) &&
1233 			    (pnode->nlp_flag & NLP_FIRSTBURST)) {
1234 				req_len = lpfc_ncmd->nvmeCmd->payload_length;
1235 				if (req_len < pnode->nvme_fb_size)
1236 					wqe->fcp_iwrite.initial_xfer_len =
1237 						req_len;
1238 				else
1239 					wqe->fcp_iwrite.initial_xfer_len =
1240 						pnode->nvme_fb_size;
1241 			} else {
1242 				wqe->fcp_iwrite.initial_xfer_len = 0;
1243 			}
1244 			cstat->output_requests++;
1245 		} else {
1246 			/* From the iread template, initialize words 7 - 11 */
1247 			memcpy(&wqe->words[7],
1248 			       &lpfc_iread_cmd_template.words[7],
1249 			       sizeof(uint32_t) * 5);
1250 
1251 			/* Word 4 */
1252 			wqe->fcp_iread.total_xfer_len = nCmd->payload_length;
1253 
1254 			/* Word 5 */
1255 			wqe->fcp_iread.rsrvd5 = 0;
1256 
1257 			/* For a CMF Managed port, iod must be zero'ed */
1258 			if (phba->cmf_active_mode == LPFC_CFG_MANAGED)
1259 				bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,
1260 				       LPFC_WQE_IOD_NONE);
1261 			cstat->input_requests++;
1262 		}
1263 	} else {
1264 		/* From the icmnd template, initialize words 4 - 11 */
1265 		memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],
1266 		       sizeof(uint32_t) * 8);
1267 		cstat->control_requests++;
1268 	}
1269 
1270 	if (pnode->nlp_nvme_info & NLP_NVME_NSLER) {
1271 		bf_set(wqe_erp, &wqe->generic.wqe_com, 1);
1272 		sqe = &((struct nvme_fc_cmd_iu *)
1273 			nCmd->cmdaddr)->sqe.common;
1274 		if (sqe->opcode == nvme_admin_async_event)
1275 			bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1);
1276 	}
1277 
1278 	/*
1279 	 * Finish initializing those WQE fields that are independent
1280 	 * of the nvme_cmnd request_buffer
1281 	 */
1282 
1283 	/* Word 3 */
1284 	bf_set(payload_offset_len, &wqe->fcp_icmd,
1285 	       (nCmd->rsplen + nCmd->cmdlen));
1286 
1287 	/* Word 6 */
1288 	bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,
1289 	       phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);
1290 	bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);
1291 
1292 	/* Word 8 */
1293 	wqe->generic.wqe_com.abort_tag = pwqeq->iotag;
1294 
1295 	/* Word 9 */
1296 	bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);
1297 
1298 	/* Word 10 */
1299 	bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);
1300 
1301 	/* Words 13 14 15 are for PBDE support */
1302 
1303 	/* add the VMID tags as per switch response */
1304 	if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {
1305 		if (phba->pport->vmid_priority_tagging) {
1306 			bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);
1307 			bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,
1308 			       lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);
1309 		} else {
1310 			bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);
1311 			bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);
1312 			wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;
1313 		}
1314 	}
1315 
1316 	pwqeq->vport = vport;
1317 	return 0;
1318 }
1319 
1320 
1321 /**
1322  * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1323  * @vport: pointer to a host virtual N_Port data structure
1324  * @lpfc_ncmd: Pointer to lpfc scsi command
1325  *
1326  * Driver registers this routine as it io request handler.  This
1327  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1328  * data structure to the rport indicated in @lpfc_nvme_rport.
1329  *
1330  * Return value :
1331  *   0 - Success
1332  *   TODO: What are the failure codes.
1333  **/
1334 static int
lpfc_nvme_prep_io_dma(struct lpfc_vport * vport,struct lpfc_io_buf * lpfc_ncmd)1335 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,
1336 		      struct lpfc_io_buf *lpfc_ncmd)
1337 {
1338 	struct lpfc_hba *phba = vport->phba;
1339 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;
1340 	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;
1341 	struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;
1342 	struct sli4_hybrid_sgl *sgl_xtra = NULL;
1343 	struct scatterlist *data_sg;
1344 	struct sli4_sge *first_data_sgl;
1345 	struct ulp_bde64 *bde;
1346 	dma_addr_t physaddr = 0;
1347 	uint32_t dma_len = 0;
1348 	uint32_t dma_offset = 0;
1349 	int nseg, i, j;
1350 	bool lsp_just_set = false;
1351 
1352 	/* Fix up the command and response DMA stuff. */
1353 	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);
1354 
1355 	/*
1356 	 * There are three possibilities here - use scatter-gather segment, use
1357 	 * the single mapping, or neither.
1358 	 */
1359 	if (nCmd->sg_cnt) {
1360 		/*
1361 		 * Jump over the cmd and rsp SGEs.  The fix routine
1362 		 * has already adjusted for this.
1363 		 */
1364 		sgl += 2;
1365 
1366 		first_data_sgl = sgl;
1367 		lpfc_ncmd->seg_cnt = nCmd->sg_cnt;
1368 		if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {
1369 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1370 					"6058 Too many sg segments from "
1371 					"NVME Transport.  Max %d, "
1372 					"nvmeIO sg_cnt %d\n",
1373 					phba->cfg_nvme_seg_cnt + 1,
1374 					lpfc_ncmd->seg_cnt);
1375 			lpfc_ncmd->seg_cnt = 0;
1376 			return 1;
1377 		}
1378 
1379 		/*
1380 		 * The driver established a maximum scatter-gather segment count
1381 		 * during probe that limits the number of sg elements in any
1382 		 * single nvme command.  Just run through the seg_cnt and format
1383 		 * the sge's.
1384 		 */
1385 		nseg = nCmd->sg_cnt;
1386 		data_sg = nCmd->first_sgl;
1387 
1388 		/* for tracking the segment boundaries */
1389 		j = 2;
1390 		for (i = 0; i < nseg; i++) {
1391 			if (data_sg == NULL) {
1392 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1393 						"6059 dptr err %d, nseg %d\n",
1394 						i, nseg);
1395 				lpfc_ncmd->seg_cnt = 0;
1396 				return 1;
1397 			}
1398 
1399 			sgl->word2 = 0;
1400 			if (nseg == 1) {
1401 				bf_set(lpfc_sli4_sge_last, sgl, 1);
1402 				bf_set(lpfc_sli4_sge_type, sgl,
1403 				       LPFC_SGE_TYPE_DATA);
1404 			} else {
1405 				bf_set(lpfc_sli4_sge_last, sgl, 0);
1406 
1407 				/* expand the segment */
1408 				if (!lsp_just_set &&
1409 				    !((j + 1) % phba->border_sge_num) &&
1410 				    ((nseg - 1) != i)) {
1411 					/* set LSP type */
1412 					bf_set(lpfc_sli4_sge_type, sgl,
1413 					       LPFC_SGE_TYPE_LSP);
1414 
1415 					sgl_xtra = lpfc_get_sgl_per_hdwq(
1416 							phba, lpfc_ncmd);
1417 
1418 					if (unlikely(!sgl_xtra)) {
1419 						lpfc_ncmd->seg_cnt = 0;
1420 						return 1;
1421 					}
1422 					sgl->addr_lo = cpu_to_le32(putPaddrLow(
1423 						       sgl_xtra->dma_phys_sgl));
1424 					sgl->addr_hi = cpu_to_le32(putPaddrHigh(
1425 						       sgl_xtra->dma_phys_sgl));
1426 
1427 				} else {
1428 					bf_set(lpfc_sli4_sge_type, sgl,
1429 					       LPFC_SGE_TYPE_DATA);
1430 				}
1431 			}
1432 
1433 			if (!(bf_get(lpfc_sli4_sge_type, sgl) &
1434 				     LPFC_SGE_TYPE_LSP)) {
1435 				if ((nseg - 1) == i)
1436 					bf_set(lpfc_sli4_sge_last, sgl, 1);
1437 
1438 				physaddr = sg_dma_address(data_sg);
1439 				dma_len = sg_dma_len(data_sg);
1440 				sgl->addr_lo = cpu_to_le32(
1441 							 putPaddrLow(physaddr));
1442 				sgl->addr_hi = cpu_to_le32(
1443 							putPaddrHigh(physaddr));
1444 
1445 				bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);
1446 				sgl->word2 = cpu_to_le32(sgl->word2);
1447 				sgl->sge_len = cpu_to_le32(dma_len);
1448 
1449 				dma_offset += dma_len;
1450 				data_sg = sg_next(data_sg);
1451 
1452 				sgl++;
1453 
1454 				lsp_just_set = false;
1455 			} else {
1456 				sgl->word2 = cpu_to_le32(sgl->word2);
1457 
1458 				sgl->sge_len = cpu_to_le32(
1459 						     phba->cfg_sg_dma_buf_size);
1460 
1461 				sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;
1462 				i = i - 1;
1463 
1464 				lsp_just_set = true;
1465 			}
1466 
1467 			j++;
1468 		}
1469 
1470 		/* PBDE support for first data SGE only */
1471 		if (nseg == 1 && phba->cfg_enable_pbde) {
1472 			/* Words 13-15 */
1473 			bde = (struct ulp_bde64 *)
1474 				&wqe->words[13];
1475 			bde->addrLow = first_data_sgl->addr_lo;
1476 			bde->addrHigh = first_data_sgl->addr_hi;
1477 			bde->tus.f.bdeSize =
1478 				le32_to_cpu(first_data_sgl->sge_len);
1479 			bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
1480 			bde->tus.w = cpu_to_le32(bde->tus.w);
1481 
1482 			/* Word 11 - set PBDE bit */
1483 			bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);
1484 		} else {
1485 			memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));
1486 			/* Word 11 - PBDE bit disabled by default template */
1487 		}
1488 
1489 	} else {
1490 		lpfc_ncmd->seg_cnt = 0;
1491 
1492 		/* For this clause to be valid, the payload_length
1493 		 * and sg_cnt must zero.
1494 		 */
1495 		if (nCmd->payload_length != 0) {
1496 			lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
1497 					"6063 NVME DMA Prep Err: sg_cnt %d "
1498 					"payload_length x%x\n",
1499 					nCmd->sg_cnt, nCmd->payload_length);
1500 			return 1;
1501 		}
1502 	}
1503 	return 0;
1504 }
1505 
1506 /**
1507  * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1508  * @pnvme_lport: Pointer to the driver's local port data
1509  * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1510  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1511  * @pnvme_fcreq: IO request from nvme fc to driver.
1512  *
1513  * Driver registers this routine as it io request handler.  This
1514  * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1515  * data structure to the rport indicated in @lpfc_nvme_rport.
1516  *
1517  * Return value :
1518  *   0 - Success
1519  *   TODO: What are the failure codes.
1520  **/
1521 static int
lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1522 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,
1523 			struct nvme_fc_remote_port *pnvme_rport,
1524 			void *hw_queue_handle,
1525 			struct nvmefc_fcp_req *pnvme_fcreq)
1526 {
1527 	int ret = 0;
1528 	int expedite = 0;
1529 	int idx, cpu;
1530 	struct lpfc_nvme_lport *lport;
1531 	struct lpfc_fc4_ctrl_stat *cstat;
1532 	struct lpfc_vport *vport;
1533 	struct lpfc_hba *phba;
1534 	struct lpfc_nodelist *ndlp;
1535 	struct lpfc_io_buf *lpfc_ncmd;
1536 	struct lpfc_nvme_rport *rport;
1537 	struct lpfc_nvme_qhandle *lpfc_queue_info;
1538 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1539 	struct nvme_common_command *sqe;
1540 	uint64_t start = 0;
1541 #if (IS_ENABLED(CONFIG_NVME_FC))
1542 	u8 *uuid = NULL;
1543 	int err;
1544 	enum dma_data_direction iodir;
1545 #endif
1546 
1547 	/* Validate pointers. LLDD fault handling with transport does
1548 	 * have timing races.
1549 	 */
1550 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1551 	if (unlikely(!lport)) {
1552 		ret = -EINVAL;
1553 		goto out_fail;
1554 	}
1555 
1556 	vport = lport->vport;
1557 
1558 	if (unlikely(!hw_queue_handle)) {
1559 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1560 				 "6117 Fail IO, NULL hw_queue_handle\n");
1561 		atomic_inc(&lport->xmt_fcp_err);
1562 		ret = -EBUSY;
1563 		goto out_fail;
1564 	}
1565 
1566 	phba = vport->phba;
1567 
1568 	if ((unlikely(vport->load_flag & FC_UNLOADING)) ||
1569 	    phba->hba_flag & HBA_IOQ_FLUSH) {
1570 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1571 				 "6124 Fail IO, Driver unload\n");
1572 		atomic_inc(&lport->xmt_fcp_err);
1573 		ret = -ENODEV;
1574 		goto out_fail;
1575 	}
1576 
1577 	freqpriv = pnvme_fcreq->private;
1578 	if (unlikely(!freqpriv)) {
1579 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1580 				 "6158 Fail IO, NULL request data\n");
1581 		atomic_inc(&lport->xmt_fcp_err);
1582 		ret = -EINVAL;
1583 		goto out_fail;
1584 	}
1585 
1586 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1587 	if (phba->ktime_on)
1588 		start = ktime_get_ns();
1589 #endif
1590 	rport = (struct lpfc_nvme_rport *)pnvme_rport->private;
1591 	lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;
1592 
1593 	/*
1594 	 * Catch race where our node has transitioned, but the
1595 	 * transport is still transitioning.
1596 	 */
1597 	ndlp = rport->ndlp;
1598 	if (!ndlp) {
1599 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1600 				 "6053 Busy IO, ndlp not ready: rport x%px "
1601 				  "ndlp x%px, DID x%06x\n",
1602 				 rport, ndlp, pnvme_rport->port_id);
1603 		atomic_inc(&lport->xmt_fcp_err);
1604 		ret = -EBUSY;
1605 		goto out_fail;
1606 	}
1607 
1608 	/* The remote node has to be a mapped target or it's an error. */
1609 	if ((ndlp->nlp_type & NLP_NVME_TARGET) &&
1610 	    (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {
1611 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,
1612 				 "6036 Fail IO, DID x%06x not ready for "
1613 				 "IO. State x%x, Type x%x Flg x%x\n",
1614 				 pnvme_rport->port_id,
1615 				 ndlp->nlp_state, ndlp->nlp_type,
1616 				 ndlp->fc4_xpt_flags);
1617 		atomic_inc(&lport->xmt_fcp_bad_ndlp);
1618 		ret = -EBUSY;
1619 		goto out_fail;
1620 
1621 	}
1622 
1623 	/* Currently only NVME Keep alive commands should be expedited
1624 	 * if the driver runs out of a resource. These should only be
1625 	 * issued on the admin queue, qidx 0
1626 	 */
1627 	if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {
1628 		sqe = &((struct nvme_fc_cmd_iu *)
1629 			pnvme_fcreq->cmdaddr)->sqe.common;
1630 		if (sqe->opcode == nvme_admin_keep_alive)
1631 			expedite = 1;
1632 	}
1633 
1634 	/* Check if IO qualifies for CMF */
1635 	if (phba->cmf_active_mode != LPFC_CFG_OFF &&
1636 	    pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&
1637 	    pnvme_fcreq->payload_length) {
1638 		ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length);
1639 		if (ret) {
1640 			ret = -EBUSY;
1641 			goto out_fail;
1642 		}
1643 		/* Get start time for IO latency */
1644 		start = ktime_get_ns();
1645 	}
1646 
1647 	/* The node is shared with FCP IO, make sure the IO pending count does
1648 	 * not exceed the programmed depth.
1649 	 */
1650 	if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
1651 		if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&
1652 		    !expedite) {
1653 			lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1654 					 "6174 Fail IO, ndlp qdepth exceeded: "
1655 					 "idx %d DID %x pend %d qdepth %d\n",
1656 					 lpfc_queue_info->index, ndlp->nlp_DID,
1657 					 atomic_read(&ndlp->cmd_pending),
1658 					 ndlp->cmd_qdepth);
1659 			atomic_inc(&lport->xmt_fcp_qdepth);
1660 			ret = -EBUSY;
1661 			goto out_fail1;
1662 		}
1663 	}
1664 
1665 	/* Lookup Hardware Queue index based on fcp_io_sched module parameter */
1666 	if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {
1667 		idx = lpfc_queue_info->index;
1668 	} else {
1669 		cpu = raw_smp_processor_id();
1670 		idx = phba->sli4_hba.cpu_map[cpu].hdwq;
1671 	}
1672 
1673 	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);
1674 	if (lpfc_ncmd == NULL) {
1675 		atomic_inc(&lport->xmt_fcp_noxri);
1676 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1677 				 "6065 Fail IO, driver buffer pool is empty: "
1678 				 "idx %d DID %x\n",
1679 				 lpfc_queue_info->index, ndlp->nlp_DID);
1680 		ret = -EBUSY;
1681 		goto out_fail1;
1682 	}
1683 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1684 	if (start) {
1685 		lpfc_ncmd->ts_cmd_start = start;
1686 		lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;
1687 	} else {
1688 		lpfc_ncmd->ts_cmd_start = 0;
1689 	}
1690 #endif
1691 	lpfc_ncmd->rx_cmd_start = start;
1692 
1693 	/*
1694 	 * Store the data needed by the driver to issue, abort, and complete
1695 	 * an IO.
1696 	 * Do not let the IO hang out forever.  There is no midlayer issuing
1697 	 * an abort so inform the FW of the maximum IO pending time.
1698 	 */
1699 	freqpriv->nvme_buf = lpfc_ncmd;
1700 	lpfc_ncmd->nvmeCmd = pnvme_fcreq;
1701 	lpfc_ncmd->ndlp = ndlp;
1702 	lpfc_ncmd->qidx = lpfc_queue_info->qidx;
1703 
1704 #if (IS_ENABLED(CONFIG_NVME_FC))
1705 	/* check the necessary and sufficient condition to support VMID */
1706 	if (lpfc_is_vmid_enabled(phba) &&
1707 	    (ndlp->vmid_support ||
1708 	     phba->pport->vmid_priority_tagging ==
1709 	     LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {
1710 		/* is the I/O generated by a VM, get the associated virtual */
1711 		/* entity id */
1712 		uuid = nvme_fc_io_getuuid(pnvme_fcreq);
1713 
1714 		if (uuid) {
1715 			if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE)
1716 				iodir = DMA_TO_DEVICE;
1717 			else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ)
1718 				iodir = DMA_FROM_DEVICE;
1719 			else
1720 				iodir = DMA_NONE;
1721 
1722 			err = lpfc_vmid_get_appid(vport, uuid, iodir,
1723 					(union lpfc_vmid_io_tag *)
1724 						&lpfc_ncmd->cur_iocbq.vmid_tag);
1725 			if (!err)
1726 				lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;
1727 		}
1728 	}
1729 #endif
1730 
1731 	/*
1732 	 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1733 	 * This identfier was create in our hardware queue create callback
1734 	 * routine. The driver now is dependent on the IO queue steering from
1735 	 * the transport.  We are trusting the upper NVME layers know which
1736 	 * index to use and that they have affinitized a CPU to this hardware
1737 	 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1738 	 */
1739 	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;
1740 	cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;
1741 
1742 	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);
1743 	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);
1744 	if (ret) {
1745 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1746 				 "6175 Fail IO, Prep DMA: "
1747 				 "idx %d DID %x\n",
1748 				 lpfc_queue_info->index, ndlp->nlp_DID);
1749 		atomic_inc(&lport->xmt_fcp_err);
1750 		ret = -ENOMEM;
1751 		goto out_free_nvme_buf;
1752 	}
1753 
1754 	lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1755 			 lpfc_ncmd->cur_iocbq.sli4_xritag,
1756 			 lpfc_queue_info->index, ndlp->nlp_DID);
1757 
1758 	ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);
1759 	if (ret) {
1760 		atomic_inc(&lport->xmt_fcp_wqerr);
1761 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
1762 				 "6113 Fail IO, Could not issue WQE err %x "
1763 				 "sid: x%x did: x%x oxid: x%x\n",
1764 				 ret, vport->fc_myDID, ndlp->nlp_DID,
1765 				 lpfc_ncmd->cur_iocbq.sli4_xritag);
1766 		goto out_free_nvme_buf;
1767 	}
1768 
1769 	if (phba->cfg_xri_rebalancing)
1770 		lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);
1771 
1772 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1773 	if (lpfc_ncmd->ts_cmd_start)
1774 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();
1775 
1776 	if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {
1777 		cpu = raw_smp_processor_id();
1778 		this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
1779 		lpfc_ncmd->cpu = cpu;
1780 		if (idx != cpu)
1781 			lpfc_printf_vlog(vport,
1782 					 KERN_INFO, LOG_NVME_IOERR,
1783 					"6702 CPU Check cmd: "
1784 					"cpu %d wq %d\n",
1785 					lpfc_ncmd->cpu,
1786 					lpfc_queue_info->index);
1787 	}
1788 #endif
1789 	return 0;
1790 
1791  out_free_nvme_buf:
1792 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {
1793 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)
1794 			cstat->output_requests--;
1795 		else
1796 			cstat->input_requests--;
1797 	} else
1798 		cstat->control_requests--;
1799 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
1800  out_fail1:
1801 	lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,
1802 			     pnvme_fcreq->payload_length, NULL);
1803  out_fail:
1804 	return ret;
1805 }
1806 
1807 /**
1808  * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1809  * @phba: Pointer to HBA context object
1810  * @cmdiocb: Pointer to command iocb object.
1811  * @rspiocb: Pointer to response iocb object.
1812  *
1813  * This is the callback function for any NVME FCP IO that was aborted.
1814  *
1815  * Return value:
1816  *   None
1817  **/
1818 void
lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba * phba,struct lpfc_iocbq * cmdiocb,struct lpfc_iocbq * rspiocb)1819 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,
1820 			   struct lpfc_iocbq *rspiocb)
1821 {
1822 	struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl;
1823 
1824 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
1825 			"6145 ABORT_XRI_CN completing on rpi x%x "
1826 			"original iotag x%x, abort cmd iotag x%x "
1827 			"req_tag x%x, status x%x, hwstatus x%x\n",
1828 			bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com),
1829 			get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag,
1830 			bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),
1831 			bf_get(lpfc_wcqe_c_status, abts_cmpl),
1832 			bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));
1833 	lpfc_sli_release_iocbq(phba, cmdiocb);
1834 }
1835 
1836 /**
1837  * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1838  * @pnvme_lport: Pointer to the driver's local port data
1839  * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1840  * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1841  * @pnvme_fcreq: IO request from nvme fc to driver.
1842  *
1843  * Driver registers this routine as its nvme request io abort handler.  This
1844  * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1845  * data structure to the rport indicated in @lpfc_nvme_rport.  This routine
1846  * is executed asynchronously - one the target is validated as "MAPPED" and
1847  * ready for IO, the driver issues the abort request and returns.
1848  *
1849  * Return value:
1850  *   None
1851  **/
1852 static void
lpfc_nvme_fcp_abort(struct nvme_fc_local_port * pnvme_lport,struct nvme_fc_remote_port * pnvme_rport,void * hw_queue_handle,struct nvmefc_fcp_req * pnvme_fcreq)1853 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,
1854 		    struct nvme_fc_remote_port *pnvme_rport,
1855 		    void *hw_queue_handle,
1856 		    struct nvmefc_fcp_req *pnvme_fcreq)
1857 {
1858 	struct lpfc_nvme_lport *lport;
1859 	struct lpfc_vport *vport;
1860 	struct lpfc_hba *phba;
1861 	struct lpfc_io_buf *lpfc_nbuf;
1862 	struct lpfc_iocbq *nvmereq_wqe;
1863 	struct lpfc_nvme_fcpreq_priv *freqpriv;
1864 	unsigned long flags;
1865 	int ret_val;
1866 
1867 	/* Validate pointers. LLDD fault handling with transport does
1868 	 * have timing races.
1869 	 */
1870 	lport = (struct lpfc_nvme_lport *)pnvme_lport->private;
1871 	if (unlikely(!lport))
1872 		return;
1873 
1874 	vport = lport->vport;
1875 
1876 	if (unlikely(!hw_queue_handle)) {
1877 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1878 				 "6129 Fail Abort, HW Queue Handle NULL.\n");
1879 		return;
1880 	}
1881 
1882 	phba = vport->phba;
1883 	freqpriv = pnvme_fcreq->private;
1884 
1885 	if (unlikely(!freqpriv))
1886 		return;
1887 	if (vport->load_flag & FC_UNLOADING)
1888 		return;
1889 
1890 	/* Announce entry to new IO submit field. */
1891 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1892 			 "6002 Abort Request to rport DID x%06x "
1893 			 "for nvme_fc_req x%px\n",
1894 			 pnvme_rport->port_id,
1895 			 pnvme_fcreq);
1896 
1897 	lpfc_nbuf = freqpriv->nvme_buf;
1898 	if (!lpfc_nbuf) {
1899 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1900 				 "6140 NVME IO req has no matching lpfc nvme "
1901 				 "io buffer.  Skipping abort req.\n");
1902 		return;
1903 	} else if (!lpfc_nbuf->nvmeCmd) {
1904 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1905 				 "6141 lpfc NVME IO req has no nvme_fcreq "
1906 				 "io buffer.  Skipping abort req.\n");
1907 		return;
1908 	}
1909 
1910 	/* Guard against IO completion being called at same time */
1911 	spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags);
1912 
1913 	/* If the hba is getting reset, this flag is set.  It is
1914 	 * cleared when the reset is complete and rings reestablished.
1915 	 */
1916 	spin_lock(&phba->hbalock);
1917 	/* driver queued commands are in process of being flushed */
1918 	if (phba->hba_flag & HBA_IOQ_FLUSH) {
1919 		spin_unlock(&phba->hbalock);
1920 		spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1921 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1922 				 "6139 Driver in reset cleanup - flushing "
1923 				 "NVME Req now.  hba_flag x%x\n",
1924 				 phba->hba_flag);
1925 		return;
1926 	}
1927 
1928 	nvmereq_wqe = &lpfc_nbuf->cur_iocbq;
1929 
1930 	/*
1931 	 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1932 	 * state must match the nvme_fcreq passed by the nvme
1933 	 * transport.  If they don't match, it is likely the driver
1934 	 * has already completed the NVME IO and the nvme transport
1935 	 * has not seen it yet.
1936 	 */
1937 	if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {
1938 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1939 				 "6143 NVME req mismatch: "
1940 				 "lpfc_nbuf x%px nvmeCmd x%px, "
1941 				 "pnvme_fcreq x%px.  Skipping Abort xri x%x\n",
1942 				 lpfc_nbuf, lpfc_nbuf->nvmeCmd,
1943 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1944 		goto out_unlock;
1945 	}
1946 
1947 	/* Don't abort IOs no longer on the pending queue. */
1948 	if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {
1949 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1950 				 "6142 NVME IO req x%px not queued - skipping "
1951 				 "abort req xri x%x\n",
1952 				 pnvme_fcreq, nvmereq_wqe->sli4_xritag);
1953 		goto out_unlock;
1954 	}
1955 
1956 	atomic_inc(&lport->xmt_fcp_abort);
1957 	lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1958 			 nvmereq_wqe->sli4_xritag,
1959 			 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);
1960 
1961 	/* Outstanding abort is in progress */
1962 	if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) {
1963 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1964 				 "6144 Outstanding NVME I/O Abort Request "
1965 				 "still pending on nvme_fcreq x%px, "
1966 				 "lpfc_ncmd x%px xri x%x\n",
1967 				 pnvme_fcreq, lpfc_nbuf,
1968 				 nvmereq_wqe->sli4_xritag);
1969 		goto out_unlock;
1970 	}
1971 
1972 	ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe,
1973 					      lpfc_nvme_abort_fcreq_cmpl);
1974 
1975 	spin_unlock(&phba->hbalock);
1976 	spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1977 
1978 	/* Make sure HBA is alive */
1979 	lpfc_issue_hb_tmo(phba);
1980 
1981 	if (ret_val != WQE_SUCCESS) {
1982 		lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
1983 				 "6137 Failed abts issue_wqe with status x%x "
1984 				 "for nvme_fcreq x%px.\n",
1985 				 ret_val, pnvme_fcreq);
1986 		return;
1987 	}
1988 
1989 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,
1990 			 "6138 Transport Abort NVME Request Issued for "
1991 			 "ox_id x%x\n",
1992 			 nvmereq_wqe->sli4_xritag);
1993 	return;
1994 
1995 out_unlock:
1996 	spin_unlock(&phba->hbalock);
1997 	spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);
1998 	return;
1999 }
2000 
2001 /* Declare and initialization an instance of the FC NVME template. */
2002 static struct nvme_fc_port_template lpfc_nvme_template = {
2003 	/* initiator-based functions */
2004 	.localport_delete  = lpfc_nvme_localport_delete,
2005 	.remoteport_delete = lpfc_nvme_remoteport_delete,
2006 	.create_queue = lpfc_nvme_create_queue,
2007 	.delete_queue = lpfc_nvme_delete_queue,
2008 	.ls_req       = lpfc_nvme_ls_req,
2009 	.fcp_io       = lpfc_nvme_fcp_io_submit,
2010 	.ls_abort     = lpfc_nvme_ls_abort,
2011 	.fcp_abort    = lpfc_nvme_fcp_abort,
2012 	.xmt_ls_rsp   = lpfc_nvme_xmt_ls_rsp,
2013 
2014 	.max_hw_queues = 1,
2015 	.max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2016 	.max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,
2017 	.dma_boundary = 0xFFFFFFFF,
2018 
2019 	/* Sizes of additional private data for data structures.
2020 	 * No use for the last two sizes at this time.
2021 	 */
2022 	.local_priv_sz = sizeof(struct lpfc_nvme_lport),
2023 	.remote_priv_sz = sizeof(struct lpfc_nvme_rport),
2024 	.lsrqst_priv_sz = 0,
2025 	.fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),
2026 };
2027 
2028 /*
2029  * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA
2030  *
2031  * This routine removes a nvme buffer from head of @hdwq io_buf_list
2032  * and returns to caller.
2033  *
2034  * Return codes:
2035  *   NULL - Error
2036  *   Pointer to lpfc_nvme_buf - Success
2037  **/
2038 static struct lpfc_io_buf *
lpfc_get_nvme_buf(struct lpfc_hba * phba,struct lpfc_nodelist * ndlp,int idx,int expedite)2039 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,
2040 		  int idx, int expedite)
2041 {
2042 	struct lpfc_io_buf *lpfc_ncmd;
2043 	struct lpfc_sli4_hdw_queue *qp;
2044 	struct sli4_sge *sgl;
2045 	struct lpfc_iocbq *pwqeq;
2046 	union lpfc_wqe128 *wqe;
2047 
2048 	lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);
2049 
2050 	if (lpfc_ncmd) {
2051 		pwqeq = &(lpfc_ncmd->cur_iocbq);
2052 		wqe = &pwqeq->wqe;
2053 
2054 		/* Setup key fields in buffer that may have been changed
2055 		 * if other protocols used this buffer.
2056 		 */
2057 		pwqeq->cmd_flag = LPFC_IO_NVME;
2058 		pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl;
2059 		lpfc_ncmd->start_time = jiffies;
2060 		lpfc_ncmd->flags = 0;
2061 
2062 		/* Rsp SGE will be filled in when we rcv an IO
2063 		 * from the NVME Layer to be sent.
2064 		 * The cmd is going to be embedded so we need a SKIP SGE.
2065 		 */
2066 		sgl = lpfc_ncmd->dma_sgl;
2067 		bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
2068 		bf_set(lpfc_sli4_sge_last, sgl, 0);
2069 		sgl->word2 = cpu_to_le32(sgl->word2);
2070 		/* Fill in word 3 / sgl_len during cmd submission */
2071 
2072 		/* Initialize 64 bytes only */
2073 		memset(wqe, 0, sizeof(union lpfc_wqe));
2074 
2075 		if (lpfc_ndlp_check_qdepth(phba, ndlp)) {
2076 			atomic_inc(&ndlp->cmd_pending);
2077 			lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;
2078 		}
2079 
2080 	} else {
2081 		qp = &phba->sli4_hba.hdwq[idx];
2082 		qp->empty_io_bufs++;
2083 	}
2084 
2085 	return  lpfc_ncmd;
2086 }
2087 
2088 /**
2089  * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2090  * @phba: The Hba for which this call is being executed.
2091  * @lpfc_ncmd: The nvme buffer which is being released.
2092  *
2093  * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2094  * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2095  * and cannot be reused for at least RA_TOV amount of time if it was
2096  * aborted.
2097  **/
2098 static void
lpfc_release_nvme_buf(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2099 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)
2100 {
2101 	struct lpfc_sli4_hdw_queue *qp;
2102 	unsigned long iflag = 0;
2103 
2104 	if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)
2105 		atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);
2106 
2107 	lpfc_ncmd->ndlp = NULL;
2108 	lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;
2109 
2110 	qp = lpfc_ncmd->hdwq;
2111 	if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {
2112 		lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2113 				"6310 XB release deferred for "
2114 				"ox_id x%x on reqtag x%x\n",
2115 				lpfc_ncmd->cur_iocbq.sli4_xritag,
2116 				lpfc_ncmd->cur_iocbq.iotag);
2117 
2118 		spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);
2119 		list_add_tail(&lpfc_ncmd->list,
2120 			&qp->lpfc_abts_io_buf_list);
2121 		qp->abts_nvme_io_bufs++;
2122 		spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);
2123 	} else
2124 		lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);
2125 }
2126 
2127 /**
2128  * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2129  * @vport: the lpfc_vport instance requesting a localport.
2130  *
2131  * This routine is invoked to create an nvme localport instance to bind
2132  * to the nvme_fc_transport.  It is called once during driver load
2133  * like lpfc_create_shost after all other services are initialized.
2134  * It requires a vport, vpi, and wwns at call time.  Other localport
2135  * parameters are modified as the driver's FCID and the Fabric WWN
2136  * are established.
2137  *
2138  * Return codes
2139  *      0 - successful
2140  *      -ENOMEM - no heap memory available
2141  *      other values - from nvme registration upcall
2142  **/
2143 int
lpfc_nvme_create_localport(struct lpfc_vport * vport)2144 lpfc_nvme_create_localport(struct lpfc_vport *vport)
2145 {
2146 	int ret = 0;
2147 	struct lpfc_hba  *phba = vport->phba;
2148 	struct nvme_fc_port_info nfcp_info;
2149 	struct nvme_fc_local_port *localport;
2150 	struct lpfc_nvme_lport *lport;
2151 
2152 	/* Initialize this localport instance.  The vport wwn usage ensures
2153 	 * that NPIV is accounted for.
2154 	 */
2155 	memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));
2156 	nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;
2157 	nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
2158 	nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
2159 
2160 	/* We need to tell the transport layer + 1 because it takes page
2161 	 * alignment into account. When space for the SGL is allocated we
2162 	 * allocate + 3, one for cmd, one for rsp and one for this alignment
2163 	 */
2164 	lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
2165 
2166 	/* Advertise how many hw queues we support based on cfg_hdw_queue,
2167 	 * which will not exceed cpu count.
2168 	 */
2169 	lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;
2170 
2171 	if (!IS_ENABLED(CONFIG_NVME_FC))
2172 		return ret;
2173 
2174 	/* localport is allocated from the stack, but the registration
2175 	 * call allocates heap memory as well as the private area.
2176 	 */
2177 
2178 	ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,
2179 					 &vport->phba->pcidev->dev, &localport);
2180 	if (!ret) {
2181 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,
2182 				 "6005 Successfully registered local "
2183 				 "NVME port num %d, localP x%px, private "
2184 				 "x%px, sg_seg %d\n",
2185 				 localport->port_num, localport,
2186 				 localport->private,
2187 				 lpfc_nvme_template.max_sgl_segments);
2188 
2189 		/* Private is our lport size declared in the template. */
2190 		lport = (struct lpfc_nvme_lport *)localport->private;
2191 		vport->localport = localport;
2192 		lport->vport = vport;
2193 		vport->nvmei_support = 1;
2194 
2195 		atomic_set(&lport->xmt_fcp_noxri, 0);
2196 		atomic_set(&lport->xmt_fcp_bad_ndlp, 0);
2197 		atomic_set(&lport->xmt_fcp_qdepth, 0);
2198 		atomic_set(&lport->xmt_fcp_err, 0);
2199 		atomic_set(&lport->xmt_fcp_wqerr, 0);
2200 		atomic_set(&lport->xmt_fcp_abort, 0);
2201 		atomic_set(&lport->xmt_ls_abort, 0);
2202 		atomic_set(&lport->xmt_ls_err, 0);
2203 		atomic_set(&lport->cmpl_fcp_xb, 0);
2204 		atomic_set(&lport->cmpl_fcp_err, 0);
2205 		atomic_set(&lport->cmpl_ls_xb, 0);
2206 		atomic_set(&lport->cmpl_ls_err, 0);
2207 
2208 		atomic_set(&lport->fc4NvmeLsRequests, 0);
2209 		atomic_set(&lport->fc4NvmeLsCmpls, 0);
2210 	}
2211 
2212 	return ret;
2213 }
2214 
2215 #if (IS_ENABLED(CONFIG_NVME_FC))
2216 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2217  *
2218  * The driver has to wait for the host nvme transport to callback
2219  * indicating the localport has successfully unregistered all
2220  * resources.  Since this is an uninterruptible wait, loop every ten
2221  * seconds and print a message indicating no progress.
2222  *
2223  * An uninterruptible wait is used because of the risk of transport-to-
2224  * driver state mismatch.
2225  */
2226 static void
lpfc_nvme_lport_unreg_wait(struct lpfc_vport * vport,struct lpfc_nvme_lport * lport,struct completion * lport_unreg_cmp)2227 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,
2228 			   struct lpfc_nvme_lport *lport,
2229 			   struct completion *lport_unreg_cmp)
2230 {
2231 	u32 wait_tmo;
2232 	int ret, i, pending = 0;
2233 	struct lpfc_sli_ring  *pring;
2234 	struct lpfc_hba  *phba = vport->phba;
2235 	struct lpfc_sli4_hdw_queue *qp;
2236 	int abts_scsi, abts_nvme;
2237 
2238 	/* Host transport has to clean up and confirm requiring an indefinite
2239 	 * wait. Print a message if a 10 second wait expires and renew the
2240 	 * wait. This is unexpected.
2241 	 */
2242 	wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);
2243 	while (true) {
2244 		ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);
2245 		if (unlikely(!ret)) {
2246 			pending = 0;
2247 			abts_scsi = 0;
2248 			abts_nvme = 0;
2249 			for (i = 0; i < phba->cfg_hdw_queue; i++) {
2250 				qp = &phba->sli4_hba.hdwq[i];
2251 				if (!vport->localport || !qp || !qp->io_wq)
2252 					return;
2253 
2254 				pring = qp->io_wq->pring;
2255 				if (!pring)
2256 					continue;
2257 				pending += pring->txcmplq_cnt;
2258 				abts_scsi += qp->abts_scsi_io_bufs;
2259 				abts_nvme += qp->abts_nvme_io_bufs;
2260 			}
2261 			if (!vport->localport ||
2262 			    test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||
2263 			    phba->link_state == LPFC_HBA_ERROR ||
2264 			    vport->load_flag & FC_UNLOADING)
2265 				return;
2266 
2267 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2268 					 "6176 Lport x%px Localport x%px wait "
2269 					 "timed out. Pending %d [%d:%d]. "
2270 					 "Renewing.\n",
2271 					 lport, vport->localport, pending,
2272 					 abts_scsi, abts_nvme);
2273 			continue;
2274 		}
2275 		break;
2276 	}
2277 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,
2278 			 "6177 Lport x%px Localport x%px Complete Success\n",
2279 			 lport, vport->localport);
2280 }
2281 #endif
2282 
2283 /**
2284  * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2285  * @vport: pointer to a host virtual N_Port data structure
2286  *
2287  * This routine is invoked to destroy all lports bound to the phba.
2288  * The lport memory was allocated by the nvme fc transport and is
2289  * released there.  This routine ensures all rports bound to the
2290  * lport have been disconnected.
2291  *
2292  **/
2293 void
lpfc_nvme_destroy_localport(struct lpfc_vport * vport)2294 lpfc_nvme_destroy_localport(struct lpfc_vport *vport)
2295 {
2296 #if (IS_ENABLED(CONFIG_NVME_FC))
2297 	struct nvme_fc_local_port *localport;
2298 	struct lpfc_nvme_lport *lport;
2299 	int ret;
2300 	DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);
2301 
2302 	if (vport->nvmei_support == 0)
2303 		return;
2304 
2305 	localport = vport->localport;
2306 	if (!localport)
2307 		return;
2308 	lport = (struct lpfc_nvme_lport *)localport->private;
2309 
2310 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2311 			 "6011 Destroying NVME localport x%px\n",
2312 			 localport);
2313 
2314 	/* lport's rport list is clear.  Unregister
2315 	 * lport and release resources.
2316 	 */
2317 	lport->lport_unreg_cmp = &lport_unreg_cmp;
2318 	ret = nvme_fc_unregister_localport(localport);
2319 
2320 	/* Wait for completion.  This either blocks
2321 	 * indefinitely or succeeds
2322 	 */
2323 	lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);
2324 	vport->localport = NULL;
2325 
2326 	/* Regardless of the unregister upcall response, clear
2327 	 * nvmei_support.  All rports are unregistered and the
2328 	 * driver will clean up.
2329 	 */
2330 	vport->nvmei_support = 0;
2331 	if (ret == 0) {
2332 		lpfc_printf_vlog(vport,
2333 				 KERN_INFO, LOG_NVME_DISC,
2334 				 "6009 Unregistered lport Success\n");
2335 	} else {
2336 		lpfc_printf_vlog(vport,
2337 				 KERN_INFO, LOG_NVME_DISC,
2338 				 "6010 Unregistered lport "
2339 				 "Failed, status x%x\n",
2340 				 ret);
2341 	}
2342 #endif
2343 }
2344 
2345 void
lpfc_nvme_update_localport(struct lpfc_vport * vport)2346 lpfc_nvme_update_localport(struct lpfc_vport *vport)
2347 {
2348 #if (IS_ENABLED(CONFIG_NVME_FC))
2349 	struct nvme_fc_local_port *localport;
2350 	struct lpfc_nvme_lport *lport;
2351 
2352 	localport = vport->localport;
2353 	if (!localport) {
2354 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2355 				 "6710 Update NVME fail. No localport\n");
2356 		return;
2357 	}
2358 	lport = (struct lpfc_nvme_lport *)localport->private;
2359 	if (!lport) {
2360 		lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,
2361 				 "6171 Update NVME fail. localP x%px, No lport\n",
2362 				 localport);
2363 		return;
2364 	}
2365 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
2366 			 "6012 Update NVME lport x%px did x%x\n",
2367 			 localport, vport->fc_myDID);
2368 
2369 	localport->port_id = vport->fc_myDID;
2370 	if (localport->port_id == 0)
2371 		localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;
2372 	else
2373 		localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;
2374 
2375 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2376 			 "6030 bound lport x%px to DID x%06x\n",
2377 			 lport, localport->port_id);
2378 #endif
2379 }
2380 
2381 int
lpfc_nvme_register_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2382 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2383 {
2384 #if (IS_ENABLED(CONFIG_NVME_FC))
2385 	int ret = 0;
2386 	struct nvme_fc_local_port *localport;
2387 	struct lpfc_nvme_lport *lport;
2388 	struct lpfc_nvme_rport *rport;
2389 	struct lpfc_nvme_rport *oldrport;
2390 	struct nvme_fc_remote_port *remote_port;
2391 	struct nvme_fc_port_info rpinfo;
2392 	struct lpfc_nodelist *prev_ndlp = NULL;
2393 	struct fc_rport *srport = ndlp->rport;
2394 
2395 	lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,
2396 			 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2397 			 ndlp->nlp_DID, ndlp->nlp_type);
2398 
2399 	localport = vport->localport;
2400 	if (!localport)
2401 		return 0;
2402 
2403 	lport = (struct lpfc_nvme_lport *)localport->private;
2404 
2405 	/* NVME rports are not preserved across devloss.
2406 	 * Just register this instance.  Note, rpinfo->dev_loss_tmo
2407 	 * is left 0 to indicate accept transport defaults.  The
2408 	 * driver communicates port role capabilities consistent
2409 	 * with the PRLI response data.
2410 	 */
2411 	memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));
2412 	rpinfo.port_id = ndlp->nlp_DID;
2413 	if (ndlp->nlp_type & NLP_NVME_TARGET)
2414 		rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;
2415 	if (ndlp->nlp_type & NLP_NVME_INITIATOR)
2416 		rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;
2417 
2418 	if (ndlp->nlp_type & NLP_NVME_DISCOVERY)
2419 		rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;
2420 
2421 	rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);
2422 	rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);
2423 	if (srport)
2424 		rpinfo.dev_loss_tmo = srport->dev_loss_tmo;
2425 	else
2426 		rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;
2427 
2428 	spin_lock_irq(&ndlp->lock);
2429 
2430 	/* If an oldrport exists, so does the ndlp reference.  If not
2431 	 * a new reference is needed because either the node has never
2432 	 * been registered or it's been unregistered and getting deleted.
2433 	 */
2434 	oldrport = lpfc_ndlp_get_nrport(ndlp);
2435 	if (oldrport) {
2436 		prev_ndlp = oldrport->ndlp;
2437 		spin_unlock_irq(&ndlp->lock);
2438 	} else {
2439 		spin_unlock_irq(&ndlp->lock);
2440 		if (!lpfc_nlp_get(ndlp)) {
2441 			dev_warn(&vport->phba->pcidev->dev,
2442 				 "Warning - No node ref - exit register\n");
2443 			return 0;
2444 		}
2445 	}
2446 
2447 	ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);
2448 	if (!ret) {
2449 		/* If the ndlp already has an nrport, this is just
2450 		 * a resume of the existing rport.  Else this is a
2451 		 * new rport.
2452 		 */
2453 		/* Guard against an unregister/reregister
2454 		 * race that leaves the WAIT flag set.
2455 		 */
2456 		spin_lock_irq(&ndlp->lock);
2457 		ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2458 		ndlp->fc4_xpt_flags |= NVME_XPT_REGD;
2459 		spin_unlock_irq(&ndlp->lock);
2460 		rport = remote_port->private;
2461 		if (oldrport) {
2462 
2463 			/* Sever the ndlp<->rport association
2464 			 * before dropping the ndlp ref from
2465 			 * register.
2466 			 */
2467 			spin_lock_irq(&ndlp->lock);
2468 			ndlp->nrport = NULL;
2469 			ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;
2470 			spin_unlock_irq(&ndlp->lock);
2471 			rport->ndlp = NULL;
2472 			rport->remoteport = NULL;
2473 
2474 			/* Reference only removed if previous NDLP is no longer
2475 			 * active. It might be just a swap and removing the
2476 			 * reference would cause a premature cleanup.
2477 			 */
2478 			if (prev_ndlp && prev_ndlp != ndlp) {
2479 				if (!prev_ndlp->nrport)
2480 					lpfc_nlp_put(prev_ndlp);
2481 			}
2482 		}
2483 
2484 		/* Clean bind the rport to the ndlp. */
2485 		rport->remoteport = remote_port;
2486 		rport->lport = lport;
2487 		rport->ndlp = ndlp;
2488 		spin_lock_irq(&ndlp->lock);
2489 		ndlp->nrport = rport;
2490 		spin_unlock_irq(&ndlp->lock);
2491 		lpfc_printf_vlog(vport, KERN_INFO,
2492 				 LOG_NVME_DISC | LOG_NODE,
2493 				 "6022 Bind lport x%px to remoteport x%px "
2494 				 "rport x%px WWNN 0x%llx, "
2495 				 "Rport WWPN 0x%llx DID "
2496 				 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",
2497 				 lport, remote_port, rport,
2498 				 rpinfo.node_name, rpinfo.port_name,
2499 				 rpinfo.port_id, rpinfo.port_role,
2500 				 ndlp, prev_ndlp);
2501 	} else {
2502 		lpfc_printf_vlog(vport, KERN_ERR,
2503 				 LOG_TRACE_EVENT,
2504 				 "6031 RemotePort Registration failed "
2505 				 "err: %d, DID x%06x ref %u\n",
2506 				 ret, ndlp->nlp_DID, kref_read(&ndlp->kref));
2507 		lpfc_nlp_put(ndlp);
2508 	}
2509 
2510 	return ret;
2511 #else
2512 	return 0;
2513 #endif
2514 }
2515 
2516 /*
2517  * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport
2518  *
2519  * If the ndlp represents an NVME Target, that we are logged into,
2520  * ping the NVME FC Transport layer to initiate a device rescan
2521  * on this remote NPort.
2522  */
2523 void
lpfc_nvme_rescan_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2524 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2525 {
2526 #if (IS_ENABLED(CONFIG_NVME_FC))
2527 	struct lpfc_nvme_rport *nrport;
2528 	struct nvme_fc_remote_port *remoteport = NULL;
2529 
2530 	spin_lock_irq(&ndlp->lock);
2531 	nrport = lpfc_ndlp_get_nrport(ndlp);
2532 	if (nrport)
2533 		remoteport = nrport->remoteport;
2534 	spin_unlock_irq(&ndlp->lock);
2535 
2536 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2537 			 "6170 Rescan NPort DID x%06x type x%x "
2538 			 "state x%x nrport x%px remoteport x%px\n",
2539 			 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,
2540 			 nrport, remoteport);
2541 
2542 	if (!nrport || !remoteport)
2543 		goto rescan_exit;
2544 
2545 	/* Rescan an NVME target in MAPPED state with DISCOVERY role set */
2546 	if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&
2547 	    ndlp->nlp_state == NLP_STE_MAPPED_NODE) {
2548 		nvme_fc_rescan_remoteport(remoteport);
2549 
2550 		lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2551 				 "6172 NVME rescanned DID x%06x "
2552 				 "port_state x%x\n",
2553 				 ndlp->nlp_DID, remoteport->port_state);
2554 	}
2555 	return;
2556  rescan_exit:
2557 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2558 			 "6169 Skip NVME Rport Rescan, NVME remoteport "
2559 			 "unregistered\n");
2560 #endif
2561 }
2562 
2563 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2564  *
2565  * There is no notion of Devloss or rport recovery from the current
2566  * nvme_transport perspective.  Loss of an rport just means IO cannot
2567  * be sent and recovery is completely up to the initator.
2568  * For now, the driver just unbinds the DID and port_role so that
2569  * no further IO can be issued.
2570  */
2571 void
lpfc_nvme_unregister_port(struct lpfc_vport * vport,struct lpfc_nodelist * ndlp)2572 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)
2573 {
2574 #if (IS_ENABLED(CONFIG_NVME_FC))
2575 	int ret;
2576 	struct nvme_fc_local_port *localport;
2577 	struct lpfc_nvme_lport *lport;
2578 	struct lpfc_nvme_rport *rport;
2579 	struct nvme_fc_remote_port *remoteport = NULL;
2580 
2581 	localport = vport->localport;
2582 
2583 	/* This is fundamental error.  The localport is always
2584 	 * available until driver unload.  Just exit.
2585 	 */
2586 	if (!localport)
2587 		return;
2588 
2589 	lport = (struct lpfc_nvme_lport *)localport->private;
2590 	if (!lport)
2591 		goto input_err;
2592 
2593 	spin_lock_irq(&ndlp->lock);
2594 	rport = lpfc_ndlp_get_nrport(ndlp);
2595 	if (rport)
2596 		remoteport = rport->remoteport;
2597 	spin_unlock_irq(&ndlp->lock);
2598 	if (!remoteport)
2599 		goto input_err;
2600 
2601 	lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,
2602 			 "6033 Unreg nvme remoteport x%px, portname x%llx, "
2603 			 "port_id x%06x, portstate x%x port type x%x "
2604 			 "refcnt %d\n",
2605 			 remoteport, remoteport->port_name,
2606 			 remoteport->port_id, remoteport->port_state,
2607 			 ndlp->nlp_type, kref_read(&ndlp->kref));
2608 
2609 	/* Sanity check ndlp type.  Only call for NVME ports. Don't
2610 	 * clear any rport state until the transport calls back.
2611 	 */
2612 
2613 	if (ndlp->nlp_type & NLP_NVME_TARGET) {
2614 		/* No concern about the role change on the nvme remoteport.
2615 		 * The transport will update it.
2616 		 */
2617 		spin_lock_irq(&ndlp->lock);
2618 		ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;
2619 		spin_unlock_irq(&ndlp->lock);
2620 
2621 		/* Don't let the host nvme transport keep sending keep-alives
2622 		 * on this remoteport. Vport is unloading, no recovery. The
2623 		 * return values is ignored.  The upcall is a courtesy to the
2624 		 * transport.
2625 		 */
2626 		if (vport->load_flag & FC_UNLOADING ||
2627 		    unlikely(vport->phba->link_state == LPFC_HBA_ERROR))
2628 			(void)nvme_fc_set_remoteport_devloss(remoteport, 0);
2629 
2630 		ret = nvme_fc_unregister_remoteport(remoteport);
2631 
2632 		/* The driver no longer knows if the nrport memory is valid.
2633 		 * because the controller teardown process has begun and
2634 		 * is asynchronous.  Break the binding in the ndlp. Also
2635 		 * remove the register ndlp reference to setup node release.
2636 		 */
2637 		ndlp->nrport = NULL;
2638 		lpfc_nlp_put(ndlp);
2639 		if (ret != 0) {
2640 			lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2641 					 "6167 NVME unregister failed %d "
2642 					 "port_state x%x\n",
2643 					 ret, remoteport->port_state);
2644 
2645 			if (vport->load_flag & FC_UNLOADING) {
2646 				/* Only 1 thread can drop the initial node
2647 				 * reference. Check if another thread has set
2648 				 * NLP_DROPPED.
2649 				 */
2650 				spin_lock_irq(&ndlp->lock);
2651 				if (!(ndlp->nlp_flag & NLP_DROPPED)) {
2652 					ndlp->nlp_flag |= NLP_DROPPED;
2653 					spin_unlock_irq(&ndlp->lock);
2654 					lpfc_nlp_put(ndlp);
2655 					return;
2656 				}
2657 				spin_unlock_irq(&ndlp->lock);
2658 			}
2659 		}
2660 	}
2661 	return;
2662 
2663  input_err:
2664 #endif
2665 	lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,
2666 			 "6168 State error: lport x%px, rport x%px FCID x%06x\n",
2667 			 vport->localport, ndlp->rport, ndlp->nlp_DID);
2668 }
2669 
2670 /**
2671  * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort
2672  * @phba: pointer to lpfc hba data structure.
2673  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2674  *
2675  * This routine is invoked by the worker thread to process a SLI4 fast-path
2676  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2677  * here.
2678  **/
2679 void
lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba * phba,struct lpfc_io_buf * lpfc_ncmd)2680 lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba,
2681 				   struct lpfc_io_buf *lpfc_ncmd)
2682 {
2683 	struct nvmefc_fcp_req *nvme_cmd = NULL;
2684 
2685 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2686 			"6533 %s nvme_cmd %p tag x%x abort complete and "
2687 			"xri released\n", __func__,
2688 			lpfc_ncmd->nvmeCmd,
2689 			lpfc_ncmd->cur_iocbq.iotag);
2690 
2691 	/* Aborted NVME commands are required to not complete
2692 	 * before the abort exchange command fully completes.
2693 	 * Once completed, it is available via the put list.
2694 	 */
2695 	if (lpfc_ncmd->nvmeCmd) {
2696 		nvme_cmd = lpfc_ncmd->nvmeCmd;
2697 		nvme_cmd->transferred_length = 0;
2698 		nvme_cmd->rcv_rsplen = 0;
2699 		nvme_cmd->status = NVME_SC_INTERNAL;
2700 		nvme_cmd->done(nvme_cmd);
2701 		lpfc_ncmd->nvmeCmd = NULL;
2702 	}
2703 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2704 }
2705 
2706 /**
2707  * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2708  * @phba: pointer to lpfc hba data structure.
2709  * @axri: pointer to the fcp xri abort wcqe structure.
2710  * @lpfc_ncmd: The nvme job structure for the request being aborted.
2711  *
2712  * This routine is invoked by the worker thread to process a SLI4 fast-path
2713  * NVME aborted xri.  Aborted NVME IO commands are completed to the transport
2714  * here.
2715  **/
2716 void
lpfc_sli4_nvme_xri_aborted(struct lpfc_hba * phba,struct sli4_wcqe_xri_aborted * axri,struct lpfc_io_buf * lpfc_ncmd)2717 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,
2718 			   struct sli4_wcqe_xri_aborted *axri,
2719 			   struct lpfc_io_buf *lpfc_ncmd)
2720 {
2721 	uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
2722 	struct nvmefc_fcp_req *nvme_cmd = NULL;
2723 	struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;
2724 
2725 
2726 	if (ndlp)
2727 		lpfc_sli4_abts_err_handler(phba, ndlp, axri);
2728 
2729 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
2730 			"6311 nvme_cmd %p xri x%x tag x%x abort complete and "
2731 			"xri released\n",
2732 			lpfc_ncmd->nvmeCmd, xri,
2733 			lpfc_ncmd->cur_iocbq.iotag);
2734 
2735 	/* Aborted NVME commands are required to not complete
2736 	 * before the abort exchange command fully completes.
2737 	 * Once completed, it is available via the put list.
2738 	 */
2739 	if (lpfc_ncmd->nvmeCmd) {
2740 		nvme_cmd = lpfc_ncmd->nvmeCmd;
2741 		nvme_cmd->done(nvme_cmd);
2742 		lpfc_ncmd->nvmeCmd = NULL;
2743 	}
2744 	lpfc_release_nvme_buf(phba, lpfc_ncmd);
2745 }
2746 
2747 /**
2748  * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2749  * @phba: Pointer to HBA context object.
2750  *
2751  * This function flushes all wqes in the nvme rings and frees all resources
2752  * in the txcmplq. This function does not issue abort wqes for the IO
2753  * commands in txcmplq, they will just be returned with
2754  * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2755  * slot has been permanently disabled.
2756  **/
2757 void
lpfc_nvme_wait_for_io_drain(struct lpfc_hba * phba)2758 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)
2759 {
2760 	struct lpfc_sli_ring  *pring;
2761 	u32 i, wait_cnt = 0;
2762 
2763 	if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)
2764 		return;
2765 
2766 	/* Cycle through all IO rings and make sure all outstanding
2767 	 * WQEs have been removed from the txcmplqs.
2768 	 */
2769 	for (i = 0; i < phba->cfg_hdw_queue; i++) {
2770 		if (!phba->sli4_hba.hdwq[i].io_wq)
2771 			continue;
2772 		pring = phba->sli4_hba.hdwq[i].io_wq->pring;
2773 
2774 		if (!pring)
2775 			continue;
2776 
2777 		/* Retrieve everything on the txcmplq */
2778 		while (!list_empty(&pring->txcmplq)) {
2779 			msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);
2780 			wait_cnt++;
2781 
2782 			/* The sleep is 10mS.  Every ten seconds,
2783 			 * dump a message.  Something is wrong.
2784 			 */
2785 			if ((wait_cnt % 1000) == 0) {
2786 				lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
2787 						"6178 NVME IO not empty, "
2788 						"cnt %d\n", wait_cnt);
2789 			}
2790 		}
2791 	}
2792 
2793 	/* Make sure HBA is alive */
2794 	lpfc_issue_hb_tmo(phba);
2795 
2796 }
2797 
2798 void
lpfc_nvme_cancel_iocb(struct lpfc_hba * phba,struct lpfc_iocbq * pwqeIn,uint32_t stat,uint32_t param)2799 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,
2800 		      uint32_t stat, uint32_t param)
2801 {
2802 #if (IS_ENABLED(CONFIG_NVME_FC))
2803 	struct lpfc_io_buf *lpfc_ncmd;
2804 	struct nvmefc_fcp_req *nCmd;
2805 	struct lpfc_wcqe_complete wcqe;
2806 	struct lpfc_wcqe_complete *wcqep = &wcqe;
2807 
2808 	lpfc_ncmd = pwqeIn->io_buf;
2809 	if (!lpfc_ncmd) {
2810 		lpfc_sli_release_iocbq(phba, pwqeIn);
2811 		return;
2812 	}
2813 	/* For abort iocb just return, IO iocb will do a done call */
2814 	if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==
2815 	    CMD_ABORT_XRI_CX) {
2816 		lpfc_sli_release_iocbq(phba, pwqeIn);
2817 		return;
2818 	}
2819 
2820 	spin_lock(&lpfc_ncmd->buf_lock);
2821 	nCmd = lpfc_ncmd->nvmeCmd;
2822 	if (!nCmd) {
2823 		spin_unlock(&lpfc_ncmd->buf_lock);
2824 		lpfc_release_nvme_buf(phba, lpfc_ncmd);
2825 		return;
2826 	}
2827 	spin_unlock(&lpfc_ncmd->buf_lock);
2828 
2829 	lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
2830 			"6194 NVME Cancel xri %x\n",
2831 			lpfc_ncmd->cur_iocbq.sli4_xritag);
2832 
2833 	wcqep->word0 = 0;
2834 	bf_set(lpfc_wcqe_c_status, wcqep, stat);
2835 	wcqep->parameter = param;
2836 	wcqep->total_data_placed = 0;
2837 	wcqep->word3 = 0; /* xb is 0 */
2838 
2839 	/* Call release with XB=1 to queue the IO into the abort list. */
2840 	if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)
2841 		bf_set(lpfc_wcqe_c_xb, wcqep, 1);
2842 
2843 	memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep));
2844 	(pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn);
2845 #endif
2846 }
2847