1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include <linux/nvme.h> 40 #include <linux/nvme-fc-driver.h> 41 #include <linux/nvme-fc.h> 42 #include "lpfc_version.h" 43 #include "lpfc_hw4.h" 44 #include "lpfc_hw.h" 45 #include "lpfc_sli.h" 46 #include "lpfc_sli4.h" 47 #include "lpfc_nl.h" 48 #include "lpfc_disc.h" 49 #include "lpfc.h" 50 #include "lpfc_nvme.h" 51 #include "lpfc_scsi.h" 52 #include "lpfc_logmsg.h" 53 #include "lpfc_crtn.h" 54 #include "lpfc_vport.h" 55 #include "lpfc_debugfs.h" 56 57 /* NVME initiator-based functions */ 58 59 static struct lpfc_io_buf * 60 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 61 int idx, int expedite); 62 63 static void 64 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 65 66 static struct nvme_fc_port_template lpfc_nvme_template; 67 68 static union lpfc_wqe128 lpfc_iread_cmd_template; 69 static union lpfc_wqe128 lpfc_iwrite_cmd_template; 70 static union lpfc_wqe128 lpfc_icmnd_cmd_template; 71 72 /* Setup WQE templates for NVME IOs */ 73 void 74 lpfc_nvme_cmd_template(void) 75 { 76 union lpfc_wqe128 *wqe; 77 78 /* IREAD template */ 79 wqe = &lpfc_iread_cmd_template; 80 memset(wqe, 0, sizeof(union lpfc_wqe128)); 81 82 /* Word 0, 1, 2 - BDE is variable */ 83 84 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 85 86 /* Word 4 - total_xfer_len is variable */ 87 88 /* Word 5 - is zero */ 89 90 /* Word 6 - ctxt_tag, xri_tag is variable */ 91 92 /* Word 7 */ 93 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); 94 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); 95 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); 96 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); 97 98 /* Word 8 - abort_tag is variable */ 99 100 /* Word 9 - reqtag is variable */ 101 102 /* Word 10 - dbde, wqes is variable */ 103 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 104 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); 105 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); 106 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); 107 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); 108 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); 109 110 /* Word 11 - pbde is variable */ 111 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); 112 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 113 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); 114 115 /* Word 12 - is zero */ 116 117 /* Word 13, 14, 15 - PBDE is variable */ 118 119 /* IWRITE template */ 120 wqe = &lpfc_iwrite_cmd_template; 121 memset(wqe, 0, sizeof(union lpfc_wqe128)); 122 123 /* Word 0, 1, 2 - BDE is variable */ 124 125 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 126 127 /* Word 4 - total_xfer_len is variable */ 128 129 /* Word 5 - initial_xfer_len is variable */ 130 131 /* Word 6 - ctxt_tag, xri_tag is variable */ 132 133 /* Word 7 */ 134 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); 135 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); 136 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); 137 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); 138 139 /* Word 8 - abort_tag is variable */ 140 141 /* Word 9 - reqtag is variable */ 142 143 /* Word 10 - dbde, wqes is variable */ 144 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 145 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); 146 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); 147 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); 148 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); 149 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 150 151 /* Word 11 - pbde is variable */ 152 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); 153 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 154 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); 155 156 /* Word 12 - is zero */ 157 158 /* Word 13, 14, 15 - PBDE is variable */ 159 160 /* ICMND template */ 161 wqe = &lpfc_icmnd_cmd_template; 162 memset(wqe, 0, sizeof(union lpfc_wqe128)); 163 164 /* Word 0, 1, 2 - BDE is variable */ 165 166 /* Word 3 - payload_offset_len is variable */ 167 168 /* Word 4, 5 - is zero */ 169 170 /* Word 6 - ctxt_tag, xri_tag is variable */ 171 172 /* Word 7 */ 173 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); 174 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); 175 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); 176 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); 177 178 /* Word 8 - abort_tag is variable */ 179 180 /* Word 9 - reqtag is variable */ 181 182 /* Word 10 - dbde, wqes is variable */ 183 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 184 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 185 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); 186 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); 187 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); 188 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 189 190 /* Word 11 */ 191 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); 192 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 193 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); 194 195 /* Word 12, 13, 14, 15 - is zero */ 196 } 197 198 /** 199 * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry. 200 * @pwqeq: Pointer to command iocb. 201 * @xritag: Tag that uniqely identifies the local exchange resource. 202 * @opt: Option bits - 203 * bit 0 = inhibit sending abts on the link 204 * 205 * This function is called with hbalock held. 206 **/ 207 void 208 lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt) 209 { 210 union lpfc_wqe128 *wqe = &pwqeq->wqe; 211 212 /* WQEs are reused. Clear stale data and set key fields to 213 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 214 */ 215 memset(wqe, 0, sizeof(*wqe)); 216 217 if (opt & INHIBIT_ABORT) 218 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); 219 /* Abort specified xri tag, with the mask deliberately zeroed */ 220 bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); 221 222 bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 223 224 /* Abort the IO associated with this outstanding exchange ID. */ 225 wqe->abort_cmd.wqe_com.abort_tag = xritag; 226 227 /* iotag for the wqe completion. */ 228 bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag); 229 230 bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); 231 bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 232 233 bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND); 234 bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1); 235 bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 236 } 237 238 /** 239 * lpfc_nvme_create_queue - 240 * @lpfc_pnvme: Pointer to the driver's nvme instance data 241 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 242 * @handle: An opaque driver handle used in follow-up calls. 243 * 244 * Driver registers this routine to preallocate and initialize any 245 * internal data structures to bind the @qidx to its internal IO queues. 246 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 247 * 248 * Return value : 249 * 0 - Success 250 * -EINVAL - Unsupported input value. 251 * -ENOMEM - Could not alloc necessary memory 252 **/ 253 static int 254 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 255 unsigned int qidx, u16 qsize, 256 void **handle) 257 { 258 struct lpfc_nvme_lport *lport; 259 struct lpfc_vport *vport; 260 struct lpfc_nvme_qhandle *qhandle; 261 char *str; 262 263 if (!pnvme_lport->private) 264 return -ENOMEM; 265 266 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 267 vport = lport->vport; 268 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 269 if (qhandle == NULL) 270 return -ENOMEM; 271 272 qhandle->cpu_id = raw_smp_processor_id(); 273 qhandle->qidx = qidx; 274 /* 275 * NVME qidx == 0 is the admin queue, so both admin queue 276 * and first IO queue will use MSI-X vector and associated 277 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 278 */ 279 if (qidx) { 280 str = "IO "; /* IO queue */ 281 qhandle->index = ((qidx - 1) % 282 lpfc_nvme_template.max_hw_queues); 283 } else { 284 str = "ADM"; /* Admin queue */ 285 qhandle->index = qidx; 286 } 287 288 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 289 "6073 Binding %s HdwQueue %d (cpu %d) to " 290 "hdw_queue %d qhandle x%px\n", str, 291 qidx, qhandle->cpu_id, qhandle->index, qhandle); 292 *handle = (void *)qhandle; 293 return 0; 294 } 295 296 /** 297 * lpfc_nvme_delete_queue - 298 * @lpfc_pnvme: Pointer to the driver's nvme instance data 299 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 300 * @handle: An opaque driver handle from lpfc_nvme_create_queue 301 * 302 * Driver registers this routine to free 303 * any internal data structures to bind the @qidx to its internal 304 * IO queues. 305 * 306 * Return value : 307 * 0 - Success 308 * TODO: What are the failure codes. 309 **/ 310 static void 311 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 312 unsigned int qidx, 313 void *handle) 314 { 315 struct lpfc_nvme_lport *lport; 316 struct lpfc_vport *vport; 317 318 if (!pnvme_lport->private) 319 return; 320 321 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 322 vport = lport->vport; 323 324 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 325 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 326 lport, qidx, handle); 327 kfree(handle); 328 } 329 330 static void 331 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 332 { 333 struct lpfc_nvme_lport *lport = localport->private; 334 335 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 336 "6173 localport x%px delete complete\n", 337 lport); 338 339 /* release any threads waiting for the unreg to complete */ 340 if (lport->vport->localport) 341 complete(lport->lport_unreg_cmp); 342 } 343 344 /* lpfc_nvme_remoteport_delete 345 * 346 * @remoteport: Pointer to an nvme transport remoteport instance. 347 * 348 * This is a template downcall. NVME transport calls this function 349 * when it has completed the unregistration of a previously 350 * registered remoteport. 351 * 352 * Return value : 353 * None 354 */ 355 static void 356 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 357 { 358 struct lpfc_nvme_rport *rport = remoteport->private; 359 struct lpfc_vport *vport; 360 struct lpfc_nodelist *ndlp; 361 362 ndlp = rport->ndlp; 363 if (!ndlp) 364 goto rport_err; 365 366 vport = ndlp->vport; 367 if (!vport) 368 goto rport_err; 369 370 /* Remove this rport from the lport's list - memory is owned by the 371 * transport. Remove the ndlp reference for the NVME transport before 372 * calling state machine to remove the node. 373 */ 374 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 375 "6146 remoteport delete of remoteport x%px\n", 376 remoteport); 377 spin_lock_irq(&vport->phba->hbalock); 378 379 /* The register rebind might have occurred before the delete 380 * downcall. Guard against this race. 381 */ 382 if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) { 383 ndlp->nrport = NULL; 384 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 385 } 386 spin_unlock_irq(&vport->phba->hbalock); 387 388 /* Remove original register reference. The host transport 389 * won't reference this rport/remoteport any further. 390 */ 391 lpfc_nlp_put(ndlp); 392 393 rport_err: 394 return; 395 } 396 397 static void 398 lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 399 struct lpfc_wcqe_complete *wcqe) 400 { 401 struct lpfc_vport *vport = cmdwqe->vport; 402 struct lpfc_nvme_lport *lport; 403 uint32_t status; 404 struct nvmefc_ls_req *pnvme_lsreq; 405 struct lpfc_dmabuf *buf_ptr; 406 struct lpfc_nodelist *ndlp; 407 408 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 409 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 410 411 if (vport->localport) { 412 lport = (struct lpfc_nvme_lport *)vport->localport->private; 413 if (lport) { 414 atomic_inc(&lport->fc4NvmeLsCmpls); 415 if (status) { 416 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 417 atomic_inc(&lport->cmpl_ls_xb); 418 atomic_inc(&lport->cmpl_ls_err); 419 } 420 } 421 } 422 423 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 424 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 425 "6047 nvme cmpl Enter " 426 "Data %px DID %x Xri: %x status %x reason x%x " 427 "cmd:x%px lsreg:x%px bmp:x%px ndlp:x%px\n", 428 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 429 cmdwqe->sli4_xritag, status, 430 (wcqe->parameter & 0xffff), 431 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 432 433 lpfc_nvmeio_data(phba, "NVME LS CMPL: xri x%x stat x%x parm x%x\n", 434 cmdwqe->sli4_xritag, status, wcqe->parameter); 435 436 if (cmdwqe->context3) { 437 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 438 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 439 kfree(buf_ptr); 440 cmdwqe->context3 = NULL; 441 } 442 if (pnvme_lsreq->done) 443 pnvme_lsreq->done(pnvme_lsreq, status); 444 else 445 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 446 "6046 nvme cmpl without done call back? " 447 "Data %px DID %x Xri: %x status %x\n", 448 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 449 cmdwqe->sli4_xritag, status); 450 if (ndlp) { 451 lpfc_nlp_put(ndlp); 452 cmdwqe->context1 = NULL; 453 } 454 lpfc_sli_release_iocbq(phba, cmdwqe); 455 } 456 457 static int 458 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 459 struct lpfc_dmabuf *inp, 460 struct nvmefc_ls_req *pnvme_lsreq, 461 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 462 struct lpfc_wcqe_complete *), 463 struct lpfc_nodelist *ndlp, uint32_t num_entry, 464 uint32_t tmo, uint8_t retry) 465 { 466 struct lpfc_hba *phba = vport->phba; 467 union lpfc_wqe128 *wqe; 468 struct lpfc_iocbq *genwqe; 469 struct ulp_bde64 *bpl; 470 struct ulp_bde64 bde; 471 int i, rc, xmit_len, first_len; 472 473 /* Allocate buffer for command WQE */ 474 genwqe = lpfc_sli_get_iocbq(phba); 475 if (genwqe == NULL) 476 return 1; 477 478 wqe = &genwqe->wqe; 479 /* Initialize only 64 bytes */ 480 memset(wqe, 0, sizeof(union lpfc_wqe)); 481 482 genwqe->context3 = (uint8_t *)bmp; 483 genwqe->iocb_flag |= LPFC_IO_NVME_LS; 484 485 /* Save for completion so we can release these resources */ 486 genwqe->context1 = lpfc_nlp_get(ndlp); 487 genwqe->context2 = (uint8_t *)pnvme_lsreq; 488 /* Fill in payload, bp points to frame payload */ 489 490 if (!tmo) 491 /* FC spec states we need 3 * ratov for CT requests */ 492 tmo = (3 * phba->fc_ratov); 493 494 /* For this command calculate the xmit length of the request bde. */ 495 xmit_len = 0; 496 first_len = 0; 497 bpl = (struct ulp_bde64 *)bmp->virt; 498 for (i = 0; i < num_entry; i++) { 499 bde.tus.w = bpl[i].tus.w; 500 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 501 break; 502 xmit_len += bde.tus.f.bdeSize; 503 if (i == 0) 504 first_len = xmit_len; 505 } 506 507 genwqe->rsvd2 = num_entry; 508 genwqe->hba_wqidx = 0; 509 510 /* Words 0 - 2 */ 511 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 512 wqe->generic.bde.tus.f.bdeSize = first_len; 513 wqe->generic.bde.addrLow = bpl[0].addrLow; 514 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 515 516 /* Word 3 */ 517 wqe->gen_req.request_payload_len = first_len; 518 519 /* Word 4 */ 520 521 /* Word 5 */ 522 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 523 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 524 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 525 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 526 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 527 528 /* Word 6 */ 529 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 530 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 531 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 532 533 /* Word 7 */ 534 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 535 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 536 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 537 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 538 539 /* Word 8 */ 540 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 541 542 /* Word 9 */ 543 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 544 545 /* Word 10 */ 546 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 547 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 548 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 549 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 550 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 551 552 /* Word 11 */ 553 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 554 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 555 556 557 /* Issue GEN REQ WQE for NPORT <did> */ 558 lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS, 559 "6050 Issue GEN REQ WQE to NPORT x%x " 560 "Data: x%x x%x wq:x%px lsreq:x%px bmp:x%px " 561 "xmit:%d 1st:%d\n", 562 ndlp->nlp_DID, genwqe->iotag, 563 vport->port_state, 564 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 565 genwqe->wqe_cmpl = cmpl; 566 genwqe->iocb_cmpl = NULL; 567 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 568 genwqe->vport = vport; 569 genwqe->retry = retry; 570 571 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 572 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 573 574 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 575 if (rc) { 576 lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, 577 "6045 Issue GEN REQ WQE to NPORT x%x " 578 "Data: x%x x%x\n", 579 ndlp->nlp_DID, genwqe->iotag, 580 vport->port_state); 581 lpfc_sli_release_iocbq(phba, genwqe); 582 return 1; 583 } 584 return 0; 585 } 586 587 /** 588 * lpfc_nvme_ls_req - Issue an Link Service request 589 * @lpfc_pnvme: Pointer to the driver's nvme instance data 590 * @lpfc_nvme_lport: Pointer to the driver's local port data 591 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 592 * 593 * Driver registers this routine to handle any link service request 594 * from the nvme_fc transport to a remote nvme-aware port. 595 * 596 * Return value : 597 * 0 - Success 598 * TODO: What are the failure codes. 599 **/ 600 static int 601 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 602 struct nvme_fc_remote_port *pnvme_rport, 603 struct nvmefc_ls_req *pnvme_lsreq) 604 { 605 int ret = 0; 606 struct lpfc_nvme_lport *lport; 607 struct lpfc_nvme_rport *rport; 608 struct lpfc_vport *vport; 609 struct lpfc_nodelist *ndlp; 610 struct ulp_bde64 *bpl; 611 struct lpfc_dmabuf *bmp; 612 uint16_t ntype, nstate; 613 614 /* there are two dma buf in the request, actually there is one and 615 * the second one is just the start address + cmd size. 616 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 617 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 618 * because the nvem layer owns the data bufs. 619 * We do not have to break these packets open, we don't care what is in 620 * them. And we do not have to look at the resonse data, we only care 621 * that we got a response. All of the caring is going to happen in the 622 * nvme-fc layer. 623 */ 624 625 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 626 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 627 if (unlikely(!lport) || unlikely(!rport)) 628 return -EINVAL; 629 630 vport = lport->vport; 631 632 if (vport->load_flag & FC_UNLOADING) 633 return -ENODEV; 634 635 /* Need the ndlp. It is stored in the driver's rport. */ 636 ndlp = rport->ndlp; 637 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 638 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 639 "6051 Remoteport x%px, rport has invalid ndlp. " 640 "Failing LS Req\n", pnvme_rport); 641 return -ENODEV; 642 } 643 644 /* The remote node has to be a mapped nvme target or an 645 * unmapped nvme initiator or it's an error. 646 */ 647 ntype = ndlp->nlp_type; 648 nstate = ndlp->nlp_state; 649 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 650 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 651 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 652 "6088 DID x%06x not ready for " 653 "IO. State x%x, Type x%x\n", 654 pnvme_rport->port_id, 655 ndlp->nlp_state, ndlp->nlp_type); 656 return -ENODEV; 657 } 658 bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); 659 if (!bmp) { 660 661 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 662 "6044 Could not find node for DID %x\n", 663 pnvme_rport->port_id); 664 return 2; 665 } 666 INIT_LIST_HEAD(&bmp->list); 667 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 668 if (!bmp->virt) { 669 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 670 "6042 Could not find node for DID %x\n", 671 pnvme_rport->port_id); 672 kfree(bmp); 673 return 3; 674 } 675 bpl = (struct ulp_bde64 *)bmp->virt; 676 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 677 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 678 bpl->tus.f.bdeFlags = 0; 679 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 680 bpl->tus.w = le32_to_cpu(bpl->tus.w); 681 bpl++; 682 683 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 684 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 685 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 686 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 687 bpl->tus.w = le32_to_cpu(bpl->tus.w); 688 689 /* Expand print to include key fields. */ 690 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 691 "6149 Issue LS Req to DID 0x%06x lport x%px, " 692 "rport x%px lsreq x%px rqstlen:%d rsplen:%d " 693 "%pad %pad\n", 694 ndlp->nlp_DID, pnvme_lport, pnvme_rport, 695 pnvme_lsreq, pnvme_lsreq->rqstlen, 696 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 697 &pnvme_lsreq->rspdma); 698 699 atomic_inc(&lport->fc4NvmeLsRequests); 700 701 /* Hardcode the wait to 30 seconds. Connections are failing otherwise. 702 * This code allows it all to work. 703 */ 704 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 705 pnvme_lsreq, lpfc_nvme_cmpl_gen_req, 706 ndlp, 2, 30, 0); 707 if (ret != WQE_SUCCESS) { 708 atomic_inc(&lport->xmt_ls_err); 709 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 710 "6052 EXIT. issue ls wqe failed lport x%px, " 711 "rport x%px lsreq x%px Status %x DID %x\n", 712 pnvme_lport, pnvme_rport, pnvme_lsreq, 713 ret, ndlp->nlp_DID); 714 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 715 kfree(bmp); 716 return ret; 717 } 718 719 /* Stub in routine and return 0 for now. */ 720 return ret; 721 } 722 723 /** 724 * lpfc_nvme_ls_abort - Issue an Link Service request 725 * @lpfc_pnvme: Pointer to the driver's nvme instance data 726 * @lpfc_nvme_lport: Pointer to the driver's local port data 727 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 728 * 729 * Driver registers this routine to handle any link service request 730 * from the nvme_fc transport to a remote nvme-aware port. 731 * 732 * Return value : 733 * 0 - Success 734 * TODO: What are the failure codes. 735 **/ 736 static void 737 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 738 struct nvme_fc_remote_port *pnvme_rport, 739 struct nvmefc_ls_req *pnvme_lsreq) 740 { 741 struct lpfc_nvme_lport *lport; 742 struct lpfc_vport *vport; 743 struct lpfc_hba *phba; 744 struct lpfc_nodelist *ndlp; 745 LIST_HEAD(abort_list); 746 struct lpfc_sli_ring *pring; 747 struct lpfc_iocbq *wqe, *next_wqe; 748 749 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 750 if (unlikely(!lport)) 751 return; 752 vport = lport->vport; 753 phba = vport->phba; 754 755 if (vport->load_flag & FC_UNLOADING) 756 return; 757 758 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 759 if (!ndlp) { 760 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 761 "6049 Could not find node for DID %x\n", 762 pnvme_rport->port_id); 763 return; 764 } 765 766 /* Expand print to include key fields. */ 767 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 768 "6040 ENTER. lport x%px, rport x%px lsreq x%px rqstlen:%d " 769 "rsplen:%d %pad %pad\n", 770 pnvme_lport, pnvme_rport, 771 pnvme_lsreq, pnvme_lsreq->rqstlen, 772 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 773 &pnvme_lsreq->rspdma); 774 775 /* 776 * Lock the ELS ring txcmplq and build a local list of all ELS IOs 777 * that need an ABTS. The IOs need to stay on the txcmplq so that 778 * the abort operation completes them successfully. 779 */ 780 pring = phba->sli4_hba.nvmels_wq->pring; 781 spin_lock_irq(&phba->hbalock); 782 spin_lock(&pring->ring_lock); 783 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 784 /* Add to abort_list on on NDLP match. */ 785 if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) { 786 wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 787 list_add_tail(&wqe->dlist, &abort_list); 788 } 789 } 790 spin_unlock(&pring->ring_lock); 791 spin_unlock_irq(&phba->hbalock); 792 793 /* Abort the targeted IOs and remove them from the abort list. */ 794 list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) { 795 atomic_inc(&lport->xmt_ls_abort); 796 spin_lock_irq(&phba->hbalock); 797 list_del_init(&wqe->dlist); 798 lpfc_sli_issue_abort_iotag(phba, pring, wqe); 799 spin_unlock_irq(&phba->hbalock); 800 } 801 } 802 803 /* Fix up the existing sgls for NVME IO. */ 804 static inline void 805 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 806 struct lpfc_io_buf *lpfc_ncmd, 807 struct nvmefc_fcp_req *nCmd) 808 { 809 struct lpfc_hba *phba = vport->phba; 810 struct sli4_sge *sgl; 811 union lpfc_wqe128 *wqe; 812 uint32_t *wptr, *dptr; 813 814 /* 815 * Get a local pointer to the built-in wqe and correct 816 * the cmd size to match NVME's 96 bytes and fix 817 * the dma address. 818 */ 819 820 wqe = &lpfc_ncmd->cur_iocbq.wqe; 821 822 /* 823 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 824 * match NVME. NVME sends 96 bytes. Also, use the 825 * nvme commands command and response dma addresses 826 * rather than the virtual memory to ease the restore 827 * operation. 828 */ 829 sgl = lpfc_ncmd->dma_sgl; 830 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 831 if (phba->cfg_nvme_embed_cmd) { 832 sgl->addr_hi = 0; 833 sgl->addr_lo = 0; 834 835 /* Word 0-2 - NVME CMND IU (embedded payload) */ 836 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 837 wqe->generic.bde.tus.f.bdeSize = 56; 838 wqe->generic.bde.addrHigh = 0; 839 wqe->generic.bde.addrLow = 64; /* Word 16 */ 840 841 /* Word 10 - dbde is 0, wqes is 1 in template */ 842 843 /* 844 * Embed the payload in the last half of the WQE 845 * WQE words 16-30 get the NVME CMD IU payload 846 * 847 * WQE words 16-19 get payload Words 1-4 848 * WQE words 20-21 get payload Words 6-7 849 * WQE words 22-29 get payload Words 16-23 850 */ 851 wptr = &wqe->words[16]; /* WQE ptr */ 852 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 853 dptr++; /* Skip Word 0 in payload */ 854 855 *wptr++ = *dptr++; /* Word 1 */ 856 *wptr++ = *dptr++; /* Word 2 */ 857 *wptr++ = *dptr++; /* Word 3 */ 858 *wptr++ = *dptr++; /* Word 4 */ 859 dptr++; /* Skip Word 5 in payload */ 860 *wptr++ = *dptr++; /* Word 6 */ 861 *wptr++ = *dptr++; /* Word 7 */ 862 dptr += 8; /* Skip Words 8-15 in payload */ 863 *wptr++ = *dptr++; /* Word 16 */ 864 *wptr++ = *dptr++; /* Word 17 */ 865 *wptr++ = *dptr++; /* Word 18 */ 866 *wptr++ = *dptr++; /* Word 19 */ 867 *wptr++ = *dptr++; /* Word 20 */ 868 *wptr++ = *dptr++; /* Word 21 */ 869 *wptr++ = *dptr++; /* Word 22 */ 870 *wptr = *dptr; /* Word 23 */ 871 } else { 872 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 873 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 874 875 /* Word 0-2 - NVME CMND IU Inline BDE */ 876 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 877 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 878 wqe->generic.bde.addrHigh = sgl->addr_hi; 879 wqe->generic.bde.addrLow = sgl->addr_lo; 880 881 /* Word 10 */ 882 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 883 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 884 } 885 886 sgl++; 887 888 /* Setup the physical region for the FCP RSP */ 889 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 890 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 891 sgl->word2 = le32_to_cpu(sgl->word2); 892 if (nCmd->sg_cnt) 893 bf_set(lpfc_sli4_sge_last, sgl, 0); 894 else 895 bf_set(lpfc_sli4_sge_last, sgl, 1); 896 sgl->word2 = cpu_to_le32(sgl->word2); 897 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 898 } 899 900 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 901 static void 902 lpfc_nvme_ktime(struct lpfc_hba *phba, 903 struct lpfc_io_buf *lpfc_ncmd) 904 { 905 uint64_t seg1, seg2, seg3, seg4; 906 uint64_t segsum; 907 908 if (!lpfc_ncmd->ts_last_cmd || 909 !lpfc_ncmd->ts_cmd_start || 910 !lpfc_ncmd->ts_cmd_wqput || 911 !lpfc_ncmd->ts_isr_cmpl || 912 !lpfc_ncmd->ts_data_nvme) 913 return; 914 915 if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_cmd_start) 916 return; 917 if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd) 918 return; 919 if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start) 920 return; 921 if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput) 922 return; 923 if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl) 924 return; 925 /* 926 * Segment 1 - Time from Last FCP command cmpl is handed 927 * off to NVME Layer to start of next command. 928 * Segment 2 - Time from Driver receives a IO cmd start 929 * from NVME Layer to WQ put is done on IO cmd. 930 * Segment 3 - Time from Driver WQ put is done on IO cmd 931 * to MSI-X ISR for IO cmpl. 932 * Segment 4 - Time from MSI-X ISR for IO cmpl to when 933 * cmpl is handled off to the NVME Layer. 934 */ 935 seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd; 936 if (seg1 > 5000000) /* 5 ms - for sequential IOs only */ 937 seg1 = 0; 938 939 /* Calculate times relative to start of IO */ 940 seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start); 941 segsum = seg2; 942 seg3 = lpfc_ncmd->ts_isr_cmpl - lpfc_ncmd->ts_cmd_start; 943 if (segsum > seg3) 944 return; 945 seg3 -= segsum; 946 segsum += seg3; 947 948 seg4 = lpfc_ncmd->ts_data_nvme - lpfc_ncmd->ts_cmd_start; 949 if (segsum > seg4) 950 return; 951 seg4 -= segsum; 952 953 phba->ktime_data_samples++; 954 phba->ktime_seg1_total += seg1; 955 if (seg1 < phba->ktime_seg1_min) 956 phba->ktime_seg1_min = seg1; 957 else if (seg1 > phba->ktime_seg1_max) 958 phba->ktime_seg1_max = seg1; 959 phba->ktime_seg2_total += seg2; 960 if (seg2 < phba->ktime_seg2_min) 961 phba->ktime_seg2_min = seg2; 962 else if (seg2 > phba->ktime_seg2_max) 963 phba->ktime_seg2_max = seg2; 964 phba->ktime_seg3_total += seg3; 965 if (seg3 < phba->ktime_seg3_min) 966 phba->ktime_seg3_min = seg3; 967 else if (seg3 > phba->ktime_seg3_max) 968 phba->ktime_seg3_max = seg3; 969 phba->ktime_seg4_total += seg4; 970 if (seg4 < phba->ktime_seg4_min) 971 phba->ktime_seg4_min = seg4; 972 else if (seg4 > phba->ktime_seg4_max) 973 phba->ktime_seg4_max = seg4; 974 975 lpfc_ncmd->ts_last_cmd = 0; 976 lpfc_ncmd->ts_cmd_start = 0; 977 lpfc_ncmd->ts_cmd_wqput = 0; 978 lpfc_ncmd->ts_isr_cmpl = 0; 979 lpfc_ncmd->ts_data_nvme = 0; 980 } 981 #endif 982 983 /** 984 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 985 * @lpfc_pnvme: Pointer to the driver's nvme instance data 986 * @lpfc_nvme_lport: Pointer to the driver's local port data 987 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 988 * 989 * Driver registers this routine as it io request handler. This 990 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 991 * data structure to the rport indicated in @lpfc_nvme_rport. 992 * 993 * Return value : 994 * 0 - Success 995 * TODO: What are the failure codes. 996 **/ 997 static void 998 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 999 struct lpfc_wcqe_complete *wcqe) 1000 { 1001 struct lpfc_io_buf *lpfc_ncmd = 1002 (struct lpfc_io_buf *)pwqeIn->context1; 1003 struct lpfc_vport *vport = pwqeIn->vport; 1004 struct nvmefc_fcp_req *nCmd; 1005 struct nvme_fc_ersp_iu *ep; 1006 struct nvme_fc_cmd_iu *cp; 1007 struct lpfc_nodelist *ndlp; 1008 struct lpfc_nvme_fcpreq_priv *freqpriv; 1009 struct lpfc_nvme_lport *lport; 1010 uint32_t code, status, idx; 1011 uint16_t cid, sqhd, data; 1012 uint32_t *ptr; 1013 1014 /* Sanity check on return of outstanding command */ 1015 if (!lpfc_ncmd) { 1016 lpfc_printf_vlog(vport, KERN_ERR, 1017 LOG_NODE | LOG_NVME_IOERR, 1018 "6071 Null lpfc_ncmd pointer. No " 1019 "release, skip completion\n"); 1020 return; 1021 } 1022 1023 /* Guard against abort handler being called at same time */ 1024 spin_lock(&lpfc_ncmd->buf_lock); 1025 1026 if (!lpfc_ncmd->nvmeCmd) { 1027 spin_unlock(&lpfc_ncmd->buf_lock); 1028 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1029 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 1030 "nvmeCmd x%px\n", 1031 lpfc_ncmd, lpfc_ncmd->nvmeCmd); 1032 1033 /* Release the lpfc_ncmd regardless of the missing elements. */ 1034 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1035 return; 1036 } 1037 nCmd = lpfc_ncmd->nvmeCmd; 1038 status = bf_get(lpfc_wcqe_c_status, wcqe); 1039 1040 idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 1041 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 1042 1043 if (unlikely(status && vport->localport)) { 1044 lport = (struct lpfc_nvme_lport *)vport->localport->private; 1045 if (lport) { 1046 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1047 atomic_inc(&lport->cmpl_fcp_xb); 1048 atomic_inc(&lport->cmpl_fcp_err); 1049 } 1050 } 1051 1052 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 1053 lpfc_ncmd->cur_iocbq.sli4_xritag, 1054 status, wcqe->parameter); 1055 /* 1056 * Catch race where our node has transitioned, but the 1057 * transport is still transitioning. 1058 */ 1059 ndlp = lpfc_ncmd->ndlp; 1060 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1061 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1062 "6062 Ignoring NVME cmpl. No ndlp\n"); 1063 goto out_err; 1064 } 1065 1066 code = bf_get(lpfc_wcqe_c_code, wcqe); 1067 if (code == CQE_CODE_NVME_ERSP) { 1068 /* For this type of CQE, we need to rebuild the rsp */ 1069 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 1070 1071 /* 1072 * Get Command Id from cmd to plug into response. This 1073 * code is not needed in the next NVME Transport drop. 1074 */ 1075 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1076 cid = cp->sqe.common.command_id; 1077 1078 /* 1079 * RSN is in CQE word 2 1080 * SQHD is in CQE Word 3 bits 15:0 1081 * Cmd Specific info is in CQE Word 1 1082 * and in CQE Word 0 bits 15:0 1083 */ 1084 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 1085 1086 /* Now lets build the NVME ERSP IU */ 1087 ep->iu_len = cpu_to_be16(8); 1088 ep->rsn = wcqe->parameter; 1089 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 1090 ep->rsvd12 = 0; 1091 ptr = (uint32_t *)&ep->cqe.result.u64; 1092 *ptr++ = wcqe->total_data_placed; 1093 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 1094 *ptr = (uint32_t)data; 1095 ep->cqe.sq_head = sqhd; 1096 ep->cqe.sq_id = nCmd->sqid; 1097 ep->cqe.command_id = cid; 1098 ep->cqe.status = 0; 1099 1100 lpfc_ncmd->status = IOSTAT_SUCCESS; 1101 lpfc_ncmd->result = 0; 1102 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 1103 nCmd->transferred_length = nCmd->payload_length; 1104 } else { 1105 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1106 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 1107 1108 /* For NVME, the only failure path that results in an 1109 * IO error is when the adapter rejects it. All other 1110 * conditions are a success case and resolved by the 1111 * transport. 1112 * IOSTAT_FCP_RSP_ERROR means: 1113 * 1. Length of data received doesn't match total 1114 * transfer length in WQE 1115 * 2. If the RSP payload does NOT match these cases: 1116 * a. RSP length 12/24 bytes and all zeros 1117 * b. NVME ERSP 1118 */ 1119 switch (lpfc_ncmd->status) { 1120 case IOSTAT_SUCCESS: 1121 nCmd->transferred_length = wcqe->total_data_placed; 1122 nCmd->rcv_rsplen = 0; 1123 nCmd->status = 0; 1124 break; 1125 case IOSTAT_FCP_RSP_ERROR: 1126 nCmd->transferred_length = wcqe->total_data_placed; 1127 nCmd->rcv_rsplen = wcqe->parameter; 1128 nCmd->status = 0; 1129 /* Sanity check */ 1130 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 1131 break; 1132 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1133 "6081 NVME Completion Protocol Error: " 1134 "xri %x status x%x result x%x " 1135 "placed x%x\n", 1136 lpfc_ncmd->cur_iocbq.sli4_xritag, 1137 lpfc_ncmd->status, lpfc_ncmd->result, 1138 wcqe->total_data_placed); 1139 break; 1140 case IOSTAT_LOCAL_REJECT: 1141 /* Let fall through to set command final state. */ 1142 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1143 lpfc_printf_vlog(vport, KERN_INFO, 1144 LOG_NVME_IOERR, 1145 "6032 Delay Aborted cmd x%px " 1146 "nvme cmd x%px, xri x%x, " 1147 "xb %d\n", 1148 lpfc_ncmd, nCmd, 1149 lpfc_ncmd->cur_iocbq.sli4_xritag, 1150 bf_get(lpfc_wcqe_c_xb, wcqe)); 1151 /* fall through */ 1152 default: 1153 out_err: 1154 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1155 "6072 NVME Completion Error: xri %x " 1156 "status x%x result x%x [x%x] " 1157 "placed x%x\n", 1158 lpfc_ncmd->cur_iocbq.sli4_xritag, 1159 lpfc_ncmd->status, lpfc_ncmd->result, 1160 wcqe->parameter, 1161 wcqe->total_data_placed); 1162 nCmd->transferred_length = 0; 1163 nCmd->rcv_rsplen = 0; 1164 nCmd->status = NVME_SC_INTERNAL; 1165 } 1166 } 1167 1168 /* pick up SLI4 exhange busy condition */ 1169 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1170 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1171 else 1172 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1173 1174 /* Update stats and complete the IO. There is 1175 * no need for dma unprep because the nvme_transport 1176 * owns the dma address. 1177 */ 1178 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1179 if (lpfc_ncmd->ts_cmd_start) { 1180 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 1181 lpfc_ncmd->ts_data_nvme = ktime_get_ns(); 1182 phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme; 1183 lpfc_nvme_ktime(phba, lpfc_ncmd); 1184 } 1185 if (unlikely(phba->cpucheck_on & LPFC_CHECK_NVME_IO)) { 1186 uint32_t cpu; 1187 idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 1188 cpu = raw_smp_processor_id(); 1189 if (cpu < LPFC_CHECK_CPU_CNT) { 1190 if (lpfc_ncmd->cpu != cpu) 1191 lpfc_printf_vlog(vport, 1192 KERN_INFO, LOG_NVME_IOERR, 1193 "6701 CPU Check cmpl: " 1194 "cpu %d expect %d\n", 1195 cpu, lpfc_ncmd->cpu); 1196 phba->sli4_hba.hdwq[idx].cpucheck_cmpl_io[cpu]++; 1197 } 1198 } 1199 #endif 1200 1201 /* NVME targets need completion held off until the abort exchange 1202 * completes unless the NVME Rport is getting unregistered. 1203 */ 1204 1205 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 1206 freqpriv = nCmd->private; 1207 freqpriv->nvme_buf = NULL; 1208 lpfc_ncmd->nvmeCmd = NULL; 1209 spin_unlock(&lpfc_ncmd->buf_lock); 1210 nCmd->done(nCmd); 1211 } else 1212 spin_unlock(&lpfc_ncmd->buf_lock); 1213 1214 /* Call release with XB=1 to queue the IO into the abort list. */ 1215 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1216 } 1217 1218 1219 /** 1220 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 1221 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1222 * @lpfc_nvme_lport: Pointer to the driver's local port data 1223 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1224 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1225 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1226 * 1227 * Driver registers this routine as it io request handler. This 1228 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1229 * data structure to the rport indicated in @lpfc_nvme_rport. 1230 * 1231 * Return value : 1232 * 0 - Success 1233 * TODO: What are the failure codes. 1234 **/ 1235 static int 1236 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1237 struct lpfc_io_buf *lpfc_ncmd, 1238 struct lpfc_nodelist *pnode, 1239 struct lpfc_fc4_ctrl_stat *cstat) 1240 { 1241 struct lpfc_hba *phba = vport->phba; 1242 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1243 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 1244 union lpfc_wqe128 *wqe = &pwqeq->wqe; 1245 uint32_t req_len; 1246 1247 if (!NLP_CHK_NODE_ACT(pnode)) 1248 return -EINVAL; 1249 1250 /* 1251 * There are three possibilities here - use scatter-gather segment, use 1252 * the single mapping, or neither. 1253 */ 1254 if (nCmd->sg_cnt) { 1255 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1256 /* From the iwrite template, initialize words 7 - 11 */ 1257 memcpy(&wqe->words[7], 1258 &lpfc_iwrite_cmd_template.words[7], 1259 sizeof(uint32_t) * 5); 1260 1261 /* Word 4 */ 1262 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 1263 1264 /* Word 5 */ 1265 if ((phba->cfg_nvme_enable_fb) && 1266 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1267 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1268 if (req_len < pnode->nvme_fb_size) 1269 wqe->fcp_iwrite.initial_xfer_len = 1270 req_len; 1271 else 1272 wqe->fcp_iwrite.initial_xfer_len = 1273 pnode->nvme_fb_size; 1274 } else { 1275 wqe->fcp_iwrite.initial_xfer_len = 0; 1276 } 1277 cstat->output_requests++; 1278 } else { 1279 /* From the iread template, initialize words 7 - 11 */ 1280 memcpy(&wqe->words[7], 1281 &lpfc_iread_cmd_template.words[7], 1282 sizeof(uint32_t) * 5); 1283 1284 /* Word 4 */ 1285 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1286 1287 /* Word 5 */ 1288 wqe->fcp_iread.rsrvd5 = 0; 1289 1290 cstat->input_requests++; 1291 } 1292 } else { 1293 /* From the icmnd template, initialize words 4 - 11 */ 1294 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 1295 sizeof(uint32_t) * 8); 1296 cstat->control_requests++; 1297 } 1298 1299 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) 1300 bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 1301 /* 1302 * Finish initializing those WQE fields that are independent 1303 * of the nvme_cmnd request_buffer 1304 */ 1305 1306 /* Word 3 */ 1307 bf_set(payload_offset_len, &wqe->fcp_icmd, 1308 (nCmd->rsplen + nCmd->cmdlen)); 1309 1310 /* Word 6 */ 1311 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1312 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1313 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1314 1315 /* Word 8 */ 1316 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1317 1318 /* Word 9 */ 1319 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1320 1321 /* Words 13 14 15 are for PBDE support */ 1322 1323 pwqeq->vport = vport; 1324 return 0; 1325 } 1326 1327 1328 /** 1329 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1330 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1331 * @lpfc_nvme_lport: Pointer to the driver's local port data 1332 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1333 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1334 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1335 * 1336 * Driver registers this routine as it io request handler. This 1337 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1338 * data structure to the rport indicated in @lpfc_nvme_rport. 1339 * 1340 * Return value : 1341 * 0 - Success 1342 * TODO: What are the failure codes. 1343 **/ 1344 static int 1345 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1346 struct lpfc_io_buf *lpfc_ncmd) 1347 { 1348 struct lpfc_hba *phba = vport->phba; 1349 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1350 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 1351 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1352 struct sli4_hybrid_sgl *sgl_xtra = NULL; 1353 struct scatterlist *data_sg; 1354 struct sli4_sge *first_data_sgl; 1355 struct ulp_bde64 *bde; 1356 dma_addr_t physaddr = 0; 1357 uint32_t num_bde = 0; 1358 uint32_t dma_len = 0; 1359 uint32_t dma_offset = 0; 1360 int nseg, i, j; 1361 bool lsp_just_set = false; 1362 1363 /* Fix up the command and response DMA stuff. */ 1364 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1365 1366 /* 1367 * There are three possibilities here - use scatter-gather segment, use 1368 * the single mapping, or neither. 1369 */ 1370 if (nCmd->sg_cnt) { 1371 /* 1372 * Jump over the cmd and rsp SGEs. The fix routine 1373 * has already adjusted for this. 1374 */ 1375 sgl += 2; 1376 1377 first_data_sgl = sgl; 1378 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1379 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1380 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1381 "6058 Too many sg segments from " 1382 "NVME Transport. Max %d, " 1383 "nvmeIO sg_cnt %d\n", 1384 phba->cfg_nvme_seg_cnt + 1, 1385 lpfc_ncmd->seg_cnt); 1386 lpfc_ncmd->seg_cnt = 0; 1387 return 1; 1388 } 1389 1390 /* 1391 * The driver established a maximum scatter-gather segment count 1392 * during probe that limits the number of sg elements in any 1393 * single nvme command. Just run through the seg_cnt and format 1394 * the sge's. 1395 */ 1396 nseg = nCmd->sg_cnt; 1397 data_sg = nCmd->first_sgl; 1398 1399 /* for tracking the segment boundaries */ 1400 j = 2; 1401 for (i = 0; i < nseg; i++) { 1402 if (data_sg == NULL) { 1403 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1404 "6059 dptr err %d, nseg %d\n", 1405 i, nseg); 1406 lpfc_ncmd->seg_cnt = 0; 1407 return 1; 1408 } 1409 1410 sgl->word2 = 0; 1411 if ((num_bde + 1) == nseg) { 1412 bf_set(lpfc_sli4_sge_last, sgl, 1); 1413 bf_set(lpfc_sli4_sge_type, sgl, 1414 LPFC_SGE_TYPE_DATA); 1415 } else { 1416 bf_set(lpfc_sli4_sge_last, sgl, 0); 1417 1418 /* expand the segment */ 1419 if (!lsp_just_set && 1420 !((j + 1) % phba->border_sge_num) && 1421 ((nseg - 1) != i)) { 1422 /* set LSP type */ 1423 bf_set(lpfc_sli4_sge_type, sgl, 1424 LPFC_SGE_TYPE_LSP); 1425 1426 sgl_xtra = lpfc_get_sgl_per_hdwq( 1427 phba, lpfc_ncmd); 1428 1429 if (unlikely(!sgl_xtra)) { 1430 lpfc_ncmd->seg_cnt = 0; 1431 return 1; 1432 } 1433 sgl->addr_lo = cpu_to_le32(putPaddrLow( 1434 sgl_xtra->dma_phys_sgl)); 1435 sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1436 sgl_xtra->dma_phys_sgl)); 1437 1438 } else { 1439 bf_set(lpfc_sli4_sge_type, sgl, 1440 LPFC_SGE_TYPE_DATA); 1441 } 1442 } 1443 1444 if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1445 LPFC_SGE_TYPE_LSP)) { 1446 if ((nseg - 1) == i) 1447 bf_set(lpfc_sli4_sge_last, sgl, 1); 1448 1449 physaddr = data_sg->dma_address; 1450 dma_len = data_sg->length; 1451 sgl->addr_lo = cpu_to_le32( 1452 putPaddrLow(physaddr)); 1453 sgl->addr_hi = cpu_to_le32( 1454 putPaddrHigh(physaddr)); 1455 1456 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1457 sgl->word2 = cpu_to_le32(sgl->word2); 1458 sgl->sge_len = cpu_to_le32(dma_len); 1459 1460 dma_offset += dma_len; 1461 data_sg = sg_next(data_sg); 1462 1463 sgl++; 1464 1465 lsp_just_set = false; 1466 } else { 1467 sgl->word2 = cpu_to_le32(sgl->word2); 1468 1469 sgl->sge_len = cpu_to_le32( 1470 phba->cfg_sg_dma_buf_size); 1471 1472 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1473 i = i - 1; 1474 1475 lsp_just_set = true; 1476 } 1477 1478 j++; 1479 } 1480 if (phba->cfg_enable_pbde) { 1481 /* Use PBDE support for first SGL only, offset == 0 */ 1482 /* Words 13-15 */ 1483 bde = (struct ulp_bde64 *) 1484 &wqe->words[13]; 1485 bde->addrLow = first_data_sgl->addr_lo; 1486 bde->addrHigh = first_data_sgl->addr_hi; 1487 bde->tus.f.bdeSize = 1488 le32_to_cpu(first_data_sgl->sge_len); 1489 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1490 bde->tus.w = cpu_to_le32(bde->tus.w); 1491 /* wqe_pbde is 1 in template */ 1492 } else { 1493 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1494 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 1495 } 1496 1497 } else { 1498 lpfc_ncmd->seg_cnt = 0; 1499 1500 /* For this clause to be valid, the payload_length 1501 * and sg_cnt must zero. 1502 */ 1503 if (nCmd->payload_length != 0) { 1504 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1505 "6063 NVME DMA Prep Err: sg_cnt %d " 1506 "payload_length x%x\n", 1507 nCmd->sg_cnt, nCmd->payload_length); 1508 return 1; 1509 } 1510 } 1511 return 0; 1512 } 1513 1514 /** 1515 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1516 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1517 * @lpfc_nvme_lport: Pointer to the driver's local port data 1518 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1519 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1520 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1521 * 1522 * Driver registers this routine as it io request handler. This 1523 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1524 * data structure to the rport 1525 indicated in @lpfc_nvme_rport. 1526 * 1527 * Return value : 1528 * 0 - Success 1529 * TODO: What are the failure codes. 1530 **/ 1531 static int 1532 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1533 struct nvme_fc_remote_port *pnvme_rport, 1534 void *hw_queue_handle, 1535 struct nvmefc_fcp_req *pnvme_fcreq) 1536 { 1537 int ret = 0; 1538 int expedite = 0; 1539 int idx, cpu; 1540 struct lpfc_nvme_lport *lport; 1541 struct lpfc_fc4_ctrl_stat *cstat; 1542 struct lpfc_vport *vport; 1543 struct lpfc_hba *phba; 1544 struct lpfc_nodelist *ndlp; 1545 struct lpfc_io_buf *lpfc_ncmd; 1546 struct lpfc_nvme_rport *rport; 1547 struct lpfc_nvme_qhandle *lpfc_queue_info; 1548 struct lpfc_nvme_fcpreq_priv *freqpriv; 1549 struct nvme_common_command *sqe; 1550 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1551 uint64_t start = 0; 1552 #endif 1553 1554 /* Validate pointers. LLDD fault handling with transport does 1555 * have timing races. 1556 */ 1557 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1558 if (unlikely(!lport)) { 1559 ret = -EINVAL; 1560 goto out_fail; 1561 } 1562 1563 vport = lport->vport; 1564 1565 if (unlikely(!hw_queue_handle)) { 1566 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1567 "6117 Fail IO, NULL hw_queue_handle\n"); 1568 atomic_inc(&lport->xmt_fcp_err); 1569 ret = -EBUSY; 1570 goto out_fail; 1571 } 1572 1573 phba = vport->phba; 1574 1575 if (vport->load_flag & FC_UNLOADING) { 1576 ret = -ENODEV; 1577 goto out_fail; 1578 } 1579 1580 if (unlikely(vport->load_flag & FC_UNLOADING)) { 1581 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1582 "6124 Fail IO, Driver unload\n"); 1583 atomic_inc(&lport->xmt_fcp_err); 1584 ret = -ENODEV; 1585 goto out_fail; 1586 } 1587 1588 freqpriv = pnvme_fcreq->private; 1589 if (unlikely(!freqpriv)) { 1590 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1591 "6158 Fail IO, NULL request data\n"); 1592 atomic_inc(&lport->xmt_fcp_err); 1593 ret = -EINVAL; 1594 goto out_fail; 1595 } 1596 1597 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1598 if (phba->ktime_on) 1599 start = ktime_get_ns(); 1600 #endif 1601 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1602 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1603 1604 /* 1605 * Catch race where our node has transitioned, but the 1606 * transport is still transitioning. 1607 */ 1608 ndlp = rport->ndlp; 1609 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1610 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1611 "6053 Busy IO, ndlp not ready: rport x%px " 1612 "ndlp x%px, DID x%06x\n", 1613 rport, ndlp, pnvme_rport->port_id); 1614 atomic_inc(&lport->xmt_fcp_err); 1615 ret = -EBUSY; 1616 goto out_fail; 1617 } 1618 1619 /* The remote node has to be a mapped target or it's an error. */ 1620 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1621 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1622 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1623 "6036 Fail IO, DID x%06x not ready for " 1624 "IO. State x%x, Type x%x Flg x%x\n", 1625 pnvme_rport->port_id, 1626 ndlp->nlp_state, ndlp->nlp_type, 1627 ndlp->upcall_flags); 1628 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1629 ret = -EBUSY; 1630 goto out_fail; 1631 1632 } 1633 1634 /* Currently only NVME Keep alive commands should be expedited 1635 * if the driver runs out of a resource. These should only be 1636 * issued on the admin queue, qidx 0 1637 */ 1638 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1639 sqe = &((struct nvme_fc_cmd_iu *) 1640 pnvme_fcreq->cmdaddr)->sqe.common; 1641 if (sqe->opcode == nvme_admin_keep_alive) 1642 expedite = 1; 1643 } 1644 1645 /* The node is shared with FCP IO, make sure the IO pending count does 1646 * not exceed the programmed depth. 1647 */ 1648 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1649 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1650 !expedite) { 1651 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1652 "6174 Fail IO, ndlp qdepth exceeded: " 1653 "idx %d DID %x pend %d qdepth %d\n", 1654 lpfc_queue_info->index, ndlp->nlp_DID, 1655 atomic_read(&ndlp->cmd_pending), 1656 ndlp->cmd_qdepth); 1657 atomic_inc(&lport->xmt_fcp_qdepth); 1658 ret = -EBUSY; 1659 goto out_fail; 1660 } 1661 } 1662 1663 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 1664 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 1665 idx = lpfc_queue_info->index; 1666 } else { 1667 cpu = raw_smp_processor_id(); 1668 idx = phba->sli4_hba.cpu_map[cpu].hdwq; 1669 } 1670 1671 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 1672 if (lpfc_ncmd == NULL) { 1673 atomic_inc(&lport->xmt_fcp_noxri); 1674 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1675 "6065 Fail IO, driver buffer pool is empty: " 1676 "idx %d DID %x\n", 1677 lpfc_queue_info->index, ndlp->nlp_DID); 1678 ret = -EBUSY; 1679 goto out_fail; 1680 } 1681 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1682 if (start) { 1683 lpfc_ncmd->ts_cmd_start = start; 1684 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1685 } else { 1686 lpfc_ncmd->ts_cmd_start = 0; 1687 } 1688 #endif 1689 1690 /* 1691 * Store the data needed by the driver to issue, abort, and complete 1692 * an IO. 1693 * Do not let the IO hang out forever. There is no midlayer issuing 1694 * an abort so inform the FW of the maximum IO pending time. 1695 */ 1696 freqpriv->nvme_buf = lpfc_ncmd; 1697 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1698 lpfc_ncmd->ndlp = ndlp; 1699 lpfc_ncmd->qidx = lpfc_queue_info->qidx; 1700 1701 /* 1702 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1703 * This identfier was create in our hardware queue create callback 1704 * routine. The driver now is dependent on the IO queue steering from 1705 * the transport. We are trusting the upper NVME layers know which 1706 * index to use and that they have affinitized a CPU to this hardware 1707 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1708 */ 1709 lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 1710 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 1711 1712 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 1713 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1714 if (ret) { 1715 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1716 "6175 Fail IO, Prep DMA: " 1717 "idx %d DID %x\n", 1718 lpfc_queue_info->index, ndlp->nlp_DID); 1719 atomic_inc(&lport->xmt_fcp_err); 1720 ret = -ENOMEM; 1721 goto out_free_nvme_buf; 1722 } 1723 1724 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1725 lpfc_ncmd->cur_iocbq.sli4_xritag, 1726 lpfc_queue_info->index, ndlp->nlp_DID); 1727 1728 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 1729 if (ret) { 1730 atomic_inc(&lport->xmt_fcp_wqerr); 1731 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1732 "6113 Fail IO, Could not issue WQE err %x " 1733 "sid: x%x did: x%x oxid: x%x\n", 1734 ret, vport->fc_myDID, ndlp->nlp_DID, 1735 lpfc_ncmd->cur_iocbq.sli4_xritag); 1736 goto out_free_nvme_buf; 1737 } 1738 1739 if (phba->cfg_xri_rebalancing) 1740 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1741 1742 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1743 if (lpfc_ncmd->ts_cmd_start) 1744 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1745 1746 if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { 1747 cpu = raw_smp_processor_id(); 1748 if (cpu < LPFC_CHECK_CPU_CNT) { 1749 lpfc_ncmd->cpu = cpu; 1750 if (idx != cpu) 1751 lpfc_printf_vlog(vport, 1752 KERN_INFO, LOG_NVME_IOERR, 1753 "6702 CPU Check cmd: " 1754 "cpu %d wq %d\n", 1755 lpfc_ncmd->cpu, 1756 lpfc_queue_info->index); 1757 phba->sli4_hba.hdwq[idx].cpucheck_xmt_io[cpu]++; 1758 } 1759 } 1760 #endif 1761 return 0; 1762 1763 out_free_nvme_buf: 1764 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1765 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1766 cstat->output_requests--; 1767 else 1768 cstat->input_requests--; 1769 } else 1770 cstat->control_requests--; 1771 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1772 out_fail: 1773 return ret; 1774 } 1775 1776 /** 1777 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1778 * @phba: Pointer to HBA context object 1779 * @cmdiocb: Pointer to command iocb object. 1780 * @rspiocb: Pointer to response iocb object. 1781 * 1782 * This is the callback function for any NVME FCP IO that was aborted. 1783 * 1784 * Return value: 1785 * None 1786 **/ 1787 void 1788 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1789 struct lpfc_wcqe_complete *abts_cmpl) 1790 { 1791 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1792 "6145 ABORT_XRI_CN completing on rpi x%x " 1793 "original iotag x%x, abort cmd iotag x%x " 1794 "req_tag x%x, status x%x, hwstatus x%x\n", 1795 cmdiocb->iocb.un.acxri.abortContextTag, 1796 cmdiocb->iocb.un.acxri.abortIoTag, 1797 cmdiocb->iotag, 1798 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1799 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1800 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1801 lpfc_sli_release_iocbq(phba, cmdiocb); 1802 } 1803 1804 /** 1805 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1806 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1807 * @lpfc_nvme_lport: Pointer to the driver's local port data 1808 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1809 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1810 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1811 * 1812 * Driver registers this routine as its nvme request io abort handler. This 1813 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1814 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1815 * is executed asynchronously - one the target is validated as "MAPPED" and 1816 * ready for IO, the driver issues the abort request and returns. 1817 * 1818 * Return value: 1819 * None 1820 **/ 1821 static void 1822 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1823 struct nvme_fc_remote_port *pnvme_rport, 1824 void *hw_queue_handle, 1825 struct nvmefc_fcp_req *pnvme_fcreq) 1826 { 1827 struct lpfc_nvme_lport *lport; 1828 struct lpfc_vport *vport; 1829 struct lpfc_hba *phba; 1830 struct lpfc_io_buf *lpfc_nbuf; 1831 struct lpfc_iocbq *abts_buf; 1832 struct lpfc_iocbq *nvmereq_wqe; 1833 struct lpfc_nvme_fcpreq_priv *freqpriv; 1834 unsigned long flags; 1835 int ret_val; 1836 1837 /* Validate pointers. LLDD fault handling with transport does 1838 * have timing races. 1839 */ 1840 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1841 if (unlikely(!lport)) 1842 return; 1843 1844 vport = lport->vport; 1845 1846 if (unlikely(!hw_queue_handle)) { 1847 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1848 "6129 Fail Abort, HW Queue Handle NULL.\n"); 1849 return; 1850 } 1851 1852 phba = vport->phba; 1853 freqpriv = pnvme_fcreq->private; 1854 1855 if (unlikely(!freqpriv)) 1856 return; 1857 if (vport->load_flag & FC_UNLOADING) 1858 return; 1859 1860 /* Announce entry to new IO submit field. */ 1861 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1862 "6002 Abort Request to rport DID x%06x " 1863 "for nvme_fc_req x%px\n", 1864 pnvme_rport->port_id, 1865 pnvme_fcreq); 1866 1867 /* If the hba is getting reset, this flag is set. It is 1868 * cleared when the reset is complete and rings reestablished. 1869 */ 1870 spin_lock_irqsave(&phba->hbalock, flags); 1871 /* driver queued commands are in process of being flushed */ 1872 if (phba->hba_flag & HBA_IOQ_FLUSH) { 1873 spin_unlock_irqrestore(&phba->hbalock, flags); 1874 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1875 "6139 Driver in reset cleanup - flushing " 1876 "NVME Req now. hba_flag x%x\n", 1877 phba->hba_flag); 1878 return; 1879 } 1880 1881 lpfc_nbuf = freqpriv->nvme_buf; 1882 if (!lpfc_nbuf) { 1883 spin_unlock_irqrestore(&phba->hbalock, flags); 1884 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1885 "6140 NVME IO req has no matching lpfc nvme " 1886 "io buffer. Skipping abort req.\n"); 1887 return; 1888 } else if (!lpfc_nbuf->nvmeCmd) { 1889 spin_unlock_irqrestore(&phba->hbalock, flags); 1890 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1891 "6141 lpfc NVME IO req has no nvme_fcreq " 1892 "io buffer. Skipping abort req.\n"); 1893 return; 1894 } 1895 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1896 1897 /* Guard against IO completion being called at same time */ 1898 spin_lock(&lpfc_nbuf->buf_lock); 1899 1900 /* 1901 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1902 * state must match the nvme_fcreq passed by the nvme 1903 * transport. If they don't match, it is likely the driver 1904 * has already completed the NVME IO and the nvme transport 1905 * has not seen it yet. 1906 */ 1907 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1908 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1909 "6143 NVME req mismatch: " 1910 "lpfc_nbuf x%px nvmeCmd x%px, " 1911 "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 1912 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1913 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1914 goto out_unlock; 1915 } 1916 1917 /* Don't abort IOs no longer on the pending queue. */ 1918 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 1919 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1920 "6142 NVME IO req x%px not queued - skipping " 1921 "abort req xri x%x\n", 1922 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1923 goto out_unlock; 1924 } 1925 1926 atomic_inc(&lport->xmt_fcp_abort); 1927 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 1928 nvmereq_wqe->sli4_xritag, 1929 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 1930 1931 /* Outstanding abort is in progress */ 1932 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 1933 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1934 "6144 Outstanding NVME I/O Abort Request " 1935 "still pending on nvme_fcreq x%px, " 1936 "lpfc_ncmd %px xri x%x\n", 1937 pnvme_fcreq, lpfc_nbuf, 1938 nvmereq_wqe->sli4_xritag); 1939 goto out_unlock; 1940 } 1941 1942 abts_buf = __lpfc_sli_get_iocbq(phba); 1943 if (!abts_buf) { 1944 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1945 "6136 No available abort wqes. Skipping " 1946 "Abts req for nvme_fcreq x%px xri x%x\n", 1947 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1948 goto out_unlock; 1949 } 1950 1951 /* Ready - mark outstanding as aborted by driver. */ 1952 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 1953 1954 lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0); 1955 1956 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 1957 abts_buf->iocb_flag |= LPFC_IO_NVME; 1958 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 1959 abts_buf->vport = vport; 1960 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 1961 ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf); 1962 spin_unlock(&lpfc_nbuf->buf_lock); 1963 spin_unlock_irqrestore(&phba->hbalock, flags); 1964 if (ret_val) { 1965 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1966 "6137 Failed abts issue_wqe with status x%x " 1967 "for nvme_fcreq x%px.\n", 1968 ret_val, pnvme_fcreq); 1969 lpfc_sli_release_iocbq(phba, abts_buf); 1970 return; 1971 } 1972 1973 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1974 "6138 Transport Abort NVME Request Issued for " 1975 "ox_id x%x on reqtag x%x\n", 1976 nvmereq_wqe->sli4_xritag, 1977 abts_buf->iotag); 1978 return; 1979 1980 out_unlock: 1981 spin_unlock(&lpfc_nbuf->buf_lock); 1982 spin_unlock_irqrestore(&phba->hbalock, flags); 1983 return; 1984 } 1985 1986 /* Declare and initialization an instance of the FC NVME template. */ 1987 static struct nvme_fc_port_template lpfc_nvme_template = { 1988 .module = THIS_MODULE, 1989 1990 /* initiator-based functions */ 1991 .localport_delete = lpfc_nvme_localport_delete, 1992 .remoteport_delete = lpfc_nvme_remoteport_delete, 1993 .create_queue = lpfc_nvme_create_queue, 1994 .delete_queue = lpfc_nvme_delete_queue, 1995 .ls_req = lpfc_nvme_ls_req, 1996 .fcp_io = lpfc_nvme_fcp_io_submit, 1997 .ls_abort = lpfc_nvme_ls_abort, 1998 .fcp_abort = lpfc_nvme_fcp_abort, 1999 2000 .max_hw_queues = 1, 2001 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2002 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2003 .dma_boundary = 0xFFFFFFFF, 2004 2005 /* Sizes of additional private data for data structures. 2006 * No use for the last two sizes at this time. 2007 */ 2008 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 2009 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 2010 .lsrqst_priv_sz = 0, 2011 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 2012 }; 2013 2014 /** 2015 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 2016 * @phba: The HBA for which this call is being executed. 2017 * 2018 * This routine removes a nvme buffer from head of @hdwq io_buf_list 2019 * and returns to caller. 2020 * 2021 * Return codes: 2022 * NULL - Error 2023 * Pointer to lpfc_nvme_buf - Success 2024 **/ 2025 static struct lpfc_io_buf * 2026 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 2027 int idx, int expedite) 2028 { 2029 struct lpfc_io_buf *lpfc_ncmd; 2030 struct lpfc_sli4_hdw_queue *qp; 2031 struct sli4_sge *sgl; 2032 struct lpfc_iocbq *pwqeq; 2033 union lpfc_wqe128 *wqe; 2034 2035 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 2036 2037 if (lpfc_ncmd) { 2038 pwqeq = &(lpfc_ncmd->cur_iocbq); 2039 wqe = &pwqeq->wqe; 2040 2041 /* Setup key fields in buffer that may have been changed 2042 * if other protocols used this buffer. 2043 */ 2044 pwqeq->iocb_flag = LPFC_IO_NVME; 2045 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 2046 lpfc_ncmd->start_time = jiffies; 2047 lpfc_ncmd->flags = 0; 2048 2049 /* Rsp SGE will be filled in when we rcv an IO 2050 * from the NVME Layer to be sent. 2051 * The cmd is going to be embedded so we need a SKIP SGE. 2052 */ 2053 sgl = lpfc_ncmd->dma_sgl; 2054 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2055 bf_set(lpfc_sli4_sge_last, sgl, 0); 2056 sgl->word2 = cpu_to_le32(sgl->word2); 2057 /* Fill in word 3 / sgl_len during cmd submission */ 2058 2059 /* Initialize 64 bytes only */ 2060 memset(wqe, 0, sizeof(union lpfc_wqe)); 2061 2062 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 2063 atomic_inc(&ndlp->cmd_pending); 2064 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 2065 } 2066 2067 } else { 2068 qp = &phba->sli4_hba.hdwq[idx]; 2069 qp->empty_io_bufs++; 2070 } 2071 2072 return lpfc_ncmd; 2073 } 2074 2075 /** 2076 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2077 * @phba: The Hba for which this call is being executed. 2078 * @lpfc_ncmd: The nvme buffer which is being released. 2079 * 2080 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2081 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2082 * and cannot be reused for at least RA_TOV amount of time if it was 2083 * aborted. 2084 **/ 2085 static void 2086 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 2087 { 2088 struct lpfc_sli4_hdw_queue *qp; 2089 unsigned long iflag = 0; 2090 2091 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 2092 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 2093 2094 lpfc_ncmd->ndlp = NULL; 2095 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 2096 2097 qp = lpfc_ncmd->hdwq; 2098 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 2099 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2100 "6310 XB release deferred for " 2101 "ox_id x%x on reqtag x%x\n", 2102 lpfc_ncmd->cur_iocbq.sli4_xritag, 2103 lpfc_ncmd->cur_iocbq.iotag); 2104 2105 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 2106 list_add_tail(&lpfc_ncmd->list, 2107 &qp->lpfc_abts_io_buf_list); 2108 qp->abts_nvme_io_bufs++; 2109 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2110 } else 2111 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 2112 } 2113 2114 /** 2115 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2116 * @pvport - the lpfc_vport instance requesting a localport. 2117 * 2118 * This routine is invoked to create an nvme localport instance to bind 2119 * to the nvme_fc_transport. It is called once during driver load 2120 * like lpfc_create_shost after all other services are initialized. 2121 * It requires a vport, vpi, and wwns at call time. Other localport 2122 * parameters are modified as the driver's FCID and the Fabric WWN 2123 * are established. 2124 * 2125 * Return codes 2126 * 0 - successful 2127 * -ENOMEM - no heap memory available 2128 * other values - from nvme registration upcall 2129 **/ 2130 int 2131 lpfc_nvme_create_localport(struct lpfc_vport *vport) 2132 { 2133 int ret = 0; 2134 struct lpfc_hba *phba = vport->phba; 2135 struct nvme_fc_port_info nfcp_info; 2136 struct nvme_fc_local_port *localport; 2137 struct lpfc_nvme_lport *lport; 2138 2139 /* Initialize this localport instance. The vport wwn usage ensures 2140 * that NPIV is accounted for. 2141 */ 2142 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2143 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2144 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2145 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2146 2147 /* We need to tell the transport layer + 1 because it takes page 2148 * alignment into account. When space for the SGL is allocated we 2149 * allocate + 3, one for cmd, one for rsp and one for this alignment 2150 */ 2151 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2152 2153 /* Advertise how many hw queues we support based on cfg_hdw_queue, 2154 * which will not exceed cpu count. 2155 */ 2156 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 2157 2158 if (!IS_ENABLED(CONFIG_NVME_FC)) 2159 return ret; 2160 2161 /* localport is allocated from the stack, but the registration 2162 * call allocates heap memory as well as the private area. 2163 */ 2164 2165 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2166 &vport->phba->pcidev->dev, &localport); 2167 if (!ret) { 2168 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2169 "6005 Successfully registered local " 2170 "NVME port num %d, localP x%px, private " 2171 "x%px, sg_seg %d\n", 2172 localport->port_num, localport, 2173 localport->private, 2174 lpfc_nvme_template.max_sgl_segments); 2175 2176 /* Private is our lport size declared in the template. */ 2177 lport = (struct lpfc_nvme_lport *)localport->private; 2178 vport->localport = localport; 2179 lport->vport = vport; 2180 vport->nvmei_support = 1; 2181 2182 atomic_set(&lport->xmt_fcp_noxri, 0); 2183 atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 2184 atomic_set(&lport->xmt_fcp_qdepth, 0); 2185 atomic_set(&lport->xmt_fcp_err, 0); 2186 atomic_set(&lport->xmt_fcp_wqerr, 0); 2187 atomic_set(&lport->xmt_fcp_abort, 0); 2188 atomic_set(&lport->xmt_ls_abort, 0); 2189 atomic_set(&lport->xmt_ls_err, 0); 2190 atomic_set(&lport->cmpl_fcp_xb, 0); 2191 atomic_set(&lport->cmpl_fcp_err, 0); 2192 atomic_set(&lport->cmpl_ls_xb, 0); 2193 atomic_set(&lport->cmpl_ls_err, 0); 2194 atomic_set(&lport->fc4NvmeLsRequests, 0); 2195 atomic_set(&lport->fc4NvmeLsCmpls, 0); 2196 } 2197 2198 return ret; 2199 } 2200 2201 #if (IS_ENABLED(CONFIG_NVME_FC)) 2202 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2203 * 2204 * The driver has to wait for the host nvme transport to callback 2205 * indicating the localport has successfully unregistered all 2206 * resources. Since this is an uninterruptible wait, loop every ten 2207 * seconds and print a message indicating no progress. 2208 * 2209 * An uninterruptible wait is used because of the risk of transport-to- 2210 * driver state mismatch. 2211 */ 2212 static void 2213 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 2214 struct lpfc_nvme_lport *lport, 2215 struct completion *lport_unreg_cmp) 2216 { 2217 u32 wait_tmo; 2218 int ret, i, pending = 0; 2219 struct lpfc_sli_ring *pring; 2220 struct lpfc_hba *phba = vport->phba; 2221 2222 /* Host transport has to clean up and confirm requiring an indefinite 2223 * wait. Print a message if a 10 second wait expires and renew the 2224 * wait. This is unexpected. 2225 */ 2226 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2227 while (true) { 2228 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2229 if (unlikely(!ret)) { 2230 pending = 0; 2231 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2232 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2233 if (!pring) 2234 continue; 2235 if (pring->txcmplq_cnt) 2236 pending += pring->txcmplq_cnt; 2237 } 2238 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 2239 "6176 Lport x%px Localport x%px wait " 2240 "timed out. Pending %d. Renewing.\n", 2241 lport, vport->localport, pending); 2242 continue; 2243 } 2244 break; 2245 } 2246 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 2247 "6177 Lport x%px Localport x%px Complete Success\n", 2248 lport, vport->localport); 2249 } 2250 #endif 2251 2252 /** 2253 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2254 * @pnvme: pointer to lpfc nvme data structure. 2255 * 2256 * This routine is invoked to destroy all lports bound to the phba. 2257 * The lport memory was allocated by the nvme fc transport and is 2258 * released there. This routine ensures all rports bound to the 2259 * lport have been disconnected. 2260 * 2261 **/ 2262 void 2263 lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2264 { 2265 #if (IS_ENABLED(CONFIG_NVME_FC)) 2266 struct nvme_fc_local_port *localport; 2267 struct lpfc_nvme_lport *lport; 2268 int ret; 2269 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 2270 2271 if (vport->nvmei_support == 0) 2272 return; 2273 2274 localport = vport->localport; 2275 lport = (struct lpfc_nvme_lport *)localport->private; 2276 2277 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2278 "6011 Destroying NVME localport x%px\n", 2279 localport); 2280 2281 /* lport's rport list is clear. Unregister 2282 * lport and release resources. 2283 */ 2284 lport->lport_unreg_cmp = &lport_unreg_cmp; 2285 ret = nvme_fc_unregister_localport(localport); 2286 2287 /* Wait for completion. This either blocks 2288 * indefinitely or succeeds 2289 */ 2290 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 2291 vport->localport = NULL; 2292 2293 /* Regardless of the unregister upcall response, clear 2294 * nvmei_support. All rports are unregistered and the 2295 * driver will clean up. 2296 */ 2297 vport->nvmei_support = 0; 2298 if (ret == 0) { 2299 lpfc_printf_vlog(vport, 2300 KERN_INFO, LOG_NVME_DISC, 2301 "6009 Unregistered lport Success\n"); 2302 } else { 2303 lpfc_printf_vlog(vport, 2304 KERN_INFO, LOG_NVME_DISC, 2305 "6010 Unregistered lport " 2306 "Failed, status x%x\n", 2307 ret); 2308 } 2309 #endif 2310 } 2311 2312 void 2313 lpfc_nvme_update_localport(struct lpfc_vport *vport) 2314 { 2315 #if (IS_ENABLED(CONFIG_NVME_FC)) 2316 struct nvme_fc_local_port *localport; 2317 struct lpfc_nvme_lport *lport; 2318 2319 localport = vport->localport; 2320 if (!localport) { 2321 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2322 "6710 Update NVME fail. No localport\n"); 2323 return; 2324 } 2325 lport = (struct lpfc_nvme_lport *)localport->private; 2326 if (!lport) { 2327 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2328 "6171 Update NVME fail. localP x%px, No lport\n", 2329 localport); 2330 return; 2331 } 2332 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2333 "6012 Update NVME lport x%px did x%x\n", 2334 localport, vport->fc_myDID); 2335 2336 localport->port_id = vport->fc_myDID; 2337 if (localport->port_id == 0) 2338 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2339 else 2340 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2341 2342 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2343 "6030 bound lport x%px to DID x%06x\n", 2344 lport, localport->port_id); 2345 #endif 2346 } 2347 2348 int 2349 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2350 { 2351 #if (IS_ENABLED(CONFIG_NVME_FC)) 2352 int ret = 0; 2353 struct nvme_fc_local_port *localport; 2354 struct lpfc_nvme_lport *lport; 2355 struct lpfc_nvme_rport *rport; 2356 struct lpfc_nvme_rport *oldrport; 2357 struct nvme_fc_remote_port *remote_port; 2358 struct nvme_fc_port_info rpinfo; 2359 struct lpfc_nodelist *prev_ndlp = NULL; 2360 2361 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2362 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2363 ndlp->nlp_DID, ndlp->nlp_type); 2364 2365 localport = vport->localport; 2366 if (!localport) 2367 return 0; 2368 2369 lport = (struct lpfc_nvme_lport *)localport->private; 2370 2371 /* NVME rports are not preserved across devloss. 2372 * Just register this instance. Note, rpinfo->dev_loss_tmo 2373 * is left 0 to indicate accept transport defaults. The 2374 * driver communicates port role capabilities consistent 2375 * with the PRLI response data. 2376 */ 2377 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2378 rpinfo.port_id = ndlp->nlp_DID; 2379 if (ndlp->nlp_type & NLP_NVME_TARGET) 2380 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2381 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2382 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2383 2384 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2385 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2386 2387 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2388 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2389 2390 spin_lock_irq(&vport->phba->hbalock); 2391 oldrport = lpfc_ndlp_get_nrport(ndlp); 2392 if (oldrport) { 2393 prev_ndlp = oldrport->ndlp; 2394 spin_unlock_irq(&vport->phba->hbalock); 2395 } else { 2396 spin_unlock_irq(&vport->phba->hbalock); 2397 lpfc_nlp_get(ndlp); 2398 } 2399 2400 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2401 if (!ret) { 2402 /* If the ndlp already has an nrport, this is just 2403 * a resume of the existing rport. Else this is a 2404 * new rport. 2405 */ 2406 /* Guard against an unregister/reregister 2407 * race that leaves the WAIT flag set. 2408 */ 2409 spin_lock_irq(&vport->phba->hbalock); 2410 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2411 spin_unlock_irq(&vport->phba->hbalock); 2412 rport = remote_port->private; 2413 if (oldrport) { 2414 /* New remoteport record does not guarantee valid 2415 * host private memory area. 2416 */ 2417 if (oldrport == remote_port->private) { 2418 /* Same remoteport - ndlp should match. 2419 * Just reuse. 2420 */ 2421 lpfc_printf_vlog(ndlp->vport, KERN_INFO, 2422 LOG_NVME_DISC, 2423 "6014 Rebind lport to current " 2424 "remoteport x%px wwpn 0x%llx, " 2425 "Data: x%x x%x x%px x%px x%x " 2426 " x%06x\n", 2427 remote_port, 2428 remote_port->port_name, 2429 remote_port->port_id, 2430 remote_port->port_role, 2431 oldrport->ndlp, 2432 ndlp, 2433 ndlp->nlp_type, 2434 ndlp->nlp_DID); 2435 2436 /* It's a complete rebind only if the driver 2437 * is registering with the same ndlp. Otherwise 2438 * the driver likely executed a node swap 2439 * prior to this registration and the ndlp to 2440 * remoteport binding needs to be redone. 2441 */ 2442 if (prev_ndlp == ndlp) 2443 return 0; 2444 2445 } 2446 2447 /* Sever the ndlp<->rport association 2448 * before dropping the ndlp ref from 2449 * register. 2450 */ 2451 spin_lock_irq(&vport->phba->hbalock); 2452 ndlp->nrport = NULL; 2453 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2454 spin_unlock_irq(&vport->phba->hbalock); 2455 rport->ndlp = NULL; 2456 rport->remoteport = NULL; 2457 2458 /* Reference only removed if previous NDLP is no longer 2459 * active. It might be just a swap and removing the 2460 * reference would cause a premature cleanup. 2461 */ 2462 if (prev_ndlp && prev_ndlp != ndlp) { 2463 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) || 2464 (!prev_ndlp->nrport)) 2465 lpfc_nlp_put(prev_ndlp); 2466 } 2467 } 2468 2469 /* Clean bind the rport to the ndlp. */ 2470 rport->remoteport = remote_port; 2471 rport->lport = lport; 2472 rport->ndlp = ndlp; 2473 spin_lock_irq(&vport->phba->hbalock); 2474 ndlp->nrport = rport; 2475 spin_unlock_irq(&vport->phba->hbalock); 2476 lpfc_printf_vlog(vport, KERN_INFO, 2477 LOG_NVME_DISC | LOG_NODE, 2478 "6022 Bind lport x%px to remoteport x%px " 2479 "rport x%px WWNN 0x%llx, " 2480 "Rport WWPN 0x%llx DID " 2481 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 2482 lport, remote_port, rport, 2483 rpinfo.node_name, rpinfo.port_name, 2484 rpinfo.port_id, rpinfo.port_role, 2485 ndlp, prev_ndlp); 2486 } else { 2487 lpfc_printf_vlog(vport, KERN_ERR, 2488 LOG_NVME_DISC | LOG_NODE, 2489 "6031 RemotePort Registration failed " 2490 "err: %d, DID x%06x\n", 2491 ret, ndlp->nlp_DID); 2492 } 2493 2494 return ret; 2495 #else 2496 return 0; 2497 #endif 2498 } 2499 2500 /** 2501 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 2502 * 2503 * If the ndlp represents an NVME Target, that we are logged into, 2504 * ping the NVME FC Transport layer to initiate a device rescan 2505 * on this remote NPort. 2506 */ 2507 void 2508 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2509 { 2510 #if (IS_ENABLED(CONFIG_NVME_FC)) 2511 struct lpfc_nvme_rport *nrport; 2512 struct nvme_fc_remote_port *remoteport = NULL; 2513 2514 spin_lock_irq(&vport->phba->hbalock); 2515 nrport = lpfc_ndlp_get_nrport(ndlp); 2516 if (nrport) 2517 remoteport = nrport->remoteport; 2518 spin_unlock_irq(&vport->phba->hbalock); 2519 2520 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2521 "6170 Rescan NPort DID x%06x type x%x " 2522 "state x%x nrport x%px remoteport x%px\n", 2523 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2524 nrport, remoteport); 2525 2526 if (!nrport || !remoteport) 2527 goto rescan_exit; 2528 2529 /* Only rescan if we are an NVME target in the MAPPED state */ 2530 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 2531 ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 2532 nvme_fc_rescan_remoteport(remoteport); 2533 2534 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2535 "6172 NVME rescanned DID x%06x " 2536 "port_state x%x\n", 2537 ndlp->nlp_DID, remoteport->port_state); 2538 } 2539 return; 2540 rescan_exit: 2541 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2542 "6169 Skip NVME Rport Rescan, NVME remoteport " 2543 "unregistered\n"); 2544 #endif 2545 } 2546 2547 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2548 * 2549 * There is no notion of Devloss or rport recovery from the current 2550 * nvme_transport perspective. Loss of an rport just means IO cannot 2551 * be sent and recovery is completely up to the initator. 2552 * For now, the driver just unbinds the DID and port_role so that 2553 * no further IO can be issued. Changes are planned for later. 2554 * 2555 * Notes - the ndlp reference count is not decremented here since 2556 * since there is no nvme_transport api for devloss. Node ref count 2557 * is only adjusted in driver unload. 2558 */ 2559 void 2560 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2561 { 2562 #if (IS_ENABLED(CONFIG_NVME_FC)) 2563 int ret; 2564 struct nvme_fc_local_port *localport; 2565 struct lpfc_nvme_lport *lport; 2566 struct lpfc_nvme_rport *rport; 2567 struct nvme_fc_remote_port *remoteport = NULL; 2568 2569 localport = vport->localport; 2570 2571 /* This is fundamental error. The localport is always 2572 * available until driver unload. Just exit. 2573 */ 2574 if (!localport) 2575 return; 2576 2577 lport = (struct lpfc_nvme_lport *)localport->private; 2578 if (!lport) 2579 goto input_err; 2580 2581 spin_lock_irq(&vport->phba->hbalock); 2582 rport = lpfc_ndlp_get_nrport(ndlp); 2583 if (rport) 2584 remoteport = rport->remoteport; 2585 spin_unlock_irq(&vport->phba->hbalock); 2586 if (!remoteport) 2587 goto input_err; 2588 2589 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2590 "6033 Unreg nvme remoteport x%px, portname x%llx, " 2591 "port_id x%06x, portstate x%x port type x%x\n", 2592 remoteport, remoteport->port_name, 2593 remoteport->port_id, remoteport->port_state, 2594 ndlp->nlp_type); 2595 2596 /* Sanity check ndlp type. Only call for NVME ports. Don't 2597 * clear any rport state until the transport calls back. 2598 */ 2599 2600 if (ndlp->nlp_type & NLP_NVME_TARGET) { 2601 /* No concern about the role change on the nvme remoteport. 2602 * The transport will update it. 2603 */ 2604 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG; 2605 2606 /* Don't let the host nvme transport keep sending keep-alives 2607 * on this remoteport. Vport is unloading, no recovery. The 2608 * return values is ignored. The upcall is a courtesy to the 2609 * transport. 2610 */ 2611 if (vport->load_flag & FC_UNLOADING) 2612 (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 2613 2614 ret = nvme_fc_unregister_remoteport(remoteport); 2615 if (ret != 0) { 2616 lpfc_nlp_put(ndlp); 2617 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2618 "6167 NVME unregister failed %d " 2619 "port_state x%x\n", 2620 ret, remoteport->port_state); 2621 } 2622 } 2623 return; 2624 2625 input_err: 2626 #endif 2627 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2628 "6168 State error: lport x%px, rport x%px FCID x%06x\n", 2629 vport->localport, ndlp->rport, ndlp->nlp_DID); 2630 } 2631 2632 /** 2633 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2634 * @phba: pointer to lpfc hba data structure. 2635 * @axri: pointer to the fcp xri abort wcqe structure. 2636 * @lpfc_ncmd: The nvme job structure for the request being aborted. 2637 * 2638 * This routine is invoked by the worker thread to process a SLI4 fast-path 2639 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2640 * here. 2641 **/ 2642 void 2643 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2644 struct sli4_wcqe_xri_aborted *axri, 2645 struct lpfc_io_buf *lpfc_ncmd) 2646 { 2647 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2648 struct nvmefc_fcp_req *nvme_cmd = NULL; 2649 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2650 2651 2652 if (ndlp) 2653 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2654 2655 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2656 "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2657 "xri released\n", 2658 lpfc_ncmd->nvmeCmd, xri, 2659 lpfc_ncmd->cur_iocbq.iotag); 2660 2661 /* Aborted NVME commands are required to not complete 2662 * before the abort exchange command fully completes. 2663 * Once completed, it is available via the put list. 2664 */ 2665 if (lpfc_ncmd->nvmeCmd) { 2666 nvme_cmd = lpfc_ncmd->nvmeCmd; 2667 nvme_cmd->done(nvme_cmd); 2668 lpfc_ncmd->nvmeCmd = NULL; 2669 } 2670 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2671 } 2672 2673 /** 2674 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2675 * @phba: Pointer to HBA context object. 2676 * 2677 * This function flushes all wqes in the nvme rings and frees all resources 2678 * in the txcmplq. This function does not issue abort wqes for the IO 2679 * commands in txcmplq, they will just be returned with 2680 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2681 * slot has been permanently disabled. 2682 **/ 2683 void 2684 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2685 { 2686 struct lpfc_sli_ring *pring; 2687 u32 i, wait_cnt = 0; 2688 2689 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2690 return; 2691 2692 /* Cycle through all IO rings and make sure all outstanding 2693 * WQEs have been removed from the txcmplqs. 2694 */ 2695 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2696 if (!phba->sli4_hba.hdwq[i].io_wq) 2697 continue; 2698 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2699 2700 if (!pring) 2701 continue; 2702 2703 /* Retrieve everything on the txcmplq */ 2704 while (!list_empty(&pring->txcmplq)) { 2705 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2706 wait_cnt++; 2707 2708 /* The sleep is 10mS. Every ten seconds, 2709 * dump a message. Something is wrong. 2710 */ 2711 if ((wait_cnt % 1000) == 0) { 2712 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2713 "6178 NVME IO not empty, " 2714 "cnt %d\n", wait_cnt); 2715 } 2716 } 2717 } 2718 } 2719 2720 void 2721 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn) 2722 { 2723 #if (IS_ENABLED(CONFIG_NVME_FC)) 2724 struct lpfc_io_buf *lpfc_ncmd; 2725 struct nvmefc_fcp_req *nCmd; 2726 struct lpfc_nvme_fcpreq_priv *freqpriv; 2727 2728 if (!pwqeIn->context1) { 2729 lpfc_sli_release_iocbq(phba, pwqeIn); 2730 return; 2731 } 2732 /* For abort iocb just return, IO iocb will do a done call */ 2733 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 2734 CMD_ABORT_XRI_CX) { 2735 lpfc_sli_release_iocbq(phba, pwqeIn); 2736 return; 2737 } 2738 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1; 2739 2740 spin_lock(&lpfc_ncmd->buf_lock); 2741 if (!lpfc_ncmd->nvmeCmd) { 2742 spin_unlock(&lpfc_ncmd->buf_lock); 2743 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2744 return; 2745 } 2746 2747 nCmd = lpfc_ncmd->nvmeCmd; 2748 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2749 "6194 NVME Cancel xri %x\n", 2750 lpfc_ncmd->cur_iocbq.sli4_xritag); 2751 2752 nCmd->transferred_length = 0; 2753 nCmd->rcv_rsplen = 0; 2754 nCmd->status = NVME_SC_INTERNAL; 2755 freqpriv = nCmd->private; 2756 freqpriv->nvme_buf = NULL; 2757 lpfc_ncmd->nvmeCmd = NULL; 2758 2759 spin_unlock(&lpfc_ncmd->buf_lock); 2760 nCmd->done(nCmd); 2761 2762 /* Call release with XB=1 to queue the IO into the abort list. */ 2763 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2764 #endif 2765 } 2766