1 /******************************************************************* 2 * This file is part of the Emulex Linux Device Driver for * 3 * Fibre Channel Host Bus Adapters. * 4 * Copyright (C) 2017-2019 Broadcom. All Rights Reserved. The term * 5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. * 6 * Copyright (C) 2004-2016 Emulex. All rights reserved. * 7 * EMULEX and SLI are trademarks of Emulex. * 8 * www.broadcom.com * 9 * Portions Copyright (C) 2004-2005 Christoph Hellwig * 10 * * 11 * This program is free software; you can redistribute it and/or * 12 * modify it under the terms of version 2 of the GNU General * 13 * Public License as published by the Free Software Foundation. * 14 * This program is distributed in the hope that it will be useful. * 15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * 16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * 17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * 18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * 19 * TO BE LEGALLY INVALID. See the GNU General Public License for * 20 * more details, a copy of which can be found in the file COPYING * 21 * included with this package. * 22 ********************************************************************/ 23 #include <linux/pci.h> 24 #include <linux/slab.h> 25 #include <linux/interrupt.h> 26 #include <linux/delay.h> 27 #include <asm/unaligned.h> 28 #include <linux/crc-t10dif.h> 29 #include <net/checksum.h> 30 31 #include <scsi/scsi.h> 32 #include <scsi/scsi_device.h> 33 #include <scsi/scsi_eh.h> 34 #include <scsi/scsi_host.h> 35 #include <scsi/scsi_tcq.h> 36 #include <scsi/scsi_transport_fc.h> 37 #include <scsi/fc/fc_fs.h> 38 39 #include "lpfc_version.h" 40 #include "lpfc_hw4.h" 41 #include "lpfc_hw.h" 42 #include "lpfc_sli.h" 43 #include "lpfc_sli4.h" 44 #include "lpfc_nl.h" 45 #include "lpfc_disc.h" 46 #include "lpfc.h" 47 #include "lpfc_nvme.h" 48 #include "lpfc_scsi.h" 49 #include "lpfc_logmsg.h" 50 #include "lpfc_crtn.h" 51 #include "lpfc_vport.h" 52 #include "lpfc_debugfs.h" 53 54 /* NVME initiator-based functions */ 55 56 static struct lpfc_io_buf * 57 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 58 int idx, int expedite); 59 60 static void 61 lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *); 62 63 static struct nvme_fc_port_template lpfc_nvme_template; 64 65 static union lpfc_wqe128 lpfc_iread_cmd_template; 66 static union lpfc_wqe128 lpfc_iwrite_cmd_template; 67 static union lpfc_wqe128 lpfc_icmnd_cmd_template; 68 69 /* Setup WQE templates for NVME IOs */ 70 void 71 lpfc_nvme_cmd_template(void) 72 { 73 union lpfc_wqe128 *wqe; 74 75 /* IREAD template */ 76 wqe = &lpfc_iread_cmd_template; 77 memset(wqe, 0, sizeof(union lpfc_wqe128)); 78 79 /* Word 0, 1, 2 - BDE is variable */ 80 81 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 82 83 /* Word 4 - total_xfer_len is variable */ 84 85 /* Word 5 - is zero */ 86 87 /* Word 6 - ctxt_tag, xri_tag is variable */ 88 89 /* Word 7 */ 90 bf_set(wqe_cmnd, &wqe->fcp_iread.wqe_com, CMD_FCP_IREAD64_WQE); 91 bf_set(wqe_pu, &wqe->fcp_iread.wqe_com, PARM_READ_CHECK); 92 bf_set(wqe_class, &wqe->fcp_iread.wqe_com, CLASS3); 93 bf_set(wqe_ct, &wqe->fcp_iread.wqe_com, SLI4_CT_RPI); 94 95 /* Word 8 - abort_tag is variable */ 96 97 /* Word 9 - reqtag is variable */ 98 99 /* Word 10 - dbde, wqes is variable */ 100 bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); 101 bf_set(wqe_nvme, &wqe->fcp_iread.wqe_com, 1); 102 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, LPFC_WQE_IOD_READ); 103 bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, LPFC_WQE_LENLOC_WORD4); 104 bf_set(wqe_dbde, &wqe->fcp_iread.wqe_com, 0); 105 bf_set(wqe_wqes, &wqe->fcp_iread.wqe_com, 1); 106 107 /* Word 11 - pbde is variable */ 108 bf_set(wqe_cmd_type, &wqe->fcp_iread.wqe_com, NVME_READ_CMD); 109 bf_set(wqe_cqid, &wqe->fcp_iread.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 110 bf_set(wqe_pbde, &wqe->fcp_iread.wqe_com, 1); 111 112 /* Word 12 - is zero */ 113 114 /* Word 13, 14, 15 - PBDE is variable */ 115 116 /* IWRITE template */ 117 wqe = &lpfc_iwrite_cmd_template; 118 memset(wqe, 0, sizeof(union lpfc_wqe128)); 119 120 /* Word 0, 1, 2 - BDE is variable */ 121 122 /* Word 3 - cmd_buff_len, payload_offset_len is zero */ 123 124 /* Word 4 - total_xfer_len is variable */ 125 126 /* Word 5 - initial_xfer_len is variable */ 127 128 /* Word 6 - ctxt_tag, xri_tag is variable */ 129 130 /* Word 7 */ 131 bf_set(wqe_cmnd, &wqe->fcp_iwrite.wqe_com, CMD_FCP_IWRITE64_WQE); 132 bf_set(wqe_pu, &wqe->fcp_iwrite.wqe_com, PARM_READ_CHECK); 133 bf_set(wqe_class, &wqe->fcp_iwrite.wqe_com, CLASS3); 134 bf_set(wqe_ct, &wqe->fcp_iwrite.wqe_com, SLI4_CT_RPI); 135 136 /* Word 8 - abort_tag is variable */ 137 138 /* Word 9 - reqtag is variable */ 139 140 /* Word 10 - dbde, wqes is variable */ 141 bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); 142 bf_set(wqe_nvme, &wqe->fcp_iwrite.wqe_com, 1); 143 bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_IOD_WRITE); 144 bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_LENLOC_WORD4); 145 bf_set(wqe_dbde, &wqe->fcp_iwrite.wqe_com, 0); 146 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1); 147 148 /* Word 11 - pbde is variable */ 149 bf_set(wqe_cmd_type, &wqe->fcp_iwrite.wqe_com, NVME_WRITE_CMD); 150 bf_set(wqe_cqid, &wqe->fcp_iwrite.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 151 bf_set(wqe_pbde, &wqe->fcp_iwrite.wqe_com, 1); 152 153 /* Word 12 - is zero */ 154 155 /* Word 13, 14, 15 - PBDE is variable */ 156 157 /* ICMND template */ 158 wqe = &lpfc_icmnd_cmd_template; 159 memset(wqe, 0, sizeof(union lpfc_wqe128)); 160 161 /* Word 0, 1, 2 - BDE is variable */ 162 163 /* Word 3 - payload_offset_len is variable */ 164 165 /* Word 4, 5 - is zero */ 166 167 /* Word 6 - ctxt_tag, xri_tag is variable */ 168 169 /* Word 7 */ 170 bf_set(wqe_cmnd, &wqe->fcp_icmd.wqe_com, CMD_FCP_ICMND64_WQE); 171 bf_set(wqe_pu, &wqe->fcp_icmd.wqe_com, 0); 172 bf_set(wqe_class, &wqe->fcp_icmd.wqe_com, CLASS3); 173 bf_set(wqe_ct, &wqe->fcp_icmd.wqe_com, SLI4_CT_RPI); 174 175 /* Word 8 - abort_tag is variable */ 176 177 /* Word 9 - reqtag is variable */ 178 179 /* Word 10 - dbde, wqes is variable */ 180 bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); 181 bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); 182 bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_NONE); 183 bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, LPFC_WQE_LENLOC_NONE); 184 bf_set(wqe_dbde, &wqe->fcp_icmd.wqe_com, 0); 185 bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); 186 187 /* Word 11 */ 188 bf_set(wqe_cmd_type, &wqe->fcp_icmd.wqe_com, FCP_COMMAND); 189 bf_set(wqe_cqid, &wqe->fcp_icmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 190 bf_set(wqe_pbde, &wqe->fcp_icmd.wqe_com, 0); 191 192 /* Word 12, 13, 14, 15 - is zero */ 193 } 194 195 /** 196 * lpfc_nvme_prep_abort_wqe - set up 'abort' work queue entry. 197 * @pwqeq: Pointer to command iocb. 198 * @xritag: Tag that uniqely identifies the local exchange resource. 199 * @opt: Option bits - 200 * bit 0 = inhibit sending abts on the link 201 * 202 * This function is called with hbalock held. 203 **/ 204 void 205 lpfc_nvme_prep_abort_wqe(struct lpfc_iocbq *pwqeq, u16 xritag, u8 opt) 206 { 207 union lpfc_wqe128 *wqe = &pwqeq->wqe; 208 209 /* WQEs are reused. Clear stale data and set key fields to 210 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. 211 */ 212 memset(wqe, 0, sizeof(*wqe)); 213 214 if (opt & INHIBIT_ABORT) 215 bf_set(abort_cmd_ia, &wqe->abort_cmd, 1); 216 /* Abort specified xri tag, with the mask deliberately zeroed */ 217 bf_set(abort_cmd_criteria, &wqe->abort_cmd, T_XRI_TAG); 218 219 bf_set(wqe_cmnd, &wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); 220 221 /* Abort the IO associated with this outstanding exchange ID. */ 222 wqe->abort_cmd.wqe_com.abort_tag = xritag; 223 224 /* iotag for the wqe completion. */ 225 bf_set(wqe_reqtag, &wqe->abort_cmd.wqe_com, pwqeq->iotag); 226 227 bf_set(wqe_qosd, &wqe->abort_cmd.wqe_com, 1); 228 bf_set(wqe_lenloc, &wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); 229 230 bf_set(wqe_cmd_type, &wqe->abort_cmd.wqe_com, OTHER_COMMAND); 231 bf_set(wqe_wqec, &wqe->abort_cmd.wqe_com, 1); 232 bf_set(wqe_cqid, &wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 233 } 234 235 /** 236 * lpfc_nvme_create_queue - 237 * @lpfc_pnvme: Pointer to the driver's nvme instance data 238 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 239 * @handle: An opaque driver handle used in follow-up calls. 240 * 241 * Driver registers this routine to preallocate and initialize any 242 * internal data structures to bind the @qidx to its internal IO queues. 243 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. 244 * 245 * Return value : 246 * 0 - Success 247 * -EINVAL - Unsupported input value. 248 * -ENOMEM - Could not alloc necessary memory 249 **/ 250 static int 251 lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, 252 unsigned int qidx, u16 qsize, 253 void **handle) 254 { 255 struct lpfc_nvme_lport *lport; 256 struct lpfc_vport *vport; 257 struct lpfc_nvme_qhandle *qhandle; 258 char *str; 259 260 if (!pnvme_lport->private) 261 return -ENOMEM; 262 263 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 264 vport = lport->vport; 265 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); 266 if (qhandle == NULL) 267 return -ENOMEM; 268 269 qhandle->cpu_id = raw_smp_processor_id(); 270 qhandle->qidx = qidx; 271 /* 272 * NVME qidx == 0 is the admin queue, so both admin queue 273 * and first IO queue will use MSI-X vector and associated 274 * EQ/CQ/WQ at index 0. After that they are sequentially assigned. 275 */ 276 if (qidx) { 277 str = "IO "; /* IO queue */ 278 qhandle->index = ((qidx - 1) % 279 lpfc_nvme_template.max_hw_queues); 280 } else { 281 str = "ADM"; /* Admin queue */ 282 qhandle->index = qidx; 283 } 284 285 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 286 "6073 Binding %s HdwQueue %d (cpu %d) to " 287 "hdw_queue %d qhandle x%px\n", str, 288 qidx, qhandle->cpu_id, qhandle->index, qhandle); 289 *handle = (void *)qhandle; 290 return 0; 291 } 292 293 /** 294 * lpfc_nvme_delete_queue - 295 * @lpfc_pnvme: Pointer to the driver's nvme instance data 296 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. 297 * @handle: An opaque driver handle from lpfc_nvme_create_queue 298 * 299 * Driver registers this routine to free 300 * any internal data structures to bind the @qidx to its internal 301 * IO queues. 302 * 303 * Return value : 304 * 0 - Success 305 * TODO: What are the failure codes. 306 **/ 307 static void 308 lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, 309 unsigned int qidx, 310 void *handle) 311 { 312 struct lpfc_nvme_lport *lport; 313 struct lpfc_vport *vport; 314 315 if (!pnvme_lport->private) 316 return; 317 318 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 319 vport = lport->vport; 320 321 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 322 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n", 323 lport, qidx, handle); 324 kfree(handle); 325 } 326 327 static void 328 lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) 329 { 330 struct lpfc_nvme_lport *lport = localport->private; 331 332 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME, 333 "6173 localport x%px delete complete\n", 334 lport); 335 336 /* release any threads waiting for the unreg to complete */ 337 if (lport->vport->localport) 338 complete(lport->lport_unreg_cmp); 339 } 340 341 /* lpfc_nvme_remoteport_delete 342 * 343 * @remoteport: Pointer to an nvme transport remoteport instance. 344 * 345 * This is a template downcall. NVME transport calls this function 346 * when it has completed the unregistration of a previously 347 * registered remoteport. 348 * 349 * Return value : 350 * None 351 */ 352 static void 353 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) 354 { 355 struct lpfc_nvme_rport *rport = remoteport->private; 356 struct lpfc_vport *vport; 357 struct lpfc_nodelist *ndlp; 358 359 ndlp = rport->ndlp; 360 if (!ndlp) 361 goto rport_err; 362 363 vport = ndlp->vport; 364 if (!vport) 365 goto rport_err; 366 367 /* Remove this rport from the lport's list - memory is owned by the 368 * transport. Remove the ndlp reference for the NVME transport before 369 * calling state machine to remove the node. 370 */ 371 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 372 "6146 remoteport delete of remoteport x%px\n", 373 remoteport); 374 spin_lock_irq(&vport->phba->hbalock); 375 376 /* The register rebind might have occurred before the delete 377 * downcall. Guard against this race. 378 */ 379 if (ndlp->upcall_flags & NLP_WAIT_FOR_UNREG) { 380 ndlp->nrport = NULL; 381 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 382 spin_unlock_irq(&vport->phba->hbalock); 383 384 /* Remove original register reference. The host transport 385 * won't reference this rport/remoteport any further. 386 */ 387 lpfc_nlp_put(ndlp); 388 } else { 389 spin_unlock_irq(&vport->phba->hbalock); 390 } 391 392 rport_err: 393 return; 394 } 395 396 /** 397 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request 398 * @phba: pointer to lpfc hba data structure. 399 * @axchg: pointer to exchange context for the NVME LS request 400 * 401 * This routine is used for processing an asychronously received NVME LS 402 * request. Any remaining validation is done and the LS is then forwarded 403 * to the nvme-fc transport via nvme_fc_rcv_ls_req(). 404 * 405 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing) 406 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done. 407 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg. 408 * 409 * Returns 0 if LS was handled and delivered to the transport 410 * Returns 1 if LS failed to be handled and should be dropped 411 */ 412 int 413 lpfc_nvme_handle_lsreq(struct lpfc_hba *phba, 414 struct lpfc_async_xchg_ctx *axchg) 415 { 416 #if (IS_ENABLED(CONFIG_NVME_FC)) 417 struct lpfc_vport *vport; 418 struct lpfc_nvme_rport *lpfc_rport; 419 struct nvme_fc_remote_port *remoteport; 420 struct lpfc_nvme_lport *lport; 421 uint32_t *payload = axchg->payload; 422 int rc; 423 424 vport = axchg->ndlp->vport; 425 lpfc_rport = axchg->ndlp->nrport; 426 if (!lpfc_rport) 427 return -EINVAL; 428 429 remoteport = lpfc_rport->remoteport; 430 if (!vport->localport) 431 return -EINVAL; 432 433 lport = vport->localport->private; 434 if (!lport) 435 return -EINVAL; 436 437 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload, 438 axchg->size); 439 440 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC, 441 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x " 442 "%08x %08x %08x\n", 443 axchg->size, rc, 444 *payload, *(payload+1), *(payload+2), 445 *(payload+3), *(payload+4), *(payload+5)); 446 447 if (!rc) 448 return 0; 449 #endif 450 return 1; 451 } 452 453 /** 454 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME 455 * LS request. 456 * @phba: Pointer to HBA context object 457 * @vport: The local port that issued the LS 458 * @cmdwqe: Pointer to driver command WQE object. 459 * @wcqe: Pointer to driver response CQE object. 460 * 461 * This function is the generic completion handler for NVME LS requests. 462 * The function updates any states and statistics, calls the transport 463 * ls_req done() routine, then tears down the command and buffers used 464 * for the LS request. 465 **/ 466 void 467 __lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport, 468 struct lpfc_iocbq *cmdwqe, 469 struct lpfc_wcqe_complete *wcqe) 470 { 471 struct nvmefc_ls_req *pnvme_lsreq; 472 struct lpfc_dmabuf *buf_ptr; 473 struct lpfc_nodelist *ndlp; 474 uint32_t status; 475 476 pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; 477 ndlp = (struct lpfc_nodelist *)cmdwqe->context1; 478 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 479 480 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 481 "6047 NVMEx LS REQ %px cmpl DID %x Xri: %x " 482 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px " 483 "ndlp:x%px\n", 484 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 485 cmdwqe->sli4_xritag, status, 486 (wcqe->parameter & 0xffff), 487 cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); 488 489 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n", 490 cmdwqe->sli4_xritag, status, wcqe->parameter); 491 492 if (cmdwqe->context3) { 493 buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; 494 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); 495 kfree(buf_ptr); 496 cmdwqe->context3 = NULL; 497 } 498 if (pnvme_lsreq->done) 499 pnvme_lsreq->done(pnvme_lsreq, status); 500 else 501 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 502 "6046 NVMEx cmpl without done call back? " 503 "Data %px DID %x Xri: %x status %x\n", 504 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, 505 cmdwqe->sli4_xritag, status); 506 if (ndlp) { 507 lpfc_nlp_put(ndlp); 508 cmdwqe->context1 = NULL; 509 } 510 lpfc_sli_release_iocbq(phba, cmdwqe); 511 } 512 513 static void 514 lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, 515 struct lpfc_wcqe_complete *wcqe) 516 { 517 struct lpfc_vport *vport = cmdwqe->vport; 518 struct lpfc_nvme_lport *lport; 519 uint32_t status; 520 521 status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; 522 523 if (vport->localport) { 524 lport = (struct lpfc_nvme_lport *)vport->localport->private; 525 if (lport) { 526 atomic_inc(&lport->fc4NvmeLsCmpls); 527 if (status) { 528 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 529 atomic_inc(&lport->cmpl_ls_xb); 530 atomic_inc(&lport->cmpl_ls_err); 531 } 532 } 533 } 534 535 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe); 536 } 537 538 static int 539 lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, 540 struct lpfc_dmabuf *inp, 541 struct nvmefc_ls_req *pnvme_lsreq, 542 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, 543 struct lpfc_wcqe_complete *), 544 struct lpfc_nodelist *ndlp, uint32_t num_entry, 545 uint32_t tmo, uint8_t retry) 546 { 547 struct lpfc_hba *phba = vport->phba; 548 union lpfc_wqe128 *wqe; 549 struct lpfc_iocbq *genwqe; 550 struct ulp_bde64 *bpl; 551 struct ulp_bde64 bde; 552 int i, rc, xmit_len, first_len; 553 554 /* Allocate buffer for command WQE */ 555 genwqe = lpfc_sli_get_iocbq(phba); 556 if (genwqe == NULL) 557 return 1; 558 559 wqe = &genwqe->wqe; 560 /* Initialize only 64 bytes */ 561 memset(wqe, 0, sizeof(union lpfc_wqe)); 562 563 genwqe->context3 = (uint8_t *)bmp; 564 genwqe->iocb_flag |= LPFC_IO_NVME_LS; 565 566 /* Save for completion so we can release these resources */ 567 genwqe->context1 = lpfc_nlp_get(ndlp); 568 genwqe->context2 = (uint8_t *)pnvme_lsreq; 569 /* Fill in payload, bp points to frame payload */ 570 571 if (!tmo) 572 /* FC spec states we need 3 * ratov for CT requests */ 573 tmo = (3 * phba->fc_ratov); 574 575 /* For this command calculate the xmit length of the request bde. */ 576 xmit_len = 0; 577 first_len = 0; 578 bpl = (struct ulp_bde64 *)bmp->virt; 579 for (i = 0; i < num_entry; i++) { 580 bde.tus.w = bpl[i].tus.w; 581 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) 582 break; 583 xmit_len += bde.tus.f.bdeSize; 584 if (i == 0) 585 first_len = xmit_len; 586 } 587 588 genwqe->rsvd2 = num_entry; 589 genwqe->hba_wqidx = 0; 590 591 /* Words 0 - 2 */ 592 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 593 wqe->generic.bde.tus.f.bdeSize = first_len; 594 wqe->generic.bde.addrLow = bpl[0].addrLow; 595 wqe->generic.bde.addrHigh = bpl[0].addrHigh; 596 597 /* Word 3 */ 598 wqe->gen_req.request_payload_len = first_len; 599 600 /* Word 4 */ 601 602 /* Word 5 */ 603 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); 604 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); 605 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); 606 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); 607 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); 608 609 /* Word 6 */ 610 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, 611 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); 612 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); 613 614 /* Word 7 */ 615 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); 616 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); 617 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); 618 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); 619 620 /* Word 8 */ 621 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; 622 623 /* Word 9 */ 624 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); 625 626 /* Word 10 */ 627 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); 628 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); 629 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); 630 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); 631 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); 632 633 /* Word 11 */ 634 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); 635 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); 636 637 638 /* Issue GEN REQ WQE for NPORT <did> */ 639 genwqe->wqe_cmpl = cmpl; 640 genwqe->iocb_cmpl = NULL; 641 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; 642 genwqe->vport = vport; 643 genwqe->retry = retry; 644 645 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", 646 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); 647 648 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe); 649 if (rc) { 650 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC | LOG_ELS, 651 "6045 Issue GEN REQ WQE to NPORT x%x " 652 "Data: x%x x%x rc x%x\n", 653 ndlp->nlp_DID, genwqe->iotag, 654 vport->port_state, rc); 655 lpfc_sli_release_iocbq(phba, genwqe); 656 return 1; 657 } 658 659 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS, 660 "6050 Issue GEN REQ WQE to NPORT x%x " 661 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px " 662 "bmp:x%px xmit:%d 1st:%d\n", 663 ndlp->nlp_DID, genwqe->sli4_xritag, 664 vport->port_state, 665 genwqe, pnvme_lsreq, bmp, xmit_len, first_len); 666 return 0; 667 } 668 669 670 /** 671 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request 672 * @vport: The local port issuing the LS 673 * @ndlp: The remote port to send the LS to 674 * @pnvme_lsreq: Pointer to LS request structure from the transport 675 * 676 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST 677 * WQE to perform the LS operation. 678 * 679 * Return value : 680 * 0 - Success 681 * non-zero: various error codes, in form of -Exxx 682 **/ 683 int 684 __lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 685 struct nvmefc_ls_req *pnvme_lsreq, 686 void (*gen_req_cmp)(struct lpfc_hba *phba, 687 struct lpfc_iocbq *cmdwqe, 688 struct lpfc_wcqe_complete *wcqe)) 689 { 690 struct lpfc_dmabuf *bmp; 691 struct ulp_bde64 *bpl; 692 int ret; 693 uint16_t ntype, nstate; 694 695 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 696 lpfc_printf_vlog(vport, KERN_ERR, 697 LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, 698 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing " 699 "LS Req\n", 700 ndlp); 701 return -ENODEV; 702 } 703 704 ntype = ndlp->nlp_type; 705 nstate = ndlp->nlp_state; 706 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || 707 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { 708 lpfc_printf_vlog(vport, KERN_ERR, 709 LOG_NVME_DISC | LOG_NODE | LOG_NVME_IOERR, 710 "6088 NVMEx LS REQ: Fail DID x%06x not " 711 "ready for IO. Type x%x, State x%x\n", 712 ndlp->nlp_DID, ntype, nstate); 713 return -ENODEV; 714 } 715 716 /* 717 * there are two dma buf in the request, actually there is one and 718 * the second one is just the start address + cmd size. 719 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped 720 * in a lpfc_dmabuf struct. When freeing we just free the wrapper 721 * because the nvem layer owns the data bufs. 722 * We do not have to break these packets open, we don't care what is 723 * in them. And we do not have to look at the resonse data, we only 724 * care that we got a response. All of the caring is going to happen 725 * in the nvme-fc layer. 726 */ 727 728 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL); 729 if (!bmp) { 730 731 lpfc_printf_vlog(vport, KERN_ERR, 732 LOG_NVME_DISC | LOG_NVME_IOERR, 733 "6044 NVMEx LS REQ: Could not alloc LS buf " 734 "for DID %x\n", 735 ndlp->nlp_DID); 736 return -ENOMEM; 737 } 738 739 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); 740 if (!bmp->virt) { 741 lpfc_printf_vlog(vport, KERN_ERR, 742 LOG_NVME_DISC | LOG_NVME_IOERR, 743 "6042 NVMEx LS REQ: Could not alloc mbuf " 744 "for DID %x\n", 745 ndlp->nlp_DID); 746 kfree(bmp); 747 return -ENOMEM; 748 } 749 750 INIT_LIST_HEAD(&bmp->list); 751 752 bpl = (struct ulp_bde64 *)bmp->virt; 753 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); 754 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); 755 bpl->tus.f.bdeFlags = 0; 756 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; 757 bpl->tus.w = le32_to_cpu(bpl->tus.w); 758 bpl++; 759 760 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); 761 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); 762 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; 763 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; 764 bpl->tus.w = le32_to_cpu(bpl->tus.w); 765 766 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 767 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, " 768 "rqstlen:%d rsplen:%d %pad %pad\n", 769 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen, 770 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 771 &pnvme_lsreq->rspdma); 772 773 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, 774 pnvme_lsreq, gen_req_cmp, ndlp, 2, 775 LPFC_NVME_LS_TIMEOUT, 0); 776 if (ret != WQE_SUCCESS) { 777 lpfc_printf_vlog(vport, KERN_ERR, 778 LOG_NVME_DISC | LOG_NVME_IOERR, 779 "6052 NVMEx REQ: EXIT. issue ls wqe failed " 780 "lsreq x%px Status %x DID %x\n", 781 pnvme_lsreq, ret, ndlp->nlp_DID); 782 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); 783 kfree(bmp); 784 return -EIO; 785 } 786 787 return 0; 788 } 789 790 /** 791 * lpfc_nvme_ls_req - Issue an NVME Link Service request 792 * @lpfc_nvme_lport: Transport localport that LS is to be issued from. 793 * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. 794 * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 795 * 796 * Driver registers this routine to handle any link service request 797 * from the nvme_fc transport to a remote nvme-aware port. 798 * 799 * Return value : 800 * 0 - Success 801 * non-zero: various error codes, in form of -Exxx 802 **/ 803 static int 804 lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, 805 struct nvme_fc_remote_port *pnvme_rport, 806 struct nvmefc_ls_req *pnvme_lsreq) 807 { 808 struct lpfc_nvme_lport *lport; 809 struct lpfc_nvme_rport *rport; 810 struct lpfc_vport *vport; 811 int ret; 812 813 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 814 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 815 if (unlikely(!lport) || unlikely(!rport)) 816 return -EINVAL; 817 818 vport = lport->vport; 819 if (vport->load_flag & FC_UNLOADING) 820 return -ENODEV; 821 822 atomic_inc(&lport->fc4NvmeLsRequests); 823 824 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq, 825 lpfc_nvme_ls_req_cmp); 826 if (ret) 827 atomic_inc(&lport->xmt_ls_err); 828 829 return ret; 830 } 831 832 /** 833 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior 834 * NVME LS request 835 * @vport: The local port that issued the LS 836 * @ndlp: The remote port the LS was sent to 837 * @pnvme_lsreq: Pointer to LS request structure from the transport 838 * 839 * The driver validates the ndlp, looks for the LS, and aborts the 840 * LS if found. 841 * 842 * Returns: 843 * 0 : if LS found and aborted 844 * non-zero: various error conditions in form -Exxx 845 **/ 846 int 847 __lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp, 848 struct nvmefc_ls_req *pnvme_lsreq) 849 { 850 struct lpfc_hba *phba = vport->phba; 851 struct lpfc_sli_ring *pring; 852 struct lpfc_iocbq *wqe, *next_wqe; 853 bool foundit = false; 854 855 if (!ndlp) { 856 lpfc_printf_log(phba, KERN_ERR, 857 LOG_NVME_DISC | LOG_NODE | 858 LOG_NVME_IOERR | LOG_NVME_ABTS, 859 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID " 860 "x%06x, Failing LS Req\n", 861 ndlp, ndlp ? ndlp->nlp_DID : 0); 862 return -EINVAL; 863 } 864 865 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 866 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq " 867 "x%p rqstlen:%d rsplen:%d %pad %pad\n", 868 pnvme_lsreq, pnvme_lsreq->rqstlen, 869 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, 870 &pnvme_lsreq->rspdma); 871 872 /* 873 * Lock the ELS ring txcmplq and look for the wqe that matches 874 * this ELS. If found, issue an abort on the wqe. 875 */ 876 pring = phba->sli4_hba.nvmels_wq->pring; 877 spin_lock_irq(&phba->hbalock); 878 spin_lock(&pring->ring_lock); 879 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { 880 if (wqe->context2 == pnvme_lsreq) { 881 wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 882 foundit = true; 883 break; 884 } 885 } 886 spin_unlock(&pring->ring_lock); 887 888 if (foundit) 889 lpfc_sli_issue_abort_iotag(phba, pring, wqe); 890 spin_unlock_irq(&phba->hbalock); 891 892 if (foundit) 893 return 0; 894 895 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS, 896 "6213 NVMEx LS REQ Abort: Unable to locate req x%p\n", 897 pnvme_lsreq); 898 return -EINVAL; 899 } 900 901 static int 902 lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport, 903 struct nvme_fc_remote_port *remoteport, 904 struct nvmefc_ls_rsp *ls_rsp) 905 { 906 struct lpfc_async_xchg_ctx *axchg = 907 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp); 908 struct lpfc_nvme_lport *lport; 909 int rc; 910 911 if (axchg->phba->pport->load_flag & FC_UNLOADING) 912 return -ENODEV; 913 914 lport = (struct lpfc_nvme_lport *)localport->private; 915 916 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp); 917 918 if (rc) { 919 /* 920 * unless the failure is due to having already sent 921 * the response, an abort will be generated for the 922 * exchange if the rsp can't be sent. 923 */ 924 if (rc != -EALREADY) 925 atomic_inc(&lport->xmt_ls_abort); 926 return rc; 927 } 928 929 return 0; 930 } 931 932 /** 933 * lpfc_nvme_ls_abort - Abort a prior NVME LS request 934 * @lpfc_nvme_lport: Transport localport that LS is to be issued from. 935 * @lpfc_nvme_rport: Transport remoteport that LS is to be sent to. 936 * @pnvme_lsreq - the transport nvme_ls_req structure for the LS 937 * 938 * Driver registers this routine to abort a NVME LS request that is 939 * in progress (from the transports perspective). 940 **/ 941 static void 942 lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, 943 struct nvme_fc_remote_port *pnvme_rport, 944 struct nvmefc_ls_req *pnvme_lsreq) 945 { 946 struct lpfc_nvme_lport *lport; 947 struct lpfc_vport *vport; 948 struct lpfc_hba *phba; 949 struct lpfc_nodelist *ndlp; 950 int ret; 951 952 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 953 if (unlikely(!lport)) 954 return; 955 vport = lport->vport; 956 phba = vport->phba; 957 958 if (vport->load_flag & FC_UNLOADING) 959 return; 960 961 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); 962 963 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq); 964 if (!ret) 965 atomic_inc(&lport->xmt_ls_abort); 966 } 967 968 /* Fix up the existing sgls for NVME IO. */ 969 static inline void 970 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, 971 struct lpfc_io_buf *lpfc_ncmd, 972 struct nvmefc_fcp_req *nCmd) 973 { 974 struct lpfc_hba *phba = vport->phba; 975 struct sli4_sge *sgl; 976 union lpfc_wqe128 *wqe; 977 uint32_t *wptr, *dptr; 978 979 /* 980 * Get a local pointer to the built-in wqe and correct 981 * the cmd size to match NVME's 96 bytes and fix 982 * the dma address. 983 */ 984 985 wqe = &lpfc_ncmd->cur_iocbq.wqe; 986 987 /* 988 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to 989 * match NVME. NVME sends 96 bytes. Also, use the 990 * nvme commands command and response dma addresses 991 * rather than the virtual memory to ease the restore 992 * operation. 993 */ 994 sgl = lpfc_ncmd->dma_sgl; 995 sgl->sge_len = cpu_to_le32(nCmd->cmdlen); 996 if (phba->cfg_nvme_embed_cmd) { 997 sgl->addr_hi = 0; 998 sgl->addr_lo = 0; 999 1000 /* Word 0-2 - NVME CMND IU (embedded payload) */ 1001 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; 1002 wqe->generic.bde.tus.f.bdeSize = 56; 1003 wqe->generic.bde.addrHigh = 0; 1004 wqe->generic.bde.addrLow = 64; /* Word 16 */ 1005 1006 /* Word 10 - dbde is 0, wqes is 1 in template */ 1007 1008 /* 1009 * Embed the payload in the last half of the WQE 1010 * WQE words 16-30 get the NVME CMD IU payload 1011 * 1012 * WQE words 16-19 get payload Words 1-4 1013 * WQE words 20-21 get payload Words 6-7 1014 * WQE words 22-29 get payload Words 16-23 1015 */ 1016 wptr = &wqe->words[16]; /* WQE ptr */ 1017 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ 1018 dptr++; /* Skip Word 0 in payload */ 1019 1020 *wptr++ = *dptr++; /* Word 1 */ 1021 *wptr++ = *dptr++; /* Word 2 */ 1022 *wptr++ = *dptr++; /* Word 3 */ 1023 *wptr++ = *dptr++; /* Word 4 */ 1024 dptr++; /* Skip Word 5 in payload */ 1025 *wptr++ = *dptr++; /* Word 6 */ 1026 *wptr++ = *dptr++; /* Word 7 */ 1027 dptr += 8; /* Skip Words 8-15 in payload */ 1028 *wptr++ = *dptr++; /* Word 16 */ 1029 *wptr++ = *dptr++; /* Word 17 */ 1030 *wptr++ = *dptr++; /* Word 18 */ 1031 *wptr++ = *dptr++; /* Word 19 */ 1032 *wptr++ = *dptr++; /* Word 20 */ 1033 *wptr++ = *dptr++; /* Word 21 */ 1034 *wptr++ = *dptr++; /* Word 22 */ 1035 *wptr = *dptr; /* Word 23 */ 1036 } else { 1037 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma)); 1038 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma)); 1039 1040 /* Word 0-2 - NVME CMND IU Inline BDE */ 1041 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1042 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen; 1043 wqe->generic.bde.addrHigh = sgl->addr_hi; 1044 wqe->generic.bde.addrLow = sgl->addr_lo; 1045 1046 /* Word 10 */ 1047 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); 1048 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0); 1049 } 1050 1051 sgl++; 1052 1053 /* Setup the physical region for the FCP RSP */ 1054 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); 1055 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); 1056 sgl->word2 = le32_to_cpu(sgl->word2); 1057 if (nCmd->sg_cnt) 1058 bf_set(lpfc_sli4_sge_last, sgl, 0); 1059 else 1060 bf_set(lpfc_sli4_sge_last, sgl, 1); 1061 sgl->word2 = cpu_to_le32(sgl->word2); 1062 sgl->sge_len = cpu_to_le32(nCmd->rsplen); 1063 } 1064 1065 1066 /** 1067 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO 1068 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1069 * @lpfc_nvme_lport: Pointer to the driver's local port data 1070 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1071 * 1072 * Driver registers this routine as it io request handler. This 1073 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1074 * data structure to the rport indicated in @lpfc_nvme_rport. 1075 * 1076 * Return value : 1077 * 0 - Success 1078 * TODO: What are the failure codes. 1079 **/ 1080 static void 1081 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, 1082 struct lpfc_wcqe_complete *wcqe) 1083 { 1084 struct lpfc_io_buf *lpfc_ncmd = 1085 (struct lpfc_io_buf *)pwqeIn->context1; 1086 struct lpfc_vport *vport = pwqeIn->vport; 1087 struct nvmefc_fcp_req *nCmd; 1088 struct nvme_fc_ersp_iu *ep; 1089 struct nvme_fc_cmd_iu *cp; 1090 struct lpfc_nodelist *ndlp; 1091 struct lpfc_nvme_fcpreq_priv *freqpriv; 1092 struct lpfc_nvme_lport *lport; 1093 uint32_t code, status, idx; 1094 uint16_t cid, sqhd, data; 1095 uint32_t *ptr; 1096 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1097 int cpu; 1098 #endif 1099 1100 /* Sanity check on return of outstanding command */ 1101 if (!lpfc_ncmd) { 1102 lpfc_printf_vlog(vport, KERN_ERR, 1103 LOG_NODE | LOG_NVME_IOERR, 1104 "6071 Null lpfc_ncmd pointer. No " 1105 "release, skip completion\n"); 1106 return; 1107 } 1108 1109 /* Guard against abort handler being called at same time */ 1110 spin_lock(&lpfc_ncmd->buf_lock); 1111 1112 if (!lpfc_ncmd->nvmeCmd) { 1113 spin_unlock(&lpfc_ncmd->buf_lock); 1114 lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, 1115 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, " 1116 "nvmeCmd x%px\n", 1117 lpfc_ncmd, lpfc_ncmd->nvmeCmd); 1118 1119 /* Release the lpfc_ncmd regardless of the missing elements. */ 1120 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1121 return; 1122 } 1123 nCmd = lpfc_ncmd->nvmeCmd; 1124 status = bf_get(lpfc_wcqe_c_status, wcqe); 1125 1126 idx = lpfc_ncmd->cur_iocbq.hba_wqidx; 1127 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++; 1128 1129 if (unlikely(status && vport->localport)) { 1130 lport = (struct lpfc_nvme_lport *)vport->localport->private; 1131 if (lport) { 1132 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1133 atomic_inc(&lport->cmpl_fcp_xb); 1134 atomic_inc(&lport->cmpl_fcp_err); 1135 } 1136 } 1137 1138 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", 1139 lpfc_ncmd->cur_iocbq.sli4_xritag, 1140 status, wcqe->parameter); 1141 /* 1142 * Catch race where our node has transitioned, but the 1143 * transport is still transitioning. 1144 */ 1145 ndlp = lpfc_ncmd->ndlp; 1146 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1147 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1148 "6062 Ignoring NVME cmpl. No ndlp\n"); 1149 goto out_err; 1150 } 1151 1152 code = bf_get(lpfc_wcqe_c_code, wcqe); 1153 if (code == CQE_CODE_NVME_ERSP) { 1154 /* For this type of CQE, we need to rebuild the rsp */ 1155 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; 1156 1157 /* 1158 * Get Command Id from cmd to plug into response. This 1159 * code is not needed in the next NVME Transport drop. 1160 */ 1161 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; 1162 cid = cp->sqe.common.command_id; 1163 1164 /* 1165 * RSN is in CQE word 2 1166 * SQHD is in CQE Word 3 bits 15:0 1167 * Cmd Specific info is in CQE Word 1 1168 * and in CQE Word 0 bits 15:0 1169 */ 1170 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); 1171 1172 /* Now lets build the NVME ERSP IU */ 1173 ep->iu_len = cpu_to_be16(8); 1174 ep->rsn = wcqe->parameter; 1175 ep->xfrd_len = cpu_to_be32(nCmd->payload_length); 1176 ep->rsvd12 = 0; 1177 ptr = (uint32_t *)&ep->cqe.result.u64; 1178 *ptr++ = wcqe->total_data_placed; 1179 data = bf_get(lpfc_wcqe_c_ersp0, wcqe); 1180 *ptr = (uint32_t)data; 1181 ep->cqe.sq_head = sqhd; 1182 ep->cqe.sq_id = nCmd->sqid; 1183 ep->cqe.command_id = cid; 1184 ep->cqe.status = 0; 1185 1186 lpfc_ncmd->status = IOSTAT_SUCCESS; 1187 lpfc_ncmd->result = 0; 1188 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; 1189 nCmd->transferred_length = nCmd->payload_length; 1190 } else { 1191 lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK); 1192 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK); 1193 1194 /* For NVME, the only failure path that results in an 1195 * IO error is when the adapter rejects it. All other 1196 * conditions are a success case and resolved by the 1197 * transport. 1198 * IOSTAT_FCP_RSP_ERROR means: 1199 * 1. Length of data received doesn't match total 1200 * transfer length in WQE 1201 * 2. If the RSP payload does NOT match these cases: 1202 * a. RSP length 12/24 bytes and all zeros 1203 * b. NVME ERSP 1204 */ 1205 switch (lpfc_ncmd->status) { 1206 case IOSTAT_SUCCESS: 1207 nCmd->transferred_length = wcqe->total_data_placed; 1208 nCmd->rcv_rsplen = 0; 1209 nCmd->status = 0; 1210 break; 1211 case IOSTAT_FCP_RSP_ERROR: 1212 nCmd->transferred_length = wcqe->total_data_placed; 1213 nCmd->rcv_rsplen = wcqe->parameter; 1214 nCmd->status = 0; 1215 /* Sanity check */ 1216 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) 1217 break; 1218 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 1219 "6081 NVME Completion Protocol Error: " 1220 "xri %x status x%x result x%x " 1221 "placed x%x\n", 1222 lpfc_ncmd->cur_iocbq.sli4_xritag, 1223 lpfc_ncmd->status, lpfc_ncmd->result, 1224 wcqe->total_data_placed); 1225 break; 1226 case IOSTAT_LOCAL_REJECT: 1227 /* Let fall through to set command final state. */ 1228 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED) 1229 lpfc_printf_vlog(vport, KERN_INFO, 1230 LOG_NVME_IOERR, 1231 "6032 Delay Aborted cmd x%px " 1232 "nvme cmd x%px, xri x%x, " 1233 "xb %d\n", 1234 lpfc_ncmd, nCmd, 1235 lpfc_ncmd->cur_iocbq.sli4_xritag, 1236 bf_get(lpfc_wcqe_c_xb, wcqe)); 1237 /* fall through */ 1238 default: 1239 out_err: 1240 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1241 "6072 NVME Completion Error: xri %x " 1242 "status x%x result x%x [x%x] " 1243 "placed x%x\n", 1244 lpfc_ncmd->cur_iocbq.sli4_xritag, 1245 lpfc_ncmd->status, lpfc_ncmd->result, 1246 wcqe->parameter, 1247 wcqe->total_data_placed); 1248 nCmd->transferred_length = 0; 1249 nCmd->rcv_rsplen = 0; 1250 nCmd->status = NVME_SC_INTERNAL; 1251 } 1252 } 1253 1254 /* pick up SLI4 exhange busy condition */ 1255 if (bf_get(lpfc_wcqe_c_xb, wcqe)) 1256 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; 1257 else 1258 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; 1259 1260 /* Update stats and complete the IO. There is 1261 * no need for dma unprep because the nvme_transport 1262 * owns the dma address. 1263 */ 1264 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1265 if (lpfc_ncmd->ts_cmd_start) { 1266 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; 1267 lpfc_ncmd->ts_data_io = ktime_get_ns(); 1268 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io; 1269 lpfc_io_ktime(phba, lpfc_ncmd); 1270 } 1271 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) { 1272 cpu = raw_smp_processor_id(); 1273 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io); 1274 if (lpfc_ncmd->cpu != cpu) 1275 lpfc_printf_vlog(vport, 1276 KERN_INFO, LOG_NVME_IOERR, 1277 "6701 CPU Check cmpl: " 1278 "cpu %d expect %d\n", 1279 cpu, lpfc_ncmd->cpu); 1280 } 1281 #endif 1282 1283 /* NVME targets need completion held off until the abort exchange 1284 * completes unless the NVME Rport is getting unregistered. 1285 */ 1286 1287 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 1288 freqpriv = nCmd->private; 1289 freqpriv->nvme_buf = NULL; 1290 lpfc_ncmd->nvmeCmd = NULL; 1291 spin_unlock(&lpfc_ncmd->buf_lock); 1292 nCmd->done(nCmd); 1293 } else 1294 spin_unlock(&lpfc_ncmd->buf_lock); 1295 1296 /* Call release with XB=1 to queue the IO into the abort list. */ 1297 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1298 } 1299 1300 1301 /** 1302 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO 1303 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1304 * @lpfc_nvme_lport: Pointer to the driver's local port data 1305 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1306 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1307 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1308 * 1309 * Driver registers this routine as it io request handler. This 1310 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1311 * data structure to the rport indicated in @lpfc_nvme_rport. 1312 * 1313 * Return value : 1314 * 0 - Success 1315 * TODO: What are the failure codes. 1316 **/ 1317 static int 1318 lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, 1319 struct lpfc_io_buf *lpfc_ncmd, 1320 struct lpfc_nodelist *pnode, 1321 struct lpfc_fc4_ctrl_stat *cstat) 1322 { 1323 struct lpfc_hba *phba = vport->phba; 1324 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1325 struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); 1326 union lpfc_wqe128 *wqe = &pwqeq->wqe; 1327 uint32_t req_len; 1328 1329 if (!NLP_CHK_NODE_ACT(pnode)) 1330 return -EINVAL; 1331 1332 /* 1333 * There are three possibilities here - use scatter-gather segment, use 1334 * the single mapping, or neither. 1335 */ 1336 if (nCmd->sg_cnt) { 1337 if (nCmd->io_dir == NVMEFC_FCP_WRITE) { 1338 /* From the iwrite template, initialize words 7 - 11 */ 1339 memcpy(&wqe->words[7], 1340 &lpfc_iwrite_cmd_template.words[7], 1341 sizeof(uint32_t) * 5); 1342 1343 /* Word 4 */ 1344 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length; 1345 1346 /* Word 5 */ 1347 if ((phba->cfg_nvme_enable_fb) && 1348 (pnode->nlp_flag & NLP_FIRSTBURST)) { 1349 req_len = lpfc_ncmd->nvmeCmd->payload_length; 1350 if (req_len < pnode->nvme_fb_size) 1351 wqe->fcp_iwrite.initial_xfer_len = 1352 req_len; 1353 else 1354 wqe->fcp_iwrite.initial_xfer_len = 1355 pnode->nvme_fb_size; 1356 } else { 1357 wqe->fcp_iwrite.initial_xfer_len = 0; 1358 } 1359 cstat->output_requests++; 1360 } else { 1361 /* From the iread template, initialize words 7 - 11 */ 1362 memcpy(&wqe->words[7], 1363 &lpfc_iread_cmd_template.words[7], 1364 sizeof(uint32_t) * 5); 1365 1366 /* Word 4 */ 1367 wqe->fcp_iread.total_xfer_len = nCmd->payload_length; 1368 1369 /* Word 5 */ 1370 wqe->fcp_iread.rsrvd5 = 0; 1371 1372 cstat->input_requests++; 1373 } 1374 } else { 1375 /* From the icmnd template, initialize words 4 - 11 */ 1376 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4], 1377 sizeof(uint32_t) * 8); 1378 cstat->control_requests++; 1379 } 1380 1381 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) 1382 bf_set(wqe_erp, &wqe->generic.wqe_com, 1); 1383 /* 1384 * Finish initializing those WQE fields that are independent 1385 * of the nvme_cmnd request_buffer 1386 */ 1387 1388 /* Word 3 */ 1389 bf_set(payload_offset_len, &wqe->fcp_icmd, 1390 (nCmd->rsplen + nCmd->cmdlen)); 1391 1392 /* Word 6 */ 1393 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, 1394 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); 1395 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); 1396 1397 /* Word 8 */ 1398 wqe->generic.wqe_com.abort_tag = pwqeq->iotag; 1399 1400 /* Word 9 */ 1401 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); 1402 1403 /* Words 13 14 15 are for PBDE support */ 1404 1405 pwqeq->vport = vport; 1406 return 0; 1407 } 1408 1409 1410 /** 1411 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO 1412 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1413 * @lpfc_nvme_lport: Pointer to the driver's local port data 1414 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1415 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1416 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1417 * 1418 * Driver registers this routine as it io request handler. This 1419 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1420 * data structure to the rport indicated in @lpfc_nvme_rport. 1421 * 1422 * Return value : 1423 * 0 - Success 1424 * TODO: What are the failure codes. 1425 **/ 1426 static int 1427 lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, 1428 struct lpfc_io_buf *lpfc_ncmd) 1429 { 1430 struct lpfc_hba *phba = vport->phba; 1431 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; 1432 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe; 1433 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl; 1434 struct sli4_hybrid_sgl *sgl_xtra = NULL; 1435 struct scatterlist *data_sg; 1436 struct sli4_sge *first_data_sgl; 1437 struct ulp_bde64 *bde; 1438 dma_addr_t physaddr = 0; 1439 uint32_t num_bde = 0; 1440 uint32_t dma_len = 0; 1441 uint32_t dma_offset = 0; 1442 int nseg, i, j; 1443 bool lsp_just_set = false; 1444 1445 /* Fix up the command and response DMA stuff. */ 1446 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); 1447 1448 /* 1449 * There are three possibilities here - use scatter-gather segment, use 1450 * the single mapping, or neither. 1451 */ 1452 if (nCmd->sg_cnt) { 1453 /* 1454 * Jump over the cmd and rsp SGEs. The fix routine 1455 * has already adjusted for this. 1456 */ 1457 sgl += 2; 1458 1459 first_data_sgl = sgl; 1460 lpfc_ncmd->seg_cnt = nCmd->sg_cnt; 1461 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) { 1462 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1463 "6058 Too many sg segments from " 1464 "NVME Transport. Max %d, " 1465 "nvmeIO sg_cnt %d\n", 1466 phba->cfg_nvme_seg_cnt + 1, 1467 lpfc_ncmd->seg_cnt); 1468 lpfc_ncmd->seg_cnt = 0; 1469 return 1; 1470 } 1471 1472 /* 1473 * The driver established a maximum scatter-gather segment count 1474 * during probe that limits the number of sg elements in any 1475 * single nvme command. Just run through the seg_cnt and format 1476 * the sge's. 1477 */ 1478 nseg = nCmd->sg_cnt; 1479 data_sg = nCmd->first_sgl; 1480 1481 /* for tracking the segment boundaries */ 1482 j = 2; 1483 for (i = 0; i < nseg; i++) { 1484 if (data_sg == NULL) { 1485 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1486 "6059 dptr err %d, nseg %d\n", 1487 i, nseg); 1488 lpfc_ncmd->seg_cnt = 0; 1489 return 1; 1490 } 1491 1492 sgl->word2 = 0; 1493 if ((num_bde + 1) == nseg) { 1494 bf_set(lpfc_sli4_sge_last, sgl, 1); 1495 bf_set(lpfc_sli4_sge_type, sgl, 1496 LPFC_SGE_TYPE_DATA); 1497 } else { 1498 bf_set(lpfc_sli4_sge_last, sgl, 0); 1499 1500 /* expand the segment */ 1501 if (!lsp_just_set && 1502 !((j + 1) % phba->border_sge_num) && 1503 ((nseg - 1) != i)) { 1504 /* set LSP type */ 1505 bf_set(lpfc_sli4_sge_type, sgl, 1506 LPFC_SGE_TYPE_LSP); 1507 1508 sgl_xtra = lpfc_get_sgl_per_hdwq( 1509 phba, lpfc_ncmd); 1510 1511 if (unlikely(!sgl_xtra)) { 1512 lpfc_ncmd->seg_cnt = 0; 1513 return 1; 1514 } 1515 sgl->addr_lo = cpu_to_le32(putPaddrLow( 1516 sgl_xtra->dma_phys_sgl)); 1517 sgl->addr_hi = cpu_to_le32(putPaddrHigh( 1518 sgl_xtra->dma_phys_sgl)); 1519 1520 } else { 1521 bf_set(lpfc_sli4_sge_type, sgl, 1522 LPFC_SGE_TYPE_DATA); 1523 } 1524 } 1525 1526 if (!(bf_get(lpfc_sli4_sge_type, sgl) & 1527 LPFC_SGE_TYPE_LSP)) { 1528 if ((nseg - 1) == i) 1529 bf_set(lpfc_sli4_sge_last, sgl, 1); 1530 1531 physaddr = data_sg->dma_address; 1532 dma_len = data_sg->length; 1533 sgl->addr_lo = cpu_to_le32( 1534 putPaddrLow(physaddr)); 1535 sgl->addr_hi = cpu_to_le32( 1536 putPaddrHigh(physaddr)); 1537 1538 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); 1539 sgl->word2 = cpu_to_le32(sgl->word2); 1540 sgl->sge_len = cpu_to_le32(dma_len); 1541 1542 dma_offset += dma_len; 1543 data_sg = sg_next(data_sg); 1544 1545 sgl++; 1546 1547 lsp_just_set = false; 1548 } else { 1549 sgl->word2 = cpu_to_le32(sgl->word2); 1550 1551 sgl->sge_len = cpu_to_le32( 1552 phba->cfg_sg_dma_buf_size); 1553 1554 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl; 1555 i = i - 1; 1556 1557 lsp_just_set = true; 1558 } 1559 1560 j++; 1561 } 1562 if (phba->cfg_enable_pbde) { 1563 /* Use PBDE support for first SGL only, offset == 0 */ 1564 /* Words 13-15 */ 1565 bde = (struct ulp_bde64 *) 1566 &wqe->words[13]; 1567 bde->addrLow = first_data_sgl->addr_lo; 1568 bde->addrHigh = first_data_sgl->addr_hi; 1569 bde->tus.f.bdeSize = 1570 le32_to_cpu(first_data_sgl->sge_len); 1571 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64; 1572 bde->tus.w = cpu_to_le32(bde->tus.w); 1573 /* wqe_pbde is 1 in template */ 1574 } else { 1575 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3)); 1576 bf_set(wqe_pbde, &wqe->generic.wqe_com, 0); 1577 } 1578 1579 } else { 1580 lpfc_ncmd->seg_cnt = 0; 1581 1582 /* For this clause to be valid, the payload_length 1583 * and sg_cnt must zero. 1584 */ 1585 if (nCmd->payload_length != 0) { 1586 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 1587 "6063 NVME DMA Prep Err: sg_cnt %d " 1588 "payload_length x%x\n", 1589 nCmd->sg_cnt, nCmd->payload_length); 1590 return 1; 1591 } 1592 } 1593 return 0; 1594 } 1595 1596 /** 1597 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO 1598 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1599 * @lpfc_nvme_lport: Pointer to the driver's local port data 1600 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1601 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1602 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1603 * 1604 * Driver registers this routine as it io request handler. This 1605 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq 1606 * data structure to the rport 1607 indicated in @lpfc_nvme_rport. 1608 * 1609 * Return value : 1610 * 0 - Success 1611 * TODO: What are the failure codes. 1612 **/ 1613 static int 1614 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, 1615 struct nvme_fc_remote_port *pnvme_rport, 1616 void *hw_queue_handle, 1617 struct nvmefc_fcp_req *pnvme_fcreq) 1618 { 1619 int ret = 0; 1620 int expedite = 0; 1621 int idx, cpu; 1622 struct lpfc_nvme_lport *lport; 1623 struct lpfc_fc4_ctrl_stat *cstat; 1624 struct lpfc_vport *vport; 1625 struct lpfc_hba *phba; 1626 struct lpfc_nodelist *ndlp; 1627 struct lpfc_io_buf *lpfc_ncmd; 1628 struct lpfc_nvme_rport *rport; 1629 struct lpfc_nvme_qhandle *lpfc_queue_info; 1630 struct lpfc_nvme_fcpreq_priv *freqpriv; 1631 struct nvme_common_command *sqe; 1632 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1633 uint64_t start = 0; 1634 #endif 1635 1636 /* Validate pointers. LLDD fault handling with transport does 1637 * have timing races. 1638 */ 1639 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1640 if (unlikely(!lport)) { 1641 ret = -EINVAL; 1642 goto out_fail; 1643 } 1644 1645 vport = lport->vport; 1646 1647 if (unlikely(!hw_queue_handle)) { 1648 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1649 "6117 Fail IO, NULL hw_queue_handle\n"); 1650 atomic_inc(&lport->xmt_fcp_err); 1651 ret = -EBUSY; 1652 goto out_fail; 1653 } 1654 1655 phba = vport->phba; 1656 1657 if (unlikely(vport->load_flag & FC_UNLOADING)) { 1658 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1659 "6124 Fail IO, Driver unload\n"); 1660 atomic_inc(&lport->xmt_fcp_err); 1661 ret = -ENODEV; 1662 goto out_fail; 1663 } 1664 1665 freqpriv = pnvme_fcreq->private; 1666 if (unlikely(!freqpriv)) { 1667 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1668 "6158 Fail IO, NULL request data\n"); 1669 atomic_inc(&lport->xmt_fcp_err); 1670 ret = -EINVAL; 1671 goto out_fail; 1672 } 1673 1674 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1675 if (phba->ktime_on) 1676 start = ktime_get_ns(); 1677 #endif 1678 rport = (struct lpfc_nvme_rport *)pnvme_rport->private; 1679 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; 1680 1681 /* 1682 * Catch race where our node has transitioned, but the 1683 * transport is still transitioning. 1684 */ 1685 ndlp = rport->ndlp; 1686 if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { 1687 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1688 "6053 Busy IO, ndlp not ready: rport x%px " 1689 "ndlp x%px, DID x%06x\n", 1690 rport, ndlp, pnvme_rport->port_id); 1691 atomic_inc(&lport->xmt_fcp_err); 1692 ret = -EBUSY; 1693 goto out_fail; 1694 } 1695 1696 /* The remote node has to be a mapped target or it's an error. */ 1697 if ((ndlp->nlp_type & NLP_NVME_TARGET) && 1698 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { 1699 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR, 1700 "6036 Fail IO, DID x%06x not ready for " 1701 "IO. State x%x, Type x%x Flg x%x\n", 1702 pnvme_rport->port_id, 1703 ndlp->nlp_state, ndlp->nlp_type, 1704 ndlp->upcall_flags); 1705 atomic_inc(&lport->xmt_fcp_bad_ndlp); 1706 ret = -EBUSY; 1707 goto out_fail; 1708 1709 } 1710 1711 /* Currently only NVME Keep alive commands should be expedited 1712 * if the driver runs out of a resource. These should only be 1713 * issued on the admin queue, qidx 0 1714 */ 1715 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) { 1716 sqe = &((struct nvme_fc_cmd_iu *) 1717 pnvme_fcreq->cmdaddr)->sqe.common; 1718 if (sqe->opcode == nvme_admin_keep_alive) 1719 expedite = 1; 1720 } 1721 1722 /* The node is shared with FCP IO, make sure the IO pending count does 1723 * not exceed the programmed depth. 1724 */ 1725 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 1726 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) && 1727 !expedite) { 1728 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1729 "6174 Fail IO, ndlp qdepth exceeded: " 1730 "idx %d DID %x pend %d qdepth %d\n", 1731 lpfc_queue_info->index, ndlp->nlp_DID, 1732 atomic_read(&ndlp->cmd_pending), 1733 ndlp->cmd_qdepth); 1734 atomic_inc(&lport->xmt_fcp_qdepth); 1735 ret = -EBUSY; 1736 goto out_fail; 1737 } 1738 } 1739 1740 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */ 1741 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) { 1742 idx = lpfc_queue_info->index; 1743 } else { 1744 cpu = raw_smp_processor_id(); 1745 idx = phba->sli4_hba.cpu_map[cpu].hdwq; 1746 } 1747 1748 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite); 1749 if (lpfc_ncmd == NULL) { 1750 atomic_inc(&lport->xmt_fcp_noxri); 1751 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1752 "6065 Fail IO, driver buffer pool is empty: " 1753 "idx %d DID %x\n", 1754 lpfc_queue_info->index, ndlp->nlp_DID); 1755 ret = -EBUSY; 1756 goto out_fail; 1757 } 1758 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1759 if (start) { 1760 lpfc_ncmd->ts_cmd_start = start; 1761 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; 1762 } else { 1763 lpfc_ncmd->ts_cmd_start = 0; 1764 } 1765 #endif 1766 1767 /* 1768 * Store the data needed by the driver to issue, abort, and complete 1769 * an IO. 1770 * Do not let the IO hang out forever. There is no midlayer issuing 1771 * an abort so inform the FW of the maximum IO pending time. 1772 */ 1773 freqpriv->nvme_buf = lpfc_ncmd; 1774 lpfc_ncmd->nvmeCmd = pnvme_fcreq; 1775 lpfc_ncmd->ndlp = ndlp; 1776 lpfc_ncmd->qidx = lpfc_queue_info->qidx; 1777 1778 /* 1779 * Issue the IO on the WQ indicated by index in the hw_queue_handle. 1780 * This identfier was create in our hardware queue create callback 1781 * routine. The driver now is dependent on the IO queue steering from 1782 * the transport. We are trusting the upper NVME layers know which 1783 * index to use and that they have affinitized a CPU to this hardware 1784 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. 1785 */ 1786 lpfc_ncmd->cur_iocbq.hba_wqidx = idx; 1787 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat; 1788 1789 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat); 1790 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); 1791 if (ret) { 1792 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1793 "6175 Fail IO, Prep DMA: " 1794 "idx %d DID %x\n", 1795 lpfc_queue_info->index, ndlp->nlp_DID); 1796 atomic_inc(&lport->xmt_fcp_err); 1797 ret = -ENOMEM; 1798 goto out_free_nvme_buf; 1799 } 1800 1801 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", 1802 lpfc_ncmd->cur_iocbq.sli4_xritag, 1803 lpfc_queue_info->index, ndlp->nlp_DID); 1804 1805 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq); 1806 if (ret) { 1807 atomic_inc(&lport->xmt_fcp_wqerr); 1808 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 1809 "6113 Fail IO, Could not issue WQE err %x " 1810 "sid: x%x did: x%x oxid: x%x\n", 1811 ret, vport->fc_myDID, ndlp->nlp_DID, 1812 lpfc_ncmd->cur_iocbq.sli4_xritag); 1813 goto out_free_nvme_buf; 1814 } 1815 1816 if (phba->cfg_xri_rebalancing) 1817 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no); 1818 1819 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS 1820 if (lpfc_ncmd->ts_cmd_start) 1821 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); 1822 1823 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) { 1824 cpu = raw_smp_processor_id(); 1825 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io); 1826 lpfc_ncmd->cpu = cpu; 1827 if (idx != cpu) 1828 lpfc_printf_vlog(vport, 1829 KERN_INFO, LOG_NVME_IOERR, 1830 "6702 CPU Check cmd: " 1831 "cpu %d wq %d\n", 1832 lpfc_ncmd->cpu, 1833 lpfc_queue_info->index); 1834 } 1835 #endif 1836 return 0; 1837 1838 out_free_nvme_buf: 1839 if (lpfc_ncmd->nvmeCmd->sg_cnt) { 1840 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) 1841 cstat->output_requests--; 1842 else 1843 cstat->input_requests--; 1844 } else 1845 cstat->control_requests--; 1846 lpfc_release_nvme_buf(phba, lpfc_ncmd); 1847 out_fail: 1848 return ret; 1849 } 1850 1851 /** 1852 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. 1853 * @phba: Pointer to HBA context object 1854 * @cmdiocb: Pointer to command iocb object. 1855 * @rspiocb: Pointer to response iocb object. 1856 * 1857 * This is the callback function for any NVME FCP IO that was aborted. 1858 * 1859 * Return value: 1860 * None 1861 **/ 1862 void 1863 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, 1864 struct lpfc_wcqe_complete *abts_cmpl) 1865 { 1866 lpfc_printf_log(phba, KERN_INFO, LOG_NVME, 1867 "6145 ABORT_XRI_CN completing on rpi x%x " 1868 "original iotag x%x, abort cmd iotag x%x " 1869 "req_tag x%x, status x%x, hwstatus x%x\n", 1870 cmdiocb->iocb.un.acxri.abortContextTag, 1871 cmdiocb->iocb.un.acxri.abortIoTag, 1872 cmdiocb->iotag, 1873 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), 1874 bf_get(lpfc_wcqe_c_status, abts_cmpl), 1875 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); 1876 lpfc_sli_release_iocbq(phba, cmdiocb); 1877 } 1878 1879 /** 1880 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS 1881 * @lpfc_pnvme: Pointer to the driver's nvme instance data 1882 * @lpfc_nvme_lport: Pointer to the driver's local port data 1883 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq 1884 * @lpfc_nvme_fcreq: IO request from nvme fc to driver. 1885 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue 1886 * 1887 * Driver registers this routine as its nvme request io abort handler. This 1888 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq 1889 * data structure to the rport indicated in @lpfc_nvme_rport. This routine 1890 * is executed asynchronously - one the target is validated as "MAPPED" and 1891 * ready for IO, the driver issues the abort request and returns. 1892 * 1893 * Return value: 1894 * None 1895 **/ 1896 static void 1897 lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, 1898 struct nvme_fc_remote_port *pnvme_rport, 1899 void *hw_queue_handle, 1900 struct nvmefc_fcp_req *pnvme_fcreq) 1901 { 1902 struct lpfc_nvme_lport *lport; 1903 struct lpfc_vport *vport; 1904 struct lpfc_hba *phba; 1905 struct lpfc_io_buf *lpfc_nbuf; 1906 struct lpfc_iocbq *abts_buf; 1907 struct lpfc_iocbq *nvmereq_wqe; 1908 struct lpfc_nvme_fcpreq_priv *freqpriv; 1909 unsigned long flags; 1910 int ret_val; 1911 1912 /* Validate pointers. LLDD fault handling with transport does 1913 * have timing races. 1914 */ 1915 lport = (struct lpfc_nvme_lport *)pnvme_lport->private; 1916 if (unlikely(!lport)) 1917 return; 1918 1919 vport = lport->vport; 1920 1921 if (unlikely(!hw_queue_handle)) { 1922 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1923 "6129 Fail Abort, HW Queue Handle NULL.\n"); 1924 return; 1925 } 1926 1927 phba = vport->phba; 1928 freqpriv = pnvme_fcreq->private; 1929 1930 if (unlikely(!freqpriv)) 1931 return; 1932 if (vport->load_flag & FC_UNLOADING) 1933 return; 1934 1935 /* Announce entry to new IO submit field. */ 1936 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 1937 "6002 Abort Request to rport DID x%06x " 1938 "for nvme_fc_req x%px\n", 1939 pnvme_rport->port_id, 1940 pnvme_fcreq); 1941 1942 /* If the hba is getting reset, this flag is set. It is 1943 * cleared when the reset is complete and rings reestablished. 1944 */ 1945 spin_lock_irqsave(&phba->hbalock, flags); 1946 /* driver queued commands are in process of being flushed */ 1947 if (phba->hba_flag & HBA_IOQ_FLUSH) { 1948 spin_unlock_irqrestore(&phba->hbalock, flags); 1949 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1950 "6139 Driver in reset cleanup - flushing " 1951 "NVME Req now. hba_flag x%x\n", 1952 phba->hba_flag); 1953 return; 1954 } 1955 1956 lpfc_nbuf = freqpriv->nvme_buf; 1957 if (!lpfc_nbuf) { 1958 spin_unlock_irqrestore(&phba->hbalock, flags); 1959 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1960 "6140 NVME IO req has no matching lpfc nvme " 1961 "io buffer. Skipping abort req.\n"); 1962 return; 1963 } else if (!lpfc_nbuf->nvmeCmd) { 1964 spin_unlock_irqrestore(&phba->hbalock, flags); 1965 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1966 "6141 lpfc NVME IO req has no nvme_fcreq " 1967 "io buffer. Skipping abort req.\n"); 1968 return; 1969 } 1970 nvmereq_wqe = &lpfc_nbuf->cur_iocbq; 1971 1972 /* Guard against IO completion being called at same time */ 1973 spin_lock(&lpfc_nbuf->buf_lock); 1974 1975 /* 1976 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's 1977 * state must match the nvme_fcreq passed by the nvme 1978 * transport. If they don't match, it is likely the driver 1979 * has already completed the NVME IO and the nvme transport 1980 * has not seen it yet. 1981 */ 1982 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { 1983 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1984 "6143 NVME req mismatch: " 1985 "lpfc_nbuf x%px nvmeCmd x%px, " 1986 "pnvme_fcreq x%px. Skipping Abort xri x%x\n", 1987 lpfc_nbuf, lpfc_nbuf->nvmeCmd, 1988 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1989 goto out_unlock; 1990 } 1991 1992 /* Don't abort IOs no longer on the pending queue. */ 1993 if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { 1994 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 1995 "6142 NVME IO req x%px not queued - skipping " 1996 "abort req xri x%x\n", 1997 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 1998 goto out_unlock; 1999 } 2000 2001 atomic_inc(&lport->xmt_fcp_abort); 2002 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", 2003 nvmereq_wqe->sli4_xritag, 2004 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); 2005 2006 /* Outstanding abort is in progress */ 2007 if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { 2008 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2009 "6144 Outstanding NVME I/O Abort Request " 2010 "still pending on nvme_fcreq x%px, " 2011 "lpfc_ncmd %px xri x%x\n", 2012 pnvme_fcreq, lpfc_nbuf, 2013 nvmereq_wqe->sli4_xritag); 2014 goto out_unlock; 2015 } 2016 2017 abts_buf = __lpfc_sli_get_iocbq(phba); 2018 if (!abts_buf) { 2019 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2020 "6136 No available abort wqes. Skipping " 2021 "Abts req for nvme_fcreq x%px xri x%x\n", 2022 pnvme_fcreq, nvmereq_wqe->sli4_xritag); 2023 goto out_unlock; 2024 } 2025 2026 /* Ready - mark outstanding as aborted by driver. */ 2027 nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; 2028 2029 lpfc_nvme_prep_abort_wqe(abts_buf, nvmereq_wqe->sli4_xritag, 0); 2030 2031 /* ABTS WQE must go to the same WQ as the WQE to be aborted */ 2032 abts_buf->iocb_flag |= LPFC_IO_NVME; 2033 abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; 2034 abts_buf->vport = vport; 2035 abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; 2036 ret_val = lpfc_sli4_issue_wqe(phba, lpfc_nbuf->hdwq, abts_buf); 2037 spin_unlock(&lpfc_nbuf->buf_lock); 2038 spin_unlock_irqrestore(&phba->hbalock, flags); 2039 if (ret_val) { 2040 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, 2041 "6137 Failed abts issue_wqe with status x%x " 2042 "for nvme_fcreq x%px.\n", 2043 ret_val, pnvme_fcreq); 2044 lpfc_sli_release_iocbq(phba, abts_buf); 2045 return; 2046 } 2047 2048 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, 2049 "6138 Transport Abort NVME Request Issued for " 2050 "ox_id x%x on reqtag x%x\n", 2051 nvmereq_wqe->sli4_xritag, 2052 abts_buf->iotag); 2053 return; 2054 2055 out_unlock: 2056 spin_unlock(&lpfc_nbuf->buf_lock); 2057 spin_unlock_irqrestore(&phba->hbalock, flags); 2058 return; 2059 } 2060 2061 /* Declare and initialization an instance of the FC NVME template. */ 2062 static struct nvme_fc_port_template lpfc_nvme_template = { 2063 /* initiator-based functions */ 2064 .localport_delete = lpfc_nvme_localport_delete, 2065 .remoteport_delete = lpfc_nvme_remoteport_delete, 2066 .create_queue = lpfc_nvme_create_queue, 2067 .delete_queue = lpfc_nvme_delete_queue, 2068 .ls_req = lpfc_nvme_ls_req, 2069 .fcp_io = lpfc_nvme_fcp_io_submit, 2070 .ls_abort = lpfc_nvme_ls_abort, 2071 .fcp_abort = lpfc_nvme_fcp_abort, 2072 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp, 2073 2074 .max_hw_queues = 1, 2075 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2076 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, 2077 .dma_boundary = 0xFFFFFFFF, 2078 2079 /* Sizes of additional private data for data structures. 2080 * No use for the last two sizes at this time. 2081 */ 2082 .local_priv_sz = sizeof(struct lpfc_nvme_lport), 2083 .remote_priv_sz = sizeof(struct lpfc_nvme_rport), 2084 .lsrqst_priv_sz = 0, 2085 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), 2086 }; 2087 2088 /** 2089 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA 2090 * @phba: The HBA for which this call is being executed. 2091 * 2092 * This routine removes a nvme buffer from head of @hdwq io_buf_list 2093 * and returns to caller. 2094 * 2095 * Return codes: 2096 * NULL - Error 2097 * Pointer to lpfc_nvme_buf - Success 2098 **/ 2099 static struct lpfc_io_buf * 2100 lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp, 2101 int idx, int expedite) 2102 { 2103 struct lpfc_io_buf *lpfc_ncmd; 2104 struct lpfc_sli4_hdw_queue *qp; 2105 struct sli4_sge *sgl; 2106 struct lpfc_iocbq *pwqeq; 2107 union lpfc_wqe128 *wqe; 2108 2109 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite); 2110 2111 if (lpfc_ncmd) { 2112 pwqeq = &(lpfc_ncmd->cur_iocbq); 2113 wqe = &pwqeq->wqe; 2114 2115 /* Setup key fields in buffer that may have been changed 2116 * if other protocols used this buffer. 2117 */ 2118 pwqeq->iocb_flag = LPFC_IO_NVME; 2119 pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; 2120 lpfc_ncmd->start_time = jiffies; 2121 lpfc_ncmd->flags = 0; 2122 2123 /* Rsp SGE will be filled in when we rcv an IO 2124 * from the NVME Layer to be sent. 2125 * The cmd is going to be embedded so we need a SKIP SGE. 2126 */ 2127 sgl = lpfc_ncmd->dma_sgl; 2128 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); 2129 bf_set(lpfc_sli4_sge_last, sgl, 0); 2130 sgl->word2 = cpu_to_le32(sgl->word2); 2131 /* Fill in word 3 / sgl_len during cmd submission */ 2132 2133 /* Initialize 64 bytes only */ 2134 memset(wqe, 0, sizeof(union lpfc_wqe)); 2135 2136 if (lpfc_ndlp_check_qdepth(phba, ndlp)) { 2137 atomic_inc(&ndlp->cmd_pending); 2138 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH; 2139 } 2140 2141 } else { 2142 qp = &phba->sli4_hba.hdwq[idx]; 2143 qp->empty_io_bufs++; 2144 } 2145 2146 return lpfc_ncmd; 2147 } 2148 2149 /** 2150 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. 2151 * @phba: The Hba for which this call is being executed. 2152 * @lpfc_ncmd: The nvme buffer which is being released. 2153 * 2154 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba 2155 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer 2156 * and cannot be reused for at least RA_TOV amount of time if it was 2157 * aborted. 2158 **/ 2159 static void 2160 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd) 2161 { 2162 struct lpfc_sli4_hdw_queue *qp; 2163 unsigned long iflag = 0; 2164 2165 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp) 2166 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending); 2167 2168 lpfc_ncmd->ndlp = NULL; 2169 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH; 2170 2171 qp = lpfc_ncmd->hdwq; 2172 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) { 2173 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2174 "6310 XB release deferred for " 2175 "ox_id x%x on reqtag x%x\n", 2176 lpfc_ncmd->cur_iocbq.sli4_xritag, 2177 lpfc_ncmd->cur_iocbq.iotag); 2178 2179 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag); 2180 list_add_tail(&lpfc_ncmd->list, 2181 &qp->lpfc_abts_io_buf_list); 2182 qp->abts_nvme_io_bufs++; 2183 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag); 2184 } else 2185 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp); 2186 } 2187 2188 /** 2189 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. 2190 * @pvport - the lpfc_vport instance requesting a localport. 2191 * 2192 * This routine is invoked to create an nvme localport instance to bind 2193 * to the nvme_fc_transport. It is called once during driver load 2194 * like lpfc_create_shost after all other services are initialized. 2195 * It requires a vport, vpi, and wwns at call time. Other localport 2196 * parameters are modified as the driver's FCID and the Fabric WWN 2197 * are established. 2198 * 2199 * Return codes 2200 * 0 - successful 2201 * -ENOMEM - no heap memory available 2202 * other values - from nvme registration upcall 2203 **/ 2204 int 2205 lpfc_nvme_create_localport(struct lpfc_vport *vport) 2206 { 2207 int ret = 0; 2208 struct lpfc_hba *phba = vport->phba; 2209 struct nvme_fc_port_info nfcp_info; 2210 struct nvme_fc_local_port *localport; 2211 struct lpfc_nvme_lport *lport; 2212 2213 /* Initialize this localport instance. The vport wwn usage ensures 2214 * that NPIV is accounted for. 2215 */ 2216 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); 2217 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; 2218 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); 2219 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); 2220 2221 /* We need to tell the transport layer + 1 because it takes page 2222 * alignment into account. When space for the SGL is allocated we 2223 * allocate + 3, one for cmd, one for rsp and one for this alignment 2224 */ 2225 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; 2226 2227 /* Advertise how many hw queues we support based on cfg_hdw_queue, 2228 * which will not exceed cpu count. 2229 */ 2230 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue; 2231 2232 if (!IS_ENABLED(CONFIG_NVME_FC)) 2233 return ret; 2234 2235 /* localport is allocated from the stack, but the registration 2236 * call allocates heap memory as well as the private area. 2237 */ 2238 2239 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, 2240 &vport->phba->pcidev->dev, &localport); 2241 if (!ret) { 2242 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, 2243 "6005 Successfully registered local " 2244 "NVME port num %d, localP x%px, private " 2245 "x%px, sg_seg %d\n", 2246 localport->port_num, localport, 2247 localport->private, 2248 lpfc_nvme_template.max_sgl_segments); 2249 2250 /* Private is our lport size declared in the template. */ 2251 lport = (struct lpfc_nvme_lport *)localport->private; 2252 vport->localport = localport; 2253 lport->vport = vport; 2254 vport->nvmei_support = 1; 2255 2256 atomic_set(&lport->xmt_fcp_noxri, 0); 2257 atomic_set(&lport->xmt_fcp_bad_ndlp, 0); 2258 atomic_set(&lport->xmt_fcp_qdepth, 0); 2259 atomic_set(&lport->xmt_fcp_err, 0); 2260 atomic_set(&lport->xmt_fcp_wqerr, 0); 2261 atomic_set(&lport->xmt_fcp_abort, 0); 2262 atomic_set(&lport->xmt_ls_abort, 0); 2263 atomic_set(&lport->xmt_ls_err, 0); 2264 atomic_set(&lport->cmpl_fcp_xb, 0); 2265 atomic_set(&lport->cmpl_fcp_err, 0); 2266 atomic_set(&lport->cmpl_ls_xb, 0); 2267 atomic_set(&lport->cmpl_ls_err, 0); 2268 2269 atomic_set(&lport->fc4NvmeLsRequests, 0); 2270 atomic_set(&lport->fc4NvmeLsCmpls, 0); 2271 } 2272 2273 return ret; 2274 } 2275 2276 #if (IS_ENABLED(CONFIG_NVME_FC)) 2277 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg. 2278 * 2279 * The driver has to wait for the host nvme transport to callback 2280 * indicating the localport has successfully unregistered all 2281 * resources. Since this is an uninterruptible wait, loop every ten 2282 * seconds and print a message indicating no progress. 2283 * 2284 * An uninterruptible wait is used because of the risk of transport-to- 2285 * driver state mismatch. 2286 */ 2287 static void 2288 lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport, 2289 struct lpfc_nvme_lport *lport, 2290 struct completion *lport_unreg_cmp) 2291 { 2292 u32 wait_tmo; 2293 int ret, i, pending = 0; 2294 struct lpfc_sli_ring *pring; 2295 struct lpfc_hba *phba = vport->phba; 2296 2297 /* Host transport has to clean up and confirm requiring an indefinite 2298 * wait. Print a message if a 10 second wait expires and renew the 2299 * wait. This is unexpected. 2300 */ 2301 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000); 2302 while (true) { 2303 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo); 2304 if (unlikely(!ret)) { 2305 pending = 0; 2306 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2307 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2308 if (!pring) 2309 continue; 2310 if (pring->txcmplq_cnt) 2311 pending += pring->txcmplq_cnt; 2312 } 2313 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, 2314 "6176 Lport x%px Localport x%px wait " 2315 "timed out. Pending %d. Renewing.\n", 2316 lport, vport->localport, pending); 2317 continue; 2318 } 2319 break; 2320 } 2321 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, 2322 "6177 Lport x%px Localport x%px Complete Success\n", 2323 lport, vport->localport); 2324 } 2325 #endif 2326 2327 /** 2328 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. 2329 * @pnvme: pointer to lpfc nvme data structure. 2330 * 2331 * This routine is invoked to destroy all lports bound to the phba. 2332 * The lport memory was allocated by the nvme fc transport and is 2333 * released there. This routine ensures all rports bound to the 2334 * lport have been disconnected. 2335 * 2336 **/ 2337 void 2338 lpfc_nvme_destroy_localport(struct lpfc_vport *vport) 2339 { 2340 #if (IS_ENABLED(CONFIG_NVME_FC)) 2341 struct nvme_fc_local_port *localport; 2342 struct lpfc_nvme_lport *lport; 2343 int ret; 2344 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp); 2345 2346 if (vport->nvmei_support == 0) 2347 return; 2348 2349 localport = vport->localport; 2350 lport = (struct lpfc_nvme_lport *)localport->private; 2351 2352 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2353 "6011 Destroying NVME localport x%px\n", 2354 localport); 2355 2356 /* lport's rport list is clear. Unregister 2357 * lport and release resources. 2358 */ 2359 lport->lport_unreg_cmp = &lport_unreg_cmp; 2360 ret = nvme_fc_unregister_localport(localport); 2361 2362 /* Wait for completion. This either blocks 2363 * indefinitely or succeeds 2364 */ 2365 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp); 2366 vport->localport = NULL; 2367 2368 /* Regardless of the unregister upcall response, clear 2369 * nvmei_support. All rports are unregistered and the 2370 * driver will clean up. 2371 */ 2372 vport->nvmei_support = 0; 2373 if (ret == 0) { 2374 lpfc_printf_vlog(vport, 2375 KERN_INFO, LOG_NVME_DISC, 2376 "6009 Unregistered lport Success\n"); 2377 } else { 2378 lpfc_printf_vlog(vport, 2379 KERN_INFO, LOG_NVME_DISC, 2380 "6010 Unregistered lport " 2381 "Failed, status x%x\n", 2382 ret); 2383 } 2384 #endif 2385 } 2386 2387 void 2388 lpfc_nvme_update_localport(struct lpfc_vport *vport) 2389 { 2390 #if (IS_ENABLED(CONFIG_NVME_FC)) 2391 struct nvme_fc_local_port *localport; 2392 struct lpfc_nvme_lport *lport; 2393 2394 localport = vport->localport; 2395 if (!localport) { 2396 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2397 "6710 Update NVME fail. No localport\n"); 2398 return; 2399 } 2400 lport = (struct lpfc_nvme_lport *)localport->private; 2401 if (!lport) { 2402 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, 2403 "6171 Update NVME fail. localP x%px, No lport\n", 2404 localport); 2405 return; 2406 } 2407 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, 2408 "6012 Update NVME lport x%px did x%x\n", 2409 localport, vport->fc_myDID); 2410 2411 localport->port_id = vport->fc_myDID; 2412 if (localport->port_id == 0) 2413 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; 2414 else 2415 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; 2416 2417 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2418 "6030 bound lport x%px to DID x%06x\n", 2419 lport, localport->port_id); 2420 #endif 2421 } 2422 2423 int 2424 lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2425 { 2426 #if (IS_ENABLED(CONFIG_NVME_FC)) 2427 int ret = 0; 2428 struct nvme_fc_local_port *localport; 2429 struct lpfc_nvme_lport *lport; 2430 struct lpfc_nvme_rport *rport; 2431 struct lpfc_nvme_rport *oldrport; 2432 struct nvme_fc_remote_port *remote_port; 2433 struct nvme_fc_port_info rpinfo; 2434 struct lpfc_nodelist *prev_ndlp = NULL; 2435 2436 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, 2437 "6006 Register NVME PORT. DID x%06x nlptype x%x\n", 2438 ndlp->nlp_DID, ndlp->nlp_type); 2439 2440 localport = vport->localport; 2441 if (!localport) 2442 return 0; 2443 2444 lport = (struct lpfc_nvme_lport *)localport->private; 2445 2446 /* NVME rports are not preserved across devloss. 2447 * Just register this instance. Note, rpinfo->dev_loss_tmo 2448 * is left 0 to indicate accept transport defaults. The 2449 * driver communicates port role capabilities consistent 2450 * with the PRLI response data. 2451 */ 2452 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); 2453 rpinfo.port_id = ndlp->nlp_DID; 2454 if (ndlp->nlp_type & NLP_NVME_TARGET) 2455 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; 2456 if (ndlp->nlp_type & NLP_NVME_INITIATOR) 2457 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; 2458 2459 if (ndlp->nlp_type & NLP_NVME_DISCOVERY) 2460 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; 2461 2462 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); 2463 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); 2464 2465 spin_lock_irq(&vport->phba->hbalock); 2466 oldrport = lpfc_ndlp_get_nrport(ndlp); 2467 if (oldrport) { 2468 prev_ndlp = oldrport->ndlp; 2469 spin_unlock_irq(&vport->phba->hbalock); 2470 } else { 2471 spin_unlock_irq(&vport->phba->hbalock); 2472 lpfc_nlp_get(ndlp); 2473 } 2474 2475 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); 2476 if (!ret) { 2477 /* If the ndlp already has an nrport, this is just 2478 * a resume of the existing rport. Else this is a 2479 * new rport. 2480 */ 2481 /* Guard against an unregister/reregister 2482 * race that leaves the WAIT flag set. 2483 */ 2484 spin_lock_irq(&vport->phba->hbalock); 2485 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2486 spin_unlock_irq(&vport->phba->hbalock); 2487 rport = remote_port->private; 2488 if (oldrport) { 2489 2490 /* Sever the ndlp<->rport association 2491 * before dropping the ndlp ref from 2492 * register. 2493 */ 2494 spin_lock_irq(&vport->phba->hbalock); 2495 ndlp->nrport = NULL; 2496 ndlp->upcall_flags &= ~NLP_WAIT_FOR_UNREG; 2497 spin_unlock_irq(&vport->phba->hbalock); 2498 rport->ndlp = NULL; 2499 rport->remoteport = NULL; 2500 2501 /* Reference only removed if previous NDLP is no longer 2502 * active. It might be just a swap and removing the 2503 * reference would cause a premature cleanup. 2504 */ 2505 if (prev_ndlp && prev_ndlp != ndlp) { 2506 if ((!NLP_CHK_NODE_ACT(prev_ndlp)) || 2507 (!prev_ndlp->nrport)) 2508 lpfc_nlp_put(prev_ndlp); 2509 } 2510 } 2511 2512 /* Clean bind the rport to the ndlp. */ 2513 rport->remoteport = remote_port; 2514 rport->lport = lport; 2515 rport->ndlp = ndlp; 2516 spin_lock_irq(&vport->phba->hbalock); 2517 ndlp->nrport = rport; 2518 spin_unlock_irq(&vport->phba->hbalock); 2519 lpfc_printf_vlog(vport, KERN_INFO, 2520 LOG_NVME_DISC | LOG_NODE, 2521 "6022 Bind lport x%px to remoteport x%px " 2522 "rport x%px WWNN 0x%llx, " 2523 "Rport WWPN 0x%llx DID " 2524 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n", 2525 lport, remote_port, rport, 2526 rpinfo.node_name, rpinfo.port_name, 2527 rpinfo.port_id, rpinfo.port_role, 2528 ndlp, prev_ndlp); 2529 } else { 2530 lpfc_printf_vlog(vport, KERN_ERR, 2531 LOG_NVME_DISC | LOG_NODE, 2532 "6031 RemotePort Registration failed " 2533 "err: %d, DID x%06x\n", 2534 ret, ndlp->nlp_DID); 2535 } 2536 2537 return ret; 2538 #else 2539 return 0; 2540 #endif 2541 } 2542 2543 /** 2544 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport 2545 * 2546 * If the ndlp represents an NVME Target, that we are logged into, 2547 * ping the NVME FC Transport layer to initiate a device rescan 2548 * on this remote NPort. 2549 */ 2550 void 2551 lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2552 { 2553 #if (IS_ENABLED(CONFIG_NVME_FC)) 2554 struct lpfc_nvme_rport *nrport; 2555 struct nvme_fc_remote_port *remoteport = NULL; 2556 2557 spin_lock_irq(&vport->phba->hbalock); 2558 nrport = lpfc_ndlp_get_nrport(ndlp); 2559 if (nrport) 2560 remoteport = nrport->remoteport; 2561 spin_unlock_irq(&vport->phba->hbalock); 2562 2563 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2564 "6170 Rescan NPort DID x%06x type x%x " 2565 "state x%x nrport x%px remoteport x%px\n", 2566 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state, 2567 nrport, remoteport); 2568 2569 if (!nrport || !remoteport) 2570 goto rescan_exit; 2571 2572 /* Only rescan if we are an NVME target in the MAPPED state */ 2573 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY && 2574 ndlp->nlp_state == NLP_STE_MAPPED_NODE) { 2575 nvme_fc_rescan_remoteport(remoteport); 2576 2577 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2578 "6172 NVME rescanned DID x%06x " 2579 "port_state x%x\n", 2580 ndlp->nlp_DID, remoteport->port_state); 2581 } 2582 return; 2583 rescan_exit: 2584 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2585 "6169 Skip NVME Rport Rescan, NVME remoteport " 2586 "unregistered\n"); 2587 #endif 2588 } 2589 2590 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. 2591 * 2592 * There is no notion of Devloss or rport recovery from the current 2593 * nvme_transport perspective. Loss of an rport just means IO cannot 2594 * be sent and recovery is completely up to the initator. 2595 * For now, the driver just unbinds the DID and port_role so that 2596 * no further IO can be issued. Changes are planned for later. 2597 * 2598 * Notes - the ndlp reference count is not decremented here since 2599 * since there is no nvme_transport api for devloss. Node ref count 2600 * is only adjusted in driver unload. 2601 */ 2602 void 2603 lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) 2604 { 2605 #if (IS_ENABLED(CONFIG_NVME_FC)) 2606 int ret; 2607 struct nvme_fc_local_port *localport; 2608 struct lpfc_nvme_lport *lport; 2609 struct lpfc_nvme_rport *rport; 2610 struct nvme_fc_remote_port *remoteport = NULL; 2611 2612 localport = vport->localport; 2613 2614 /* This is fundamental error. The localport is always 2615 * available until driver unload. Just exit. 2616 */ 2617 if (!localport) 2618 return; 2619 2620 lport = (struct lpfc_nvme_lport *)localport->private; 2621 if (!lport) 2622 goto input_err; 2623 2624 spin_lock_irq(&vport->phba->hbalock); 2625 rport = lpfc_ndlp_get_nrport(ndlp); 2626 if (rport) 2627 remoteport = rport->remoteport; 2628 spin_unlock_irq(&vport->phba->hbalock); 2629 if (!remoteport) 2630 goto input_err; 2631 2632 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, 2633 "6033 Unreg nvme remoteport x%px, portname x%llx, " 2634 "port_id x%06x, portstate x%x port type x%x\n", 2635 remoteport, remoteport->port_name, 2636 remoteport->port_id, remoteport->port_state, 2637 ndlp->nlp_type); 2638 2639 /* Sanity check ndlp type. Only call for NVME ports. Don't 2640 * clear any rport state until the transport calls back. 2641 */ 2642 2643 if (ndlp->nlp_type & NLP_NVME_TARGET) { 2644 /* No concern about the role change on the nvme remoteport. 2645 * The transport will update it. 2646 */ 2647 ndlp->upcall_flags |= NLP_WAIT_FOR_UNREG; 2648 2649 /* Don't let the host nvme transport keep sending keep-alives 2650 * on this remoteport. Vport is unloading, no recovery. The 2651 * return values is ignored. The upcall is a courtesy to the 2652 * transport. 2653 */ 2654 if (vport->load_flag & FC_UNLOADING) 2655 (void)nvme_fc_set_remoteport_devloss(remoteport, 0); 2656 2657 ret = nvme_fc_unregister_remoteport(remoteport); 2658 if (ret != 0) { 2659 lpfc_nlp_put(ndlp); 2660 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2661 "6167 NVME unregister failed %d " 2662 "port_state x%x\n", 2663 ret, remoteport->port_state); 2664 } 2665 } 2666 return; 2667 2668 input_err: 2669 #endif 2670 lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, 2671 "6168 State error: lport x%px, rport x%px FCID x%06x\n", 2672 vport->localport, ndlp->rport, ndlp->nlp_DID); 2673 } 2674 2675 /** 2676 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort 2677 * @phba: pointer to lpfc hba data structure. 2678 * @axri: pointer to the fcp xri abort wcqe structure. 2679 * @lpfc_ncmd: The nvme job structure for the request being aborted. 2680 * 2681 * This routine is invoked by the worker thread to process a SLI4 fast-path 2682 * NVME aborted xri. Aborted NVME IO commands are completed to the transport 2683 * here. 2684 **/ 2685 void 2686 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, 2687 struct sli4_wcqe_xri_aborted *axri, 2688 struct lpfc_io_buf *lpfc_ncmd) 2689 { 2690 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); 2691 struct nvmefc_fcp_req *nvme_cmd = NULL; 2692 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp; 2693 2694 2695 if (ndlp) 2696 lpfc_sli4_abts_err_handler(phba, ndlp, axri); 2697 2698 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, 2699 "6311 nvme_cmd %p xri x%x tag x%x abort complete and " 2700 "xri released\n", 2701 lpfc_ncmd->nvmeCmd, xri, 2702 lpfc_ncmd->cur_iocbq.iotag); 2703 2704 /* Aborted NVME commands are required to not complete 2705 * before the abort exchange command fully completes. 2706 * Once completed, it is available via the put list. 2707 */ 2708 if (lpfc_ncmd->nvmeCmd) { 2709 nvme_cmd = lpfc_ncmd->nvmeCmd; 2710 nvme_cmd->done(nvme_cmd); 2711 lpfc_ncmd->nvmeCmd = NULL; 2712 } 2713 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2714 } 2715 2716 /** 2717 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete 2718 * @phba: Pointer to HBA context object. 2719 * 2720 * This function flushes all wqes in the nvme rings and frees all resources 2721 * in the txcmplq. This function does not issue abort wqes for the IO 2722 * commands in txcmplq, they will just be returned with 2723 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI 2724 * slot has been permanently disabled. 2725 **/ 2726 void 2727 lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba) 2728 { 2729 struct lpfc_sli_ring *pring; 2730 u32 i, wait_cnt = 0; 2731 2732 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq) 2733 return; 2734 2735 /* Cycle through all IO rings and make sure all outstanding 2736 * WQEs have been removed from the txcmplqs. 2737 */ 2738 for (i = 0; i < phba->cfg_hdw_queue; i++) { 2739 if (!phba->sli4_hba.hdwq[i].io_wq) 2740 continue; 2741 pring = phba->sli4_hba.hdwq[i].io_wq->pring; 2742 2743 if (!pring) 2744 continue; 2745 2746 /* Retrieve everything on the txcmplq */ 2747 while (!list_empty(&pring->txcmplq)) { 2748 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1); 2749 wait_cnt++; 2750 2751 /* The sleep is 10mS. Every ten seconds, 2752 * dump a message. Something is wrong. 2753 */ 2754 if ((wait_cnt % 1000) == 0) { 2755 lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, 2756 "6178 NVME IO not empty, " 2757 "cnt %d\n", wait_cnt); 2758 } 2759 } 2760 } 2761 } 2762 2763 void 2764 lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn) 2765 { 2766 #if (IS_ENABLED(CONFIG_NVME_FC)) 2767 struct lpfc_io_buf *lpfc_ncmd; 2768 struct nvmefc_fcp_req *nCmd; 2769 struct lpfc_nvme_fcpreq_priv *freqpriv; 2770 2771 if (!pwqeIn->context1) { 2772 lpfc_sli_release_iocbq(phba, pwqeIn); 2773 return; 2774 } 2775 /* For abort iocb just return, IO iocb will do a done call */ 2776 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) == 2777 CMD_ABORT_XRI_CX) { 2778 lpfc_sli_release_iocbq(phba, pwqeIn); 2779 return; 2780 } 2781 lpfc_ncmd = (struct lpfc_io_buf *)pwqeIn->context1; 2782 2783 spin_lock(&lpfc_ncmd->buf_lock); 2784 if (!lpfc_ncmd->nvmeCmd) { 2785 spin_unlock(&lpfc_ncmd->buf_lock); 2786 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2787 return; 2788 } 2789 2790 nCmd = lpfc_ncmd->nvmeCmd; 2791 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR, 2792 "6194 NVME Cancel xri %x\n", 2793 lpfc_ncmd->cur_iocbq.sli4_xritag); 2794 2795 nCmd->transferred_length = 0; 2796 nCmd->rcv_rsplen = 0; 2797 nCmd->status = NVME_SC_INTERNAL; 2798 freqpriv = nCmd->private; 2799 freqpriv->nvme_buf = NULL; 2800 lpfc_ncmd->nvmeCmd = NULL; 2801 2802 spin_unlock(&lpfc_ncmd->buf_lock); 2803 nCmd->done(nCmd); 2804 2805 /* Call release with XB=1 to queue the IO into the abort list. */ 2806 lpfc_release_nvme_buf(phba, lpfc_ncmd); 2807 #endif 2808 } 2809